3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
31 from subprocess
import CalledProcessError
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self
, returncode
, cmd
):
40 self
.returncode
= returncode
43 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize
= 512
53 def p4_build_cmd(cmd
):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
62 user
= gitConfig("git-p4.user")
64 real_cmd
+= ["-u",user
]
66 password
= gitConfig("git-p4.password")
68 real_cmd
+= ["-P", password
]
70 port
= gitConfig("git-p4.port")
72 real_cmd
+= ["-p", port
]
74 host
= gitConfig("git-p4.host")
76 real_cmd
+= ["-H", host
]
78 client
= gitConfig("git-p4.client")
80 real_cmd
+= ["-c", client
]
82 retries
= gitConfigInt("git-p4.retries")
84 # Perform 3 retries by default
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd
+= ["-r", str(retries
)]
90 if isinstance(cmd
,basestring
):
91 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
100 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
101 if not d
or len(d
) == 0:
106 def chdir(path
, is_client_path
=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
124 if not is_client_path
:
126 os
.environ
['PWD'] = path
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform
.system() == 'Windows':
131 free_bytes
= ctypes
.c_ulonglong(0)
132 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
133 return free_bytes
.value
135 st
= os
.statvfs(os
.getcwd())
136 return st
.f_bavail
* st
.f_frsize
142 sys
.stderr
.write(msg
+ "\n")
145 def write_pipe(c
, stdin
):
147 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
149 expand
= isinstance(c
,basestring
)
150 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
152 val
= pipe
.write(stdin
)
155 die('Command failed: %s' % str(c
))
159 def p4_write_pipe(c
, stdin
):
160 real_cmd
= p4_build_cmd(c
)
161 return write_pipe(real_cmd
, stdin
)
163 def read_pipe_full(c
):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
169 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
171 expand
= isinstance(c
,basestring
)
172 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
173 (out
, err
) = p
.communicate()
174 return (p
.returncode
, out
, err
)
176 def read_pipe(c
, ignore_error
=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
181 (retcode
, out
, err
) = read_pipe_full(c
)
186 die('Command failed: %s\nError: %s' % (str(c
), err
))
189 def read_pipe_text(c
):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
193 (retcode
, out
, err
) = read_pipe_full(c
)
199 def p4_read_pipe(c
, ignore_error
=False):
200 real_cmd
= p4_build_cmd(c
)
201 return read_pipe(real_cmd
, ignore_error
)
203 def read_pipe_lines(c
):
205 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
207 expand
= isinstance(c
, basestring
)
208 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
210 val
= pipe
.readlines()
211 if pipe
.close() or p
.wait():
212 die('Command failed: %s' % str(c
))
216 def p4_read_pipe_lines(c
):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd
= p4_build_cmd(c
)
219 return read_pipe_lines(real_cmd
)
221 def p4_has_command(cmd
):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd
= p4_build_cmd(["help", cmd
])
225 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
226 stderr
=subprocess
.PIPE
)
228 return p
.returncode
== 0
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
236 if not p4_has_command("move"):
238 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
239 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
240 (out
, err
) = p
.communicate()
241 # return code will be 1 in either case
242 if err
.find("Invalid option") >= 0:
244 if err
.find("disabled") >= 0:
246 # assume it failed because @... was invalid changelist
249 def system(cmd
, ignore_error
=False):
250 expand
= isinstance(cmd
,basestring
)
252 sys
.stderr
.write("executing %s\n" % str(cmd
))
253 retcode
= subprocess
.call(cmd
, shell
=expand
)
254 if retcode
and not ignore_error
:
255 raise CalledProcessError(retcode
, cmd
)
260 """Specifically invoke p4 as the system command. """
261 real_cmd
= p4_build_cmd(cmd
)
262 expand
= isinstance(real_cmd
, basestring
)
263 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
265 raise CalledProcessError(retcode
, real_cmd
)
267 _p4_version_string
= None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
277 global _p4_version_string
278 if not _p4_version_string
:
279 a
= p4_read_pipe_lines(["-V"])
280 _p4_version_string
= a
[-1].rstrip()
281 return _p4_version_string
283 def p4_integrate(src
, dest
):
284 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
286 def p4_sync(f
, *options
):
287 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
290 # forcibly add file names with wildcards
291 if wildcard_present(f
):
292 p4_system(["add", "-f", f
])
294 p4_system(["add", f
])
297 p4_system(["delete", wildcard_encode(f
)])
299 def p4_edit(f
, *options
):
300 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
303 p4_system(["revert", wildcard_encode(f
)])
305 def p4_reopen(type, f
):
306 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
308 def p4_reopen_in_change(changelist
, files
):
309 cmd
= ["reopen", "-c", str(changelist
)] + files
312 def p4_move(src
, dest
):
313 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
315 def p4_last_change():
316 results
= p4CmdList(["changes", "-m", "1"])
317 return int(results
[0]['change'])
319 def p4_describe(change
):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
324 ds
= p4CmdList(["describe", "-s", str(change
)])
326 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
330 if "p4ExitCode" in d
:
331 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
334 if d
["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
338 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers. See "p4 help filetypes"
345 # for a list and explanation.
347 def split_p4_type(p4type
):
349 p4_filetypes_historical
= {
350 "ctempobj": "binary+Sw",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
362 "xtempobj": "binary+Swx",
364 "xunicode": "unicode+x",
367 if p4type
in p4_filetypes_historical
:
368 p4type
= p4_filetypes_historical
[p4type
]
370 s
= p4type
.split("+")
378 # return the raw p4 type of a file (text, text+ko, etc)
381 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
382 return results
[0]['headType']
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
388 def p4_keywords_regexp_for_type(base
, type_mods
):
389 if base
in ("text", "unicode", "binary"):
391 if "ko" in type_mods
:
393 elif "k" in type_mods
:
394 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
400 (:[^$\n]+)? # possibly an old expansion, followed by...
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
412 def p4_keywords_regexp_for_file(file):
413 if not os
.path
.exists(file):
416 (type_base
, type_mods
) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base
, type_mods
)
419 def setP4ExecBit(file, mode
):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
425 if not isModeExec(mode
):
426 p4Type
= getP4OpenedType(file)
427 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
428 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
429 if p4Type
[-1] == "+":
430 p4Type
= p4Type
[0:-1]
432 p4_reopen(p4Type
, file)
434 def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
437 result
= p4_read_pipe(["opened", wildcard_encode(file)])
438 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
440 return match
.group(1)
442 die("Could not determine file type for %s (result: '%s')" % (file, result
))
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths
):
447 if isinstance(depotPaths
,basestring
):
448 depotPaths
= [depotPaths
]
450 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
456 # Return the set of all git tags
459 for line
in read_pipe_lines(["git", "tag"]):
464 def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
471 def parseDiffTreeEntry(entry
):
472 """Parses a single diff tree entry into its component elements.
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
488 If the pattern is not matched, None is returned."""
490 match
= diffTreePattern().next().match(entry
)
493 'src_mode': match
.group(1),
494 'dst_mode': match
.group(2),
495 'src_sha1': match
.group(3),
496 'dst_sha1': match
.group(4),
497 'status': match
.group(5),
498 'status_score': match
.group(6),
499 'src': match
.group(7),
500 'dst': match
.group(10)
504 def isModeExec(mode
):
505 # Returns True if the given git mode represents an executable file,
507 return mode
[-3:] == "755"
509 def isModeExecChanged(src_mode
, dst_mode
):
510 return isModeExec(src_mode
) != isModeExec(dst_mode
)
512 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
514 if isinstance(cmd
,basestring
):
521 cmd
= p4_build_cmd(cmd
)
523 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
529 if stdin
is not None:
530 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
531 if isinstance(stdin
,basestring
):
532 stdin_file
.write(stdin
)
535 stdin_file
.write(i
+ '\n')
539 p4
= subprocess
.Popen(cmd
,
542 stdout
=subprocess
.PIPE
)
547 entry
= marshal
.load(p4
.stdout
)
557 entry
["p4ExitCode"] = exitCode
563 list = p4CmdList(cmd
)
569 def p4Where(depotPath
):
570 if not depotPath
.endswith("/"):
572 depotPathLong
= depotPath
+ "..."
573 outputList
= p4CmdList(["where", depotPathLong
])
575 for entry
in outputList
:
576 if "depotFile" in entry
:
577 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
578 # The base path always ends with "/...".
579 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
582 elif "data" in entry
:
583 data
= entry
.get("data")
584 space
= data
.find(" ")
585 if data
[:space
] == depotPath
:
590 if output
["code"] == "error":
594 clientPath
= output
.get("path")
595 elif "data" in output
:
596 data
= output
.get("data")
597 lastSpace
= data
.rfind(" ")
598 clientPath
= data
[lastSpace
+ 1:]
600 if clientPath
.endswith("..."):
601 clientPath
= clientPath
[:-3]
604 def currentGitBranch():
605 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
610 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
612 def isValidGitDir(path
):
613 return git_dir(path
) != None
615 def parseRevision(ref
):
616 return read_pipe("git rev-parse %s" % ref
).strip()
618 def branchExists(ref
):
619 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
623 def extractLogMessageFromGitCommit(commit
):
626 ## fixme: title is first line of commit, not 1st paragraph.
628 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
637 def extractSettingsGitLog(log
):
639 for line
in log
.split("\n"):
641 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
645 assignments
= m
.group(1).split (':')
646 for a
in assignments
:
648 key
= vals
[0].strip()
649 val
= ('='.join (vals
[1:])).strip()
650 if val
.endswith ('\"') and val
.startswith('"'):
655 paths
= values
.get("depot-paths")
657 paths
= values
.get("depot-path")
659 values
['depot-paths'] = paths
.split(',')
662 def gitBranchExists(branch
):
663 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
664 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
665 return proc
.wait() == 0;
669 def gitConfig(key
, typeSpecifier
=None):
670 if not _gitConfig
.has_key(key
):
671 cmd
= [ "git", "config" ]
673 cmd
+= [ typeSpecifier
]
675 s
= read_pipe(cmd
, ignore_error
=True)
676 _gitConfig
[key
] = s
.strip()
677 return _gitConfig
[key
]
679 def gitConfigBool(key
):
680 """Return a bool, using git config --bool. It is True only if the
681 variable is set to true, and False if set to false or not present
684 if not _gitConfig
.has_key(key
):
685 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
686 return _gitConfig
[key
]
688 def gitConfigInt(key
):
689 if not _gitConfig
.has_key(key
):
690 cmd
= [ "git", "config", "--int", key
]
691 s
= read_pipe(cmd
, ignore_error
=True)
694 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
696 _gitConfig
[key
] = None
697 return _gitConfig
[key
]
699 def gitConfigList(key
):
700 if not _gitConfig
.has_key(key
):
701 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
702 _gitConfig
[key
] = s
.strip().splitlines()
703 if _gitConfig
[key
] == ['']:
705 return _gitConfig
[key
]
707 def p4BranchesInGit(branchesAreInRemotes
=True):
708 """Find all the branches whose names start with "p4/", looking
709 in remotes or heads as specified by the argument. Return
710 a dictionary of { branch: revision } for each one found.
711 The branch names are the short names, without any
716 cmdline
= "git rev-parse --symbolic "
717 if branchesAreInRemotes
:
718 cmdline
+= "--remotes"
720 cmdline
+= "--branches"
722 for line
in read_pipe_lines(cmdline
):
726 if not line
.startswith('p4/'):
728 # special symbolic ref to p4/master
729 if line
== "p4/HEAD":
732 # strip off p4/ prefix
733 branch
= line
[len("p4/"):]
735 branches
[branch
] = parseRevision(line
)
739 def branch_exists(branch
):
740 """Make sure that the given ref name really exists."""
742 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
743 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
744 out
, _
= p
.communicate()
747 # expect exactly one line of output: the branch name
748 return out
.rstrip() == branch
750 def findUpstreamBranchPoint(head
= "HEAD"):
751 branches
= p4BranchesInGit()
752 # map from depot-path to branch name
753 branchByDepotPath
= {}
754 for branch
in branches
.keys():
755 tip
= branches
[branch
]
756 log
= extractLogMessageFromGitCommit(tip
)
757 settings
= extractSettingsGitLog(log
)
758 if settings
.has_key("depot-paths"):
759 paths
= ",".join(settings
["depot-paths"])
760 branchByDepotPath
[paths
] = "remotes/p4/" + branch
764 while parent
< 65535:
765 commit
= head
+ "~%s" % parent
766 log
= extractLogMessageFromGitCommit(commit
)
767 settings
= extractSettingsGitLog(log
)
768 if settings
.has_key("depot-paths"):
769 paths
= ",".join(settings
["depot-paths"])
770 if branchByDepotPath
.has_key(paths
):
771 return [branchByDepotPath
[paths
], settings
]
775 return ["", settings
]
777 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
779 print ("Creating/updating branch(es) in %s based on origin branch(es)"
782 originPrefix
= "origin/p4/"
784 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
786 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
789 headName
= line
[len(originPrefix
):]
790 remoteHead
= localRefPrefix
+ headName
793 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
794 if (not original
.has_key('depot-paths')
795 or not original
.has_key('change')):
799 if not gitBranchExists(remoteHead
):
801 print "creating %s" % remoteHead
804 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
805 if settings
.has_key('change') > 0:
806 if settings
['depot-paths'] == original
['depot-paths']:
807 originP4Change
= int(original
['change'])
808 p4Change
= int(settings
['change'])
809 if originP4Change
> p4Change
:
810 print ("%s (%s) is newer than %s (%s). "
811 "Updating p4 branch from origin."
812 % (originHead
, originP4Change
,
813 remoteHead
, p4Change
))
816 print ("Ignoring: %s was imported from %s while "
817 "%s was imported from %s"
818 % (originHead
, ','.join(original
['depot-paths']),
819 remoteHead
, ','.join(settings
['depot-paths'])))
822 system("git update-ref %s %s" % (remoteHead
, originHead
))
824 def originP4BranchesExist():
825 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
828 def p4ParseNumericChangeRange(parts
):
829 changeStart
= int(parts
[0][1:])
830 if parts
[1] == '#head':
831 changeEnd
= p4_last_change()
833 changeEnd
= int(parts
[1])
835 return (changeStart
, changeEnd
)
837 def chooseBlockSize(blockSize
):
841 return defaultBlockSize
843 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
846 # Parse the change range into start and end. Try to find integer
847 # revision ranges as these can be broken up into blocks to avoid
848 # hitting server-side limits (maxrows, maxscanresults). But if
849 # that doesn't work, fall back to using the raw revision specifier
850 # strings, without using block mode.
852 if changeRange
is None or changeRange
== '':
854 changeEnd
= p4_last_change()
855 block_size
= chooseBlockSize(requestedBlockSize
)
857 parts
= changeRange
.split(',')
858 assert len(parts
) == 2
860 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
861 block_size
= chooseBlockSize(requestedBlockSize
)
863 changeStart
= parts
[0][1:]
865 if requestedBlockSize
:
866 die("cannot use --changes-block-size with non-numeric revisions")
871 # Retrieve changes a block at a time, to prevent running
872 # into a MaxResults/MaxScanRows error from the server.
878 end
= min(changeEnd
, changeStart
+ block_size
)
879 revisionRange
= "%d,%d" % (changeStart
, end
)
881 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
884 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
886 # Insert changes in chronological order
887 for line
in reversed(p4_read_pipe_lines(cmd
)):
888 changes
.add(int(line
.split(" ")[1]))
896 changeStart
= end
+ 1
898 changes
= sorted(changes
)
901 def p4PathStartsWith(path
, prefix
):
902 # This method tries to remedy a potential mixed-case issue:
904 # If UserA adds //depot/DirA/file1
905 # and UserB adds //depot/dira/file2
907 # we may or may not have a problem. If you have core.ignorecase=true,
908 # we treat DirA and dira as the same directory
909 if gitConfigBool("core.ignorecase"):
910 return path
.lower().startswith(prefix
.lower())
911 return path
.startswith(prefix
)
914 """Look at the p4 client spec, create a View() object that contains
915 all the mappings, and return it."""
917 specList
= p4CmdList("client -o")
918 if len(specList
) != 1:
919 die('Output from "client -o" is %d lines, expecting 1' %
922 # dictionary of all client parameters
926 client_name
= entry
["Client"]
928 # just the keys that start with "View"
929 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
932 view
= View(client_name
)
934 # append the lines, in order, to the view
935 for view_num
in range(len(view_keys
)):
936 k
= "View%d" % view_num
937 if k
not in view_keys
:
938 die("Expected view key %s missing" % k
)
939 view
.append(entry
[k
])
944 """Grab the client directory."""
946 output
= p4CmdList("client -o")
948 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
951 if "Root" not in entry
:
952 die('Client has no "Root"')
957 # P4 wildcards are not allowed in filenames. P4 complains
958 # if you simply add them, but you can force it with "-f", in
959 # which case it translates them into %xx encoding internally.
961 def wildcard_decode(path
):
962 # Search for and fix just these four characters. Do % last so
963 # that fixing it does not inadvertently create new %-escapes.
964 # Cannot have * in a filename in windows; untested as to
965 # what p4 would do in such a case.
966 if not platform
.system() == "Windows":
967 path
= path
.replace("%2A", "*")
968 path
= path
.replace("%23", "#") \
969 .replace("%40", "@") \
973 def wildcard_encode(path
):
974 # do % first to avoid double-encoding the %s introduced here
975 path
= path
.replace("%", "%25") \
976 .replace("*", "%2A") \
977 .replace("#", "%23") \
981 def wildcard_present(path
):
982 m
= re
.search("[*#@%]", path
)
985 class LargeFileSystem(object):
986 """Base class for large file system support."""
988 def __init__(self
, writeToGitStream
):
989 self
.largeFiles
= set()
990 self
.writeToGitStream
= writeToGitStream
992 def generatePointer(self
, cloneDestination
, contentFile
):
993 """Return the content of a pointer file that is stored in Git instead of
994 the actual content."""
995 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
997 def pushFile(self
, localLargeFile
):
998 """Push the actual content which is not stored in the Git repository to
1000 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
1002 def hasLargeFileExtension(self
, relPath
):
1004 lambda a
, b
: a
or b
,
1005 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
1009 def generateTempFile(self
, contents
):
1010 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1012 contentFile
.write(d
)
1014 return contentFile
.name
1016 def exceedsLargeFileThreshold(self
, relPath
, contents
):
1017 if gitConfigInt('git-p4.largeFileThreshold'):
1018 contentsSize
= sum(len(d
) for d
in contents
)
1019 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
1021 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1022 contentsSize
= sum(len(d
) for d
in contents
)
1023 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1025 contentTempFile
= self
.generateTempFile(contents
)
1026 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
1027 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
1028 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
1030 compressedContentsSize
= zf
.infolist()[0].compress_size
1031 os
.remove(contentTempFile
)
1032 os
.remove(compressedContentFile
.name
)
1033 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1037 def addLargeFile(self
, relPath
):
1038 self
.largeFiles
.add(relPath
)
1040 def removeLargeFile(self
, relPath
):
1041 self
.largeFiles
.remove(relPath
)
1043 def isLargeFile(self
, relPath
):
1044 return relPath
in self
.largeFiles
1046 def processContent(self
, git_mode
, relPath
, contents
):
1047 """Processes the content of git fast import. This method decides if a
1048 file is stored in the large file system and handles all necessary
1050 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1051 contentTempFile
= self
.generateTempFile(contents
)
1052 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1053 if pointer_git_mode
:
1054 git_mode
= pointer_git_mode
1056 # Move temp file to final location in large file system
1057 largeFileDir
= os
.path
.dirname(localLargeFile
)
1058 if not os
.path
.isdir(largeFileDir
):
1059 os
.makedirs(largeFileDir
)
1060 shutil
.move(contentTempFile
, localLargeFile
)
1061 self
.addLargeFile(relPath
)
1062 if gitConfigBool('git-p4.largeFilePush'):
1063 self
.pushFile(localLargeFile
)
1065 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1066 return (git_mode
, contents
)
1068 class MockLFS(LargeFileSystem
):
1069 """Mock large file system for testing."""
1071 def generatePointer(self
, contentFile
):
1072 """The pointer content is the original content prefixed with "pointer-".
1073 The local filename of the large file storage is derived from the file content.
1075 with
open(contentFile
, 'r') as f
:
1078 pointerContents
= 'pointer-' + content
1079 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1080 return (gitMode
, pointerContents
, localLargeFile
)
1082 def pushFile(self
, localLargeFile
):
1083 """The remote filename of the large file storage is the same as the local
1084 one but in a different directory.
1086 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1087 if not os
.path
.exists(remotePath
):
1088 os
.makedirs(remotePath
)
1089 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1091 class GitLFS(LargeFileSystem
):
1092 """Git LFS as backend for the git-p4 large file system.
1093 See https://git-lfs.github.com/ for details."""
1095 def __init__(self
, *args
):
1096 LargeFileSystem
.__init
__(self
, *args
)
1097 self
.baseGitAttributes
= []
1099 def generatePointer(self
, contentFile
):
1100 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1101 mode and content which is stored in the Git repository instead of
1102 the actual content. Return also the new location of the actual
1105 if os
.path
.getsize(contentFile
) == 0:
1106 return (None, '', None)
1108 pointerProcess
= subprocess
.Popen(
1109 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1110 stdout
=subprocess
.PIPE
1112 pointerFile
= pointerProcess
.stdout
.read()
1113 if pointerProcess
.wait():
1114 os
.remove(contentFile
)
1115 die('git-lfs pointer command failed. Did you install the extension?')
1117 # Git LFS removed the preamble in the output of the 'pointer' command
1118 # starting from version 1.2.0. Check for the preamble here to support
1120 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1121 if pointerFile
.startswith('Git LFS pointer for'):
1122 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1124 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1125 localLargeFile
= os
.path
.join(
1127 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1130 # LFS Spec states that pointer files should not have the executable bit set.
1132 return (gitMode
, pointerFile
, localLargeFile
)
1134 def pushFile(self
, localLargeFile
):
1135 uploadProcess
= subprocess
.Popen(
1136 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1138 if uploadProcess
.wait():
1139 die('git-lfs push command failed. Did you define a remote?')
1141 def generateGitAttributes(self
):
1143 self
.baseGitAttributes
+
1147 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1151 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1153 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1154 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1158 def addLargeFile(self
, relPath
):
1159 LargeFileSystem
.addLargeFile(self
, relPath
)
1160 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1162 def removeLargeFile(self
, relPath
):
1163 LargeFileSystem
.removeLargeFile(self
, relPath
)
1164 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1166 def processContent(self
, git_mode
, relPath
, contents
):
1167 if relPath
== '.gitattributes':
1168 self
.baseGitAttributes
= contents
1169 return (git_mode
, self
.generateGitAttributes())
1171 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1175 self
.usage
= "usage: %prog [options]"
1176 self
.needsGit
= True
1177 self
.verbose
= False
1181 self
.userMapFromPerforceServer
= False
1182 self
.myP4UserId
= None
1186 return self
.myP4UserId
1188 results
= p4CmdList("user -o")
1190 if r
.has_key('User'):
1191 self
.myP4UserId
= r
['User']
1193 die("Could not find your p4 user id")
1195 def p4UserIsMe(self
, p4User
):
1196 # return True if the given p4 user is actually me
1197 me
= self
.p4UserId()
1198 if not p4User
or p4User
!= me
:
1203 def getUserCacheFilename(self
):
1204 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1205 return home
+ "/.gitp4-usercache.txt"
1207 def getUserMapFromPerforceServer(self
):
1208 if self
.userMapFromPerforceServer
:
1213 for output
in p4CmdList("users"):
1214 if not output
.has_key("User"):
1216 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1217 self
.emails
[output
["Email"]] = output
["User"]
1219 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1220 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1221 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1222 if mapUser
and len(mapUser
[0]) == 3:
1223 user
= mapUser
[0][0]
1224 fullname
= mapUser
[0][1]
1225 email
= mapUser
[0][2]
1226 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1227 self
.emails
[email
] = user
1230 for (key
, val
) in self
.users
.items():
1231 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1233 open(self
.getUserCacheFilename(), "wb").write(s
)
1234 self
.userMapFromPerforceServer
= True
1236 def loadUserMapFromCache(self
):
1238 self
.userMapFromPerforceServer
= False
1240 cache
= open(self
.getUserCacheFilename(), "rb")
1241 lines
= cache
.readlines()
1244 entry
= line
.strip().split("\t")
1245 self
.users
[entry
[0]] = entry
[1]
1247 self
.getUserMapFromPerforceServer()
1249 class P4Debug(Command
):
1251 Command
.__init
__(self
)
1253 self
.description
= "A tool to debug the output of p4 -G."
1254 self
.needsGit
= False
1256 def run(self
, args
):
1258 for output
in p4CmdList(args
):
1259 print 'Element: %d' % j
1264 class P4RollBack(Command
):
1266 Command
.__init
__(self
)
1268 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1270 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1271 self
.rollbackLocalBranches
= False
1273 def run(self
, args
):
1276 maxChange
= int(args
[0])
1278 if "p4ExitCode" in p4Cmd("changes -m 1"):
1279 die("Problems executing p4");
1281 if self
.rollbackLocalBranches
:
1282 refPrefix
= "refs/heads/"
1283 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1285 refPrefix
= "refs/remotes/"
1286 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1289 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1291 ref
= refPrefix
+ line
1292 log
= extractLogMessageFromGitCommit(ref
)
1293 settings
= extractSettingsGitLog(log
)
1295 depotPaths
= settings
['depot-paths']
1296 change
= settings
['change']
1300 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1301 for p
in depotPaths
]))) == 0:
1302 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1303 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1306 while change
and int(change
) > maxChange
:
1309 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1310 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1311 log
= extractLogMessageFromGitCommit(ref
)
1312 settings
= extractSettingsGitLog(log
)
1315 depotPaths
= settings
['depot-paths']
1316 change
= settings
['change']
1319 print "%s rewound to %s" % (ref
, change
)
1323 class P4Submit(Command
, P4UserMap
):
1325 conflict_behavior_choices
= ("ask", "skip", "quit")
1328 Command
.__init
__(self
)
1329 P4UserMap
.__init
__(self
)
1331 optparse
.make_option("--origin", dest
="origin"),
1332 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1333 # preserve the user, requires relevant p4 permissions
1334 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1335 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1336 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1337 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1338 optparse
.make_option("--conflict", dest
="conflict_behavior",
1339 choices
=self
.conflict_behavior_choices
),
1340 optparse
.make_option("--branch", dest
="branch"),
1341 optparse
.make_option("--shelve", dest
="shelve", action
="store_true",
1342 help="Shelve instead of submit. Shelved files are reverted, "
1343 "restoring the workspace to the state before the shelve"),
1344 optparse
.make_option("--update-shelve", dest
="update_shelve", action
="store", type="int",
1345 metavar
="CHANGELIST",
1346 help="update an existing shelved changelist, implies --shelve")
1348 self
.description
= "Submit changes from git to the perforce depot."
1349 self
.usage
+= " [name of git branch to submit into perforce depot]"
1351 self
.detectRenames
= False
1352 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1353 self
.dry_run
= False
1355 self
.update_shelve
= None
1356 self
.prepare_p4_only
= False
1357 self
.conflict_behavior
= None
1358 self
.isWindows
= (platform
.system() == "Windows")
1359 self
.exportLabels
= False
1360 self
.p4HasMoveCommand
= p4_has_move_command()
1363 if gitConfig('git-p4.largeFileSystem'):
1364 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1367 if len(p4CmdList("opened ...")) > 0:
1368 die("You have files opened with perforce! Close them before starting the sync.")
1370 def separate_jobs_from_description(self
, message
):
1371 """Extract and return a possible Jobs field in the commit
1372 message. It goes into a separate section in the p4 change
1375 A jobs line starts with "Jobs:" and looks like a new field
1376 in a form. Values are white-space separated on the same
1377 line or on following lines that start with a tab.
1379 This does not parse and extract the full git commit message
1380 like a p4 form. It just sees the Jobs: line as a marker
1381 to pass everything from then on directly into the p4 form,
1382 but outside the description section.
1384 Return a tuple (stripped log message, jobs string)."""
1386 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1388 return (message
, None)
1390 jobtext
= message
[m
.start():]
1391 stripped_message
= message
[:m
.start()].rstrip()
1392 return (stripped_message
, jobtext
)
1394 def prepareLogMessage(self
, template
, message
, jobs
):
1395 """Edits the template returned from "p4 change -o" to insert
1396 the message in the Description field, and the jobs text in
1400 inDescriptionSection
= False
1402 for line
in template
.split("\n"):
1403 if line
.startswith("#"):
1404 result
+= line
+ "\n"
1407 if inDescriptionSection
:
1408 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1409 inDescriptionSection
= False
1410 # insert Jobs section
1412 result
+= jobs
+ "\n"
1416 if line
.startswith("Description:"):
1417 inDescriptionSection
= True
1419 for messageLine
in message
.split("\n"):
1420 line
+= "\t" + messageLine
+ "\n"
1422 result
+= line
+ "\n"
1426 def patchRCSKeywords(self
, file, pattern
):
1427 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1428 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1430 outFile
= os
.fdopen(handle
, "w+")
1431 inFile
= open(file, "r")
1432 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1433 for line
in inFile
.readlines():
1434 line
= regexp
.sub(r
'$\1$', line
)
1438 # Forcibly overwrite the original file
1440 shutil
.move(outFileName
, file)
1442 # cleanup our temporary file
1443 os
.unlink(outFileName
)
1444 print "Failed to strip RCS keywords in %s" % file
1447 print "Patched up RCS keywords in %s" % file
1449 def p4UserForCommit(self
,id):
1450 # Return the tuple (perforce user,git email) for a given git commit id
1451 self
.getUserMapFromPerforceServer()
1452 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1453 "--format=%ae", id])
1454 gitEmail
= gitEmail
.strip()
1455 if not self
.emails
.has_key(gitEmail
):
1456 return (None,gitEmail
)
1458 return (self
.emails
[gitEmail
],gitEmail
)
1460 def checkValidP4Users(self
,commits
):
1461 # check if any git authors cannot be mapped to p4 users
1463 (user
,email
) = self
.p4UserForCommit(id)
1465 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1466 if gitConfigBool("git-p4.allowMissingP4Users"):
1469 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1471 def lastP4Changelist(self
):
1472 # Get back the last changelist number submitted in this client spec. This
1473 # then gets used to patch up the username in the change. If the same
1474 # client spec is being used by multiple processes then this might go
1476 results
= p4CmdList("client -o") # find the current client
1479 if r
.has_key('Client'):
1480 client
= r
['Client']
1483 die("could not get client spec")
1484 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1486 if r
.has_key('change'):
1488 die("Could not get changelist number for last submit - cannot patch up user details")
1490 def modifyChangelistUser(self
, changelist
, newUser
):
1491 # fixup the user field of a changelist after it has been submitted.
1492 changes
= p4CmdList("change -o %s" % changelist
)
1493 if len(changes
) != 1:
1494 die("Bad output from p4 change modifying %s to user %s" %
1495 (changelist
, newUser
))
1498 if c
['User'] == newUser
: return # nothing to do
1500 input = marshal
.dumps(c
)
1502 result
= p4CmdList("change -f -i", stdin
=input)
1504 if r
.has_key('code'):
1505 if r
['code'] == 'error':
1506 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1507 if r
.has_key('data'):
1508 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1510 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1512 def canChangeChangelists(self
):
1513 # check to see if we have p4 admin or super-user permissions, either of
1514 # which are required to modify changelists.
1515 results
= p4CmdList(["protects", self
.depotPath
])
1517 if r
.has_key('perm'):
1518 if r
['perm'] == 'admin':
1520 if r
['perm'] == 'super':
1524 def prepareSubmitTemplate(self
, changelist
=None):
1525 """Run "p4 change -o" to grab a change specification template.
1526 This does not use "p4 -G", as it is nice to keep the submission
1527 template in original order, since a human might edit it.
1529 Remove lines in the Files section that show changes to files
1530 outside the depot path we're committing into."""
1532 [upstream
, settings
] = findUpstreamBranchPoint()
1535 inFilesSection
= False
1536 args
= ['change', '-o']
1538 args
.append(str(changelist
))
1540 for line
in p4_read_pipe_lines(args
):
1541 if line
.endswith("\r\n"):
1542 line
= line
[:-2] + "\n"
1544 if line
.startswith("\t"):
1545 # path starts and ends with a tab
1547 lastTab
= path
.rfind("\t")
1549 path
= path
[:lastTab
]
1550 if settings
.has_key('depot-paths'):
1551 if not [p
for p
in settings
['depot-paths']
1552 if p4PathStartsWith(path
, p
)]:
1555 if not p4PathStartsWith(path
, self
.depotPath
):
1558 inFilesSection
= False
1560 if line
.startswith("Files:"):
1561 inFilesSection
= True
1567 def edit_template(self
, template_file
):
1568 """Invoke the editor to let the user change the submission
1569 message. Return true if okay to continue with the submit."""
1571 # if configured to skip the editing part, just submit
1572 if gitConfigBool("git-p4.skipSubmitEdit"):
1575 # look at the modification time, to check later if the user saved
1577 mtime
= os
.stat(template_file
).st_mtime
1580 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1581 editor
= os
.environ
.get("P4EDITOR")
1583 editor
= read_pipe("git var GIT_EDITOR").strip()
1584 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1586 # If the file was not saved, prompt to see if this patch should
1587 # be skipped. But skip this verification step if configured so.
1588 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1591 # modification time updated means user saved the file
1592 if os
.stat(template_file
).st_mtime
> mtime
:
1596 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1602 def get_diff_description(self
, editedFiles
, filesToAdd
, symlinks
):
1604 if os
.environ
.has_key("P4DIFF"):
1605 del(os
.environ
["P4DIFF"])
1607 for editedFile
in editedFiles
:
1608 diff
+= p4_read_pipe(['diff', '-du',
1609 wildcard_encode(editedFile
)])
1613 for newFile
in filesToAdd
:
1614 newdiff
+= "==== new file ====\n"
1615 newdiff
+= "--- /dev/null\n"
1616 newdiff
+= "+++ %s\n" % newFile
1618 is_link
= os
.path
.islink(newFile
)
1619 expect_link
= newFile
in symlinks
1621 if is_link
and expect_link
:
1622 newdiff
+= "+%s\n" % os
.readlink(newFile
)
1624 f
= open(newFile
, "r")
1625 for line
in f
.readlines():
1626 newdiff
+= "+" + line
1629 return (diff
+ newdiff
).replace('\r\n', '\n')
1631 def applyCommit(self
, id):
1632 """Apply one commit, return True if it succeeded."""
1634 print "Applying", read_pipe(["git", "show", "-s",
1635 "--format=format:%h %s", id])
1637 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1639 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1641 filesToChangeType
= set()
1642 filesToDelete
= set()
1644 pureRenameCopy
= set()
1646 filesToChangeExecBit
= {}
1650 diff
= parseDiffTreeEntry(line
)
1651 modifier
= diff
['status']
1653 all_files
.append(path
)
1657 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1658 filesToChangeExecBit
[path
] = diff
['dst_mode']
1659 editedFiles
.add(path
)
1660 elif modifier
== "A":
1661 filesToAdd
.add(path
)
1662 filesToChangeExecBit
[path
] = diff
['dst_mode']
1663 if path
in filesToDelete
:
1664 filesToDelete
.remove(path
)
1666 dst_mode
= int(diff
['dst_mode'], 8)
1667 if dst_mode
== 0120000:
1670 elif modifier
== "D":
1671 filesToDelete
.add(path
)
1672 if path
in filesToAdd
:
1673 filesToAdd
.remove(path
)
1674 elif modifier
== "C":
1675 src
, dest
= diff
['src'], diff
['dst']
1676 p4_integrate(src
, dest
)
1677 pureRenameCopy
.add(dest
)
1678 if diff
['src_sha1'] != diff
['dst_sha1']:
1680 pureRenameCopy
.discard(dest
)
1681 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1683 pureRenameCopy
.discard(dest
)
1684 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1686 # turn off read-only attribute
1687 os
.chmod(dest
, stat
.S_IWRITE
)
1689 editedFiles
.add(dest
)
1690 elif modifier
== "R":
1691 src
, dest
= diff
['src'], diff
['dst']
1692 if self
.p4HasMoveCommand
:
1693 p4_edit(src
) # src must be open before move
1694 p4_move(src
, dest
) # opens for (move/delete, move/add)
1696 p4_integrate(src
, dest
)
1697 if diff
['src_sha1'] != diff
['dst_sha1']:
1700 pureRenameCopy
.add(dest
)
1701 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1702 if not self
.p4HasMoveCommand
:
1703 p4_edit(dest
) # with move: already open, writable
1704 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1705 if not self
.p4HasMoveCommand
:
1707 os
.chmod(dest
, stat
.S_IWRITE
)
1709 filesToDelete
.add(src
)
1710 editedFiles
.add(dest
)
1711 elif modifier
== "T":
1712 filesToChangeType
.add(path
)
1714 die("unknown modifier %s for %s" % (modifier
, path
))
1716 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1717 patchcmd
= diffcmd
+ " | git apply "
1718 tryPatchCmd
= patchcmd
+ "--check -"
1719 applyPatchCmd
= patchcmd
+ "--check --apply -"
1720 patch_succeeded
= True
1722 if os
.system(tryPatchCmd
) != 0:
1723 fixed_rcs_keywords
= False
1724 patch_succeeded
= False
1725 print "Unfortunately applying the change failed!"
1727 # Patch failed, maybe it's just RCS keyword woes. Look through
1728 # the patch to see if that's possible.
1729 if gitConfigBool("git-p4.attemptRCSCleanup"):
1733 for file in editedFiles | filesToDelete
:
1734 # did this file's delta contain RCS keywords?
1735 pattern
= p4_keywords_regexp_for_file(file)
1738 # this file is a possibility...look for RCS keywords.
1739 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1740 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1741 if regexp
.search(line
):
1743 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1744 kwfiles
[file] = pattern
1747 for file in kwfiles
:
1749 print "zapping %s with %s" % (line
,pattern
)
1750 # File is being deleted, so not open in p4. Must
1751 # disable the read-only bit on windows.
1752 if self
.isWindows
and file not in editedFiles
:
1753 os
.chmod(file, stat
.S_IWRITE
)
1754 self
.patchRCSKeywords(file, kwfiles
[file])
1755 fixed_rcs_keywords
= True
1757 if fixed_rcs_keywords
:
1758 print "Retrying the patch with RCS keywords cleaned up"
1759 if os
.system(tryPatchCmd
) == 0:
1760 patch_succeeded
= True
1762 if not patch_succeeded
:
1763 for f
in editedFiles
:
1768 # Apply the patch for real, and do add/delete/+x handling.
1770 system(applyPatchCmd
)
1772 for f
in filesToChangeType
:
1773 p4_edit(f
, "-t", "auto")
1774 for f
in filesToAdd
:
1776 for f
in filesToDelete
:
1780 # Set/clear executable bits
1781 for f
in filesToChangeExecBit
.keys():
1782 mode
= filesToChangeExecBit
[f
]
1783 setP4ExecBit(f
, mode
)
1785 if self
.update_shelve
:
1786 print("all_files = %s" % str(all_files
))
1787 p4_reopen_in_change(self
.update_shelve
, all_files
)
1790 # Build p4 change description, starting with the contents
1791 # of the git commit message.
1793 logMessage
= extractLogMessageFromGitCommit(id)
1794 logMessage
= logMessage
.strip()
1795 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1797 template
= self
.prepareSubmitTemplate(self
.update_shelve
)
1798 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1800 if self
.preserveUser
:
1801 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1803 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1804 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1805 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1806 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1808 separatorLine
= "######## everything below this line is just the diff #######\n"
1809 if not self
.prepare_p4_only
:
1810 submitTemplate
+= separatorLine
1811 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
, symlinks
)
1813 (handle
, fileName
) = tempfile
.mkstemp()
1814 tmpFile
= os
.fdopen(handle
, "w+b")
1816 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1817 tmpFile
.write(submitTemplate
)
1820 if self
.prepare_p4_only
:
1822 # Leave the p4 tree prepared, and the submit template around
1823 # and let the user decide what to do next
1826 print "P4 workspace prepared for submission."
1827 print "To submit or revert, go to client workspace"
1828 print " " + self
.clientPath
1830 print "To submit, use \"p4 submit\" to write a new description,"
1831 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1832 " \"git p4\"." % fileName
1833 print "You can delete the file \"%s\" when finished." % fileName
1835 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1836 print "To preserve change ownership by user %s, you must\n" \
1837 "do \"p4 change -f <change>\" after submitting and\n" \
1838 "edit the User field."
1840 print "After submitting, renamed files must be re-synced."
1841 print "Invoke \"p4 sync -f\" on each of these files:"
1842 for f
in pureRenameCopy
:
1846 print "To revert the changes, use \"p4 revert ...\", and delete"
1847 print "the submit template file \"%s\"" % fileName
1849 print "Since the commit adds new files, they must be deleted:"
1850 for f
in filesToAdd
:
1856 # Let the user edit the change description, then submit it.
1861 if self
.edit_template(fileName
):
1862 # read the edited message and submit
1863 tmpFile
= open(fileName
, "rb")
1864 message
= tmpFile
.read()
1867 message
= message
.replace("\r\n", "\n")
1868 submitTemplate
= message
[:message
.index(separatorLine
)]
1870 if self
.update_shelve
:
1871 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate
)
1873 p4_write_pipe(['shelve', '-i'], submitTemplate
)
1875 p4_write_pipe(['submit', '-i'], submitTemplate
)
1876 # The rename/copy happened by applying a patch that created a
1877 # new file. This leaves it writable, which confuses p4.
1878 for f
in pureRenameCopy
:
1881 if self
.preserveUser
:
1883 # Get last changelist number. Cannot easily get it from
1884 # the submit command output as the output is
1886 changelist
= self
.lastP4Changelist()
1887 self
.modifyChangelistUser(changelist
, p4User
)
1893 if not submitted
or self
.shelve
:
1895 print ("Reverting shelved files.")
1897 print ("Submission cancelled, undoing p4 changes.")
1898 for f
in editedFiles | filesToDelete
:
1900 for f
in filesToAdd
:
1907 # Export git tags as p4 labels. Create a p4 label and then tag
1909 def exportGitTags(self
, gitTags
):
1910 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1911 if len(validLabelRegexp
) == 0:
1912 validLabelRegexp
= defaultLabelRegexp
1913 m
= re
.compile(validLabelRegexp
)
1915 for name
in gitTags
:
1917 if not m
.match(name
):
1919 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1922 # Get the p4 commit this corresponds to
1923 logMessage
= extractLogMessageFromGitCommit(name
)
1924 values
= extractSettingsGitLog(logMessage
)
1926 if not values
.has_key('change'):
1927 # a tag pointing to something not sent to p4; ignore
1929 print "git tag %s does not give a p4 commit" % name
1932 changelist
= values
['change']
1934 # Get the tag details.
1938 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1941 if re
.match(r
'tag\s+', l
):
1943 elif re
.match(r
'\s*$', l
):
1950 body
= ["lightweight tag imported by git p4\n"]
1952 # Create the label - use the same view as the client spec we are using
1953 clientSpec
= getClientSpec()
1955 labelTemplate
= "Label: %s\n" % name
1956 labelTemplate
+= "Description:\n"
1958 labelTemplate
+= "\t" + b
+ "\n"
1959 labelTemplate
+= "View:\n"
1960 for depot_side
in clientSpec
.mappings
:
1961 labelTemplate
+= "\t%s\n" % depot_side
1964 print "Would create p4 label %s for tag" % name
1965 elif self
.prepare_p4_only
:
1966 print "Not creating p4 label %s for tag due to option" \
1967 " --prepare-p4-only" % name
1969 p4_write_pipe(["label", "-i"], labelTemplate
)
1972 p4_system(["tag", "-l", name
] +
1973 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1976 print "created p4 label for tag %s" % name
1978 def run(self
, args
):
1980 self
.master
= currentGitBranch()
1981 elif len(args
) == 1:
1982 self
.master
= args
[0]
1983 if not branchExists(self
.master
):
1984 die("Branch %s does not exist" % self
.master
)
1989 allowSubmit
= gitConfig("git-p4.allowSubmit")
1990 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1991 die("%s is not in git-p4.allowSubmit" % self
.master
)
1993 [upstream
, settings
] = findUpstreamBranchPoint()
1994 self
.depotPath
= settings
['depot-paths'][0]
1995 if len(self
.origin
) == 0:
1996 self
.origin
= upstream
1998 if self
.update_shelve
:
2001 if self
.preserveUser
:
2002 if not self
.canChangeChangelists():
2003 die("Cannot preserve user names without p4 super-user or admin permissions")
2005 # if not set from the command line, try the config file
2006 if self
.conflict_behavior
is None:
2007 val
= gitConfig("git-p4.conflict")
2009 if val
not in self
.conflict_behavior_choices
:
2010 die("Invalid value '%s' for config git-p4.conflict" % val
)
2013 self
.conflict_behavior
= val
2016 print "Origin branch is " + self
.origin
2018 if len(self
.depotPath
) == 0:
2019 print "Internal error: cannot locate perforce depot path from existing branches"
2022 self
.useClientSpec
= False
2023 if gitConfigBool("git-p4.useclientspec"):
2024 self
.useClientSpec
= True
2025 if self
.useClientSpec
:
2026 self
.clientSpecDirs
= getClientSpec()
2028 # Check for the existence of P4 branches
2029 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
2031 if self
.useClientSpec
and not branchesDetected
:
2032 # all files are relative to the client spec
2033 self
.clientPath
= getClientRoot()
2035 self
.clientPath
= p4Where(self
.depotPath
)
2037 if self
.clientPath
== "":
2038 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
2040 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
2041 self
.oldWorkingDirectory
= os
.getcwd()
2043 # ensure the clientPath exists
2044 new_client_dir
= False
2045 if not os
.path
.exists(self
.clientPath
):
2046 new_client_dir
= True
2047 os
.makedirs(self
.clientPath
)
2049 chdir(self
.clientPath
, is_client_path
=True)
2051 print "Would synchronize p4 checkout in %s" % self
.clientPath
2053 print "Synchronizing p4 checkout..."
2055 # old one was destroyed, and maybe nobody told p4
2056 p4_sync("...", "-f")
2063 commitish
= self
.master
2067 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
2068 commits
.append(line
.strip())
2071 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
2072 self
.checkAuthorship
= False
2074 self
.checkAuthorship
= True
2076 if self
.preserveUser
:
2077 self
.checkValidP4Users(commits
)
2080 # Build up a set of options to be passed to diff when
2081 # submitting each commit to p4.
2083 if self
.detectRenames
:
2084 # command-line -M arg
2085 self
.diffOpts
= "-M"
2087 # If not explicitly set check the config variable
2088 detectRenames
= gitConfig("git-p4.detectRenames")
2090 if detectRenames
.lower() == "false" or detectRenames
== "":
2092 elif detectRenames
.lower() == "true":
2093 self
.diffOpts
= "-M"
2095 self
.diffOpts
= "-M%s" % detectRenames
2097 # no command-line arg for -C or --find-copies-harder, just
2099 detectCopies
= gitConfig("git-p4.detectCopies")
2100 if detectCopies
.lower() == "false" or detectCopies
== "":
2102 elif detectCopies
.lower() == "true":
2103 self
.diffOpts
+= " -C"
2105 self
.diffOpts
+= " -C%s" % detectCopies
2107 if gitConfigBool("git-p4.detectCopiesHarder"):
2108 self
.diffOpts
+= " --find-copies-harder"
2111 # Apply the commits, one at a time. On failure, ask if should
2112 # continue to try the rest of the patches, or quit.
2117 last
= len(commits
) - 1
2118 for i
, commit
in enumerate(commits
):
2120 print " ", read_pipe(["git", "show", "-s",
2121 "--format=format:%h %s", commit
])
2124 ok
= self
.applyCommit(commit
)
2126 applied
.append(commit
)
2128 if self
.prepare_p4_only
and i
< last
:
2129 print "Processing only the first commit due to option" \
2130 " --prepare-p4-only"
2135 # prompt for what to do, or use the option/variable
2136 if self
.conflict_behavior
== "ask":
2137 print "What do you want to do?"
2138 response
= raw_input("[s]kip this commit but apply"
2139 " the rest, or [q]uit? ")
2142 elif self
.conflict_behavior
== "skip":
2144 elif self
.conflict_behavior
== "quit":
2147 die("Unknown conflict_behavior '%s'" %
2148 self
.conflict_behavior
)
2150 if response
[0] == "s":
2151 print "Skipping this commit, but applying the rest"
2153 if response
[0] == "q":
2160 chdir(self
.oldWorkingDirectory
)
2161 shelved_applied
= "shelved" if self
.shelve
else "applied"
2164 elif self
.prepare_p4_only
:
2166 elif len(commits
) == len(applied
):
2167 print ("All commits {0}!".format(shelved_applied
))
2171 sync
.branch
= self
.branch
2178 if len(applied
) == 0:
2179 print ("No commits {0}.".format(shelved_applied
))
2181 print ("{0} only the commits marked with '*':".format(shelved_applied
.capitalize()))
2187 print star
, read_pipe(["git", "show", "-s",
2188 "--format=format:%h %s", c
])
2189 print "You will have to do 'git p4 sync' and rebase."
2191 if gitConfigBool("git-p4.exportLabels"):
2192 self
.exportLabels
= True
2194 if self
.exportLabels
:
2195 p4Labels
= getP4Labels(self
.depotPath
)
2196 gitTags
= getGitTags()
2198 missingGitTags
= gitTags
- p4Labels
2199 self
.exportGitTags(missingGitTags
)
2201 # exit with error unless everything applied perfectly
2202 if len(commits
) != len(applied
):
2208 """Represent a p4 view ("p4 help views"), and map files in a
2209 repo according to the view."""
2211 def __init__(self
, client_name
):
2213 self
.client_prefix
= "//%s/" % client_name
2214 # cache results of "p4 where" to lookup client file locations
2215 self
.client_spec_path_cache
= {}
2217 def append(self
, view_line
):
2218 """Parse a view line, splitting it into depot and client
2219 sides. Append to self.mappings, preserving order. This
2220 is only needed for tag creation."""
2222 # Split the view line into exactly two words. P4 enforces
2223 # structure on these lines that simplifies this quite a bit.
2225 # Either or both words may be double-quoted.
2226 # Single quotes do not matter.
2227 # Double-quote marks cannot occur inside the words.
2228 # A + or - prefix is also inside the quotes.
2229 # There are no quotes unless they contain a space.
2230 # The line is already white-space stripped.
2231 # The two words are separated by a single space.
2233 if view_line
[0] == '"':
2234 # First word is double quoted. Find its end.
2235 close_quote_index
= view_line
.find('"', 1)
2236 if close_quote_index
<= 0:
2237 die("No first-word closing quote found: %s" % view_line
)
2238 depot_side
= view_line
[1:close_quote_index
]
2239 # skip closing quote and space
2240 rhs_index
= close_quote_index
+ 1 + 1
2242 space_index
= view_line
.find(" ")
2243 if space_index
<= 0:
2244 die("No word-splitting space found: %s" % view_line
)
2245 depot_side
= view_line
[0:space_index
]
2246 rhs_index
= space_index
+ 1
2248 # prefix + means overlay on previous mapping
2249 if depot_side
.startswith("+"):
2250 depot_side
= depot_side
[1:]
2252 # prefix - means exclude this path, leave out of mappings
2254 if depot_side
.startswith("-"):
2256 depot_side
= depot_side
[1:]
2259 self
.mappings
.append(depot_side
)
2261 def convert_client_path(self
, clientFile
):
2262 # chop off //client/ part to make it relative
2263 if not clientFile
.startswith(self
.client_prefix
):
2264 die("No prefix '%s' on clientFile '%s'" %
2265 (self
.client_prefix
, clientFile
))
2266 return clientFile
[len(self
.client_prefix
):]
2268 def update_client_spec_path_cache(self
, files
):
2269 """ Caching file paths by "p4 where" batch query """
2271 # List depot file paths exclude that already cached
2272 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2274 if len(fileArgs
) == 0:
2275 return # All files in cache
2277 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2278 for res
in where_result
:
2279 if "code" in res
and res
["code"] == "error":
2280 # assume error is "... file(s) not in client view"
2282 if "clientFile" not in res
:
2283 die("No clientFile in 'p4 where' output")
2285 # it will list all of them, but only one not unmap-ped
2287 if gitConfigBool("core.ignorecase"):
2288 res
['depotFile'] = res
['depotFile'].lower()
2289 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2291 # not found files or unmap files set to ""
2292 for depotFile
in fileArgs
:
2293 if gitConfigBool("core.ignorecase"):
2294 depotFile
= depotFile
.lower()
2295 if depotFile
not in self
.client_spec_path_cache
:
2296 self
.client_spec_path_cache
[depotFile
] = ""
2298 def map_in_client(self
, depot_path
):
2299 """Return the relative location in the client where this
2300 depot file should live. Returns "" if the file should
2301 not be mapped in the client."""
2303 if gitConfigBool("core.ignorecase"):
2304 depot_path
= depot_path
.lower()
2306 if depot_path
in self
.client_spec_path_cache
:
2307 return self
.client_spec_path_cache
[depot_path
]
2309 die( "Error: %s is not found in client spec path" % depot_path
)
2312 class P4Sync(Command
, P4UserMap
):
2313 delete_actions
= ( "delete", "move/delete", "purge" )
2316 Command
.__init
__(self
)
2317 P4UserMap
.__init
__(self
)
2319 optparse
.make_option("--branch", dest
="branch"),
2320 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2321 optparse
.make_option("--changesfile", dest
="changesFile"),
2322 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2323 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2324 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2325 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2326 help="Import into refs/heads/ , not refs/remotes"),
2327 optparse
.make_option("--max-changes", dest
="maxChanges",
2328 help="Maximum number of changes to import"),
2329 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2330 help="Internal block size to use when iteratively calling p4 changes"),
2331 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2332 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2333 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2334 help="Only sync files that are included in the Perforce Client Spec"),
2335 optparse
.make_option("-/", dest
="cloneExclude",
2336 action
="append", type="string",
2337 help="exclude depot path"),
2339 self
.description
= """Imports from Perforce into a git repository.\n
2341 //depot/my/project/ -- to import the current head
2342 //depot/my/project/@all -- to import everything
2343 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2345 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2347 self
.usage
+= " //depot/path[@revRange]"
2349 self
.createdBranches
= set()
2350 self
.committedChanges
= set()
2352 self
.detectBranches
= False
2353 self
.detectLabels
= False
2354 self
.importLabels
= False
2355 self
.changesFile
= ""
2356 self
.syncWithOrigin
= True
2357 self
.importIntoRemotes
= True
2358 self
.maxChanges
= ""
2359 self
.changes_block_size
= None
2360 self
.keepRepoPath
= False
2361 self
.depotPaths
= None
2362 self
.p4BranchesInGit
= []
2363 self
.cloneExclude
= []
2364 self
.useClientSpec
= False
2365 self
.useClientSpec_from_options
= False
2366 self
.clientSpecDirs
= None
2367 self
.tempBranches
= []
2368 self
.tempBranchLocation
= "refs/git-p4-tmp"
2369 self
.largeFileSystem
= None
2371 if gitConfig('git-p4.largeFileSystem'):
2372 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2373 self
.largeFileSystem
= largeFileSystemConstructor(
2374 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2377 if gitConfig("git-p4.syncFromOrigin") == "false":
2378 self
.syncWithOrigin
= False
2380 # This is required for the "append" cloneExclude action
2381 def ensure_value(self
, attr
, value
):
2382 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2383 setattr(self
, attr
, value
)
2384 return getattr(self
, attr
)
2386 # Force a checkpoint in fast-import and wait for it to finish
2387 def checkpoint(self
):
2388 self
.gitStream
.write("checkpoint\n\n")
2389 self
.gitStream
.write("progress checkpoint\n\n")
2390 out
= self
.gitOutput
.readline()
2392 print "checkpoint finished: " + out
2394 def extractFilesFromCommit(self
, commit
):
2395 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2396 for path
in self
.cloneExclude
]
2399 while commit
.has_key("depotFile%s" % fnum
):
2400 path
= commit
["depotFile%s" % fnum
]
2402 if [p
for p
in self
.cloneExclude
2403 if p4PathStartsWith(path
, p
)]:
2406 found
= [p
for p
in self
.depotPaths
2407 if p4PathStartsWith(path
, p
)]
2414 file["rev"] = commit
["rev%s" % fnum
]
2415 file["action"] = commit
["action%s" % fnum
]
2416 file["type"] = commit
["type%s" % fnum
]
2421 def extractJobsFromCommit(self
, commit
):
2424 while commit
.has_key("job%s" % jnum
):
2425 job
= commit
["job%s" % jnum
]
2430 def stripRepoPath(self
, path
, prefixes
):
2431 """When streaming files, this is called to map a p4 depot path
2432 to where it should go in git. The prefixes are either
2433 self.depotPaths, or self.branchPrefixes in the case of
2434 branch detection."""
2436 if self
.useClientSpec
:
2437 # branch detection moves files up a level (the branch name)
2438 # from what client spec interpretation gives
2439 path
= self
.clientSpecDirs
.map_in_client(path
)
2440 if self
.detectBranches
:
2441 for b
in self
.knownBranches
:
2442 if path
.startswith(b
+ "/"):
2443 path
= path
[len(b
)+1:]
2445 elif self
.keepRepoPath
:
2446 # Preserve everything in relative path name except leading
2447 # //depot/; just look at first prefix as they all should
2448 # be in the same depot.
2449 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2450 if p4PathStartsWith(path
, depot
):
2451 path
= path
[len(depot
):]
2455 if p4PathStartsWith(path
, p
):
2456 path
= path
[len(p
):]
2459 path
= wildcard_decode(path
)
2462 def splitFilesIntoBranches(self
, commit
):
2463 """Look at each depotFile in the commit to figure out to what
2464 branch it belongs."""
2466 if self
.clientSpecDirs
:
2467 files
= self
.extractFilesFromCommit(commit
)
2468 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2472 while commit
.has_key("depotFile%s" % fnum
):
2473 path
= commit
["depotFile%s" % fnum
]
2474 found
= [p
for p
in self
.depotPaths
2475 if p4PathStartsWith(path
, p
)]
2482 file["rev"] = commit
["rev%s" % fnum
]
2483 file["action"] = commit
["action%s" % fnum
]
2484 file["type"] = commit
["type%s" % fnum
]
2487 # start with the full relative path where this file would
2489 if self
.useClientSpec
:
2490 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2492 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2494 for branch
in self
.knownBranches
.keys():
2495 # add a trailing slash so that a commit into qt/4.2foo
2496 # doesn't end up in qt/4.2, e.g.
2497 if relPath
.startswith(branch
+ "/"):
2498 if branch
not in branches
:
2499 branches
[branch
] = []
2500 branches
[branch
].append(file)
2505 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2506 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2507 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2509 self
.gitStream
.write(d
)
2510 self
.gitStream
.write('\n')
2512 def encodeWithUTF8(self
, path
):
2514 path
.decode('ascii')
2517 if gitConfig('git-p4.pathEncoding'):
2518 encoding
= gitConfig('git-p4.pathEncoding')
2519 path
= path
.decode(encoding
, 'replace').encode('utf8', 'replace')
2521 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, path
)
2524 # output one file from the P4 stream
2525 # - helper for streamP4Files
2527 def streamOneP4File(self
, file, contents
):
2528 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2529 relPath
= self
.encodeWithUTF8(relPath
)
2531 size
= int(self
.stream_file
['fileSize'])
2532 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2535 (type_base
, type_mods
) = split_p4_type(file["type"])
2538 if "x" in type_mods
:
2540 if type_base
== "symlink":
2542 # p4 print on a symlink sometimes contains "target\n";
2543 # if it does, remove the newline
2544 data
= ''.join(contents
)
2546 # Some version of p4 allowed creating a symlink that pointed
2547 # to nothing. This causes p4 errors when checking out such
2548 # a change, and errors here too. Work around it by ignoring
2549 # the bad symlink; hopefully a future change fixes it.
2550 print "\nIgnoring empty symlink in %s" % file['depotFile']
2552 elif data
[-1] == '\n':
2553 contents
= [data
[:-1]]
2557 if type_base
== "utf16":
2558 # p4 delivers different text in the python output to -G
2559 # than it does when using "print -o", or normal p4 client
2560 # operations. utf16 is converted to ascii or utf8, perhaps.
2561 # But ascii text saved as -t utf16 is completely mangled.
2562 # Invoke print -o to get the real contents.
2564 # On windows, the newlines will always be mangled by print, so put
2565 # them back too. This is not needed to the cygwin windows version,
2566 # just the native "NT" type.
2569 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2570 except Exception as e
:
2571 if 'Translation of file content failed' in str(e
):
2572 type_base
= 'binary'
2576 if p4_version_string().find('/NT') >= 0:
2577 text
= text
.replace('\r\n', '\n')
2580 if type_base
== "apple":
2581 # Apple filetype files will be streamed as a concatenation of
2582 # its appledouble header and the contents. This is useless
2583 # on both macs and non-macs. If using "print -q -o xx", it
2584 # will create "xx" with the data, and "%xx" with the header.
2585 # This is also not very useful.
2587 # Ideally, someday, this script can learn how to generate
2588 # appledouble files directly and import those to git, but
2589 # non-mac machines can never find a use for apple filetype.
2590 print "\nIgnoring apple filetype file %s" % file['depotFile']
2593 # Note that we do not try to de-mangle keywords on utf16 files,
2594 # even though in theory somebody may want that.
2595 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2597 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2598 text
= ''.join(contents
)
2599 text
= regexp
.sub(r
'$\1$', text
)
2602 if self
.largeFileSystem
:
2603 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2605 self
.writeToGitStream(git_mode
, relPath
, contents
)
2607 def streamOneP4Deletion(self
, file):
2608 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2609 relPath
= self
.encodeWithUTF8(relPath
)
2611 sys
.stdout
.write("delete %s\n" % relPath
)
2613 self
.gitStream
.write("D %s\n" % relPath
)
2615 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2616 self
.largeFileSystem
.removeLargeFile(relPath
)
2618 # handle another chunk of streaming data
2619 def streamP4FilesCb(self
, marshalled
):
2621 # catch p4 errors and complain
2623 if "code" in marshalled
:
2624 if marshalled
["code"] == "error":
2625 if "data" in marshalled
:
2626 err
= marshalled
["data"].rstrip()
2628 if not err
and 'fileSize' in self
.stream_file
:
2629 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2630 if required_bytes
> 0:
2631 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2632 os
.getcwd(), required_bytes
/1024/1024
2637 if self
.stream_have_file_info
:
2638 if "depotFile" in self
.stream_file
:
2639 f
= self
.stream_file
["depotFile"]
2640 # force a failure in fast-import, else an empty
2641 # commit will be made
2642 self
.gitStream
.write("\n")
2643 self
.gitStream
.write("die-now\n")
2644 self
.gitStream
.close()
2645 # ignore errors, but make sure it exits first
2646 self
.importProcess
.wait()
2648 die("Error from p4 print for %s: %s" % (f
, err
))
2650 die("Error from p4 print: %s" % err
)
2652 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2653 # start of a new file - output the old one first
2654 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2655 self
.stream_file
= {}
2656 self
.stream_contents
= []
2657 self
.stream_have_file_info
= False
2659 # pick up the new file information... for the
2660 # 'data' field we need to append to our array
2661 for k
in marshalled
.keys():
2663 if 'streamContentSize' not in self
.stream_file
:
2664 self
.stream_file
['streamContentSize'] = 0
2665 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2666 self
.stream_contents
.append(marshalled
['data'])
2668 self
.stream_file
[k
] = marshalled
[k
]
2671 'streamContentSize' in self
.stream_file
and
2672 'fileSize' in self
.stream_file
and
2673 'depotFile' in self
.stream_file
):
2674 size
= int(self
.stream_file
["fileSize"])
2676 progress
= 100*self
.stream_file
['streamContentSize']/size
2677 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2680 self
.stream_have_file_info
= True
2682 # Stream directly from "p4 files" into "git fast-import"
2683 def streamP4Files(self
, files
):
2689 filesForCommit
.append(f
)
2690 if f
['action'] in self
.delete_actions
:
2691 filesToDelete
.append(f
)
2693 filesToRead
.append(f
)
2696 for f
in filesToDelete
:
2697 self
.streamOneP4Deletion(f
)
2699 if len(filesToRead
) > 0:
2700 self
.stream_file
= {}
2701 self
.stream_contents
= []
2702 self
.stream_have_file_info
= False
2704 # curry self argument
2705 def streamP4FilesCbSelf(entry
):
2706 self
.streamP4FilesCb(entry
)
2708 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2710 p4CmdList(["-x", "-", "print"],
2712 cb
=streamP4FilesCbSelf
)
2715 if self
.stream_file
.has_key('depotFile'):
2716 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2718 def make_email(self
, userid
):
2719 if userid
in self
.users
:
2720 return self
.users
[userid
]
2722 return "%s <a@b>" % userid
2724 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2725 """ Stream a p4 tag.
2726 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2730 print "writing tag %s for commit %s" % (labelName
, commit
)
2731 gitStream
.write("tag %s\n" % labelName
)
2732 gitStream
.write("from %s\n" % commit
)
2734 if labelDetails
.has_key('Owner'):
2735 owner
= labelDetails
["Owner"]
2739 # Try to use the owner of the p4 label, or failing that,
2740 # the current p4 user id.
2742 email
= self
.make_email(owner
)
2744 email
= self
.make_email(self
.p4UserId())
2745 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2747 gitStream
.write("tagger %s\n" % tagger
)
2749 print "labelDetails=",labelDetails
2750 if labelDetails
.has_key('Description'):
2751 description
= labelDetails
['Description']
2753 description
= 'Label from git p4'
2755 gitStream
.write("data %d\n" % len(description
))
2756 gitStream
.write(description
)
2757 gitStream
.write("\n")
2759 def inClientSpec(self
, path
):
2760 if not self
.clientSpecDirs
:
2762 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2763 if not inClientSpec
and self
.verbose
:
2764 print('Ignoring file outside of client spec: {0}'.format(path
))
2767 def hasBranchPrefix(self
, path
):
2768 if not self
.branchPrefixes
:
2770 hasPrefix
= [p
for p
in self
.branchPrefixes
2771 if p4PathStartsWith(path
, p
)]
2772 if not hasPrefix
and self
.verbose
:
2773 print('Ignoring file outside of prefix: {0}'.format(path
))
2776 def commit(self
, details
, files
, branch
, parent
= ""):
2777 epoch
= details
["time"]
2778 author
= details
["user"]
2779 jobs
= self
.extractJobsFromCommit(details
)
2782 print('commit into {0}'.format(branch
))
2784 if self
.clientSpecDirs
:
2785 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2787 files
= [f
for f
in files
2788 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
2790 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
2791 print('Ignoring revision {0} as it would produce an empty commit.'
2792 .format(details
['change']))
2795 self
.gitStream
.write("commit %s\n" % branch
)
2796 self
.gitStream
.write("mark :%s\n" % details
["change"])
2797 self
.committedChanges
.add(int(details
["change"]))
2799 if author
not in self
.users
:
2800 self
.getUserMapFromPerforceServer()
2801 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2803 self
.gitStream
.write("committer %s\n" % committer
)
2805 self
.gitStream
.write("data <<EOT\n")
2806 self
.gitStream
.write(details
["desc"])
2808 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
2809 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2810 (','.join(self
.branchPrefixes
), details
["change"]))
2811 if len(details
['options']) > 0:
2812 self
.gitStream
.write(": options = %s" % details
['options'])
2813 self
.gitStream
.write("]\nEOT\n\n")
2817 print "parent %s" % parent
2818 self
.gitStream
.write("from %s\n" % parent
)
2820 self
.streamP4Files(files
)
2821 self
.gitStream
.write("\n")
2823 change
= int(details
["change"])
2825 if self
.labels
.has_key(change
):
2826 label
= self
.labels
[change
]
2827 labelDetails
= label
[0]
2828 labelRevisions
= label
[1]
2830 print "Change %s is labelled %s" % (change
, labelDetails
)
2832 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2833 for p
in self
.branchPrefixes
])
2835 if len(files
) == len(labelRevisions
):
2839 if info
["action"] in self
.delete_actions
:
2841 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2843 if cleanedFiles
== labelRevisions
:
2844 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2848 print ("Tag %s does not match with change %s: files do not match."
2849 % (labelDetails
["label"], change
))
2853 print ("Tag %s does not match with change %s: file count is different."
2854 % (labelDetails
["label"], change
))
2856 # Build a dictionary of changelists and labels, for "detect-labels" option.
2857 def getLabels(self
):
2860 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2861 if len(l
) > 0 and not self
.silent
:
2862 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2865 label
= output
["label"]
2869 print "Querying files for label %s" % label
2870 for file in p4CmdList(["files"] +
2871 ["%s...@%s" % (p
, label
)
2872 for p
in self
.depotPaths
]):
2873 revisions
[file["depotFile"]] = file["rev"]
2874 change
= int(file["change"])
2875 if change
> newestChange
:
2876 newestChange
= change
2878 self
.labels
[newestChange
] = [output
, revisions
]
2881 print "Label changes: %s" % self
.labels
.keys()
2883 # Import p4 labels as git tags. A direct mapping does not
2884 # exist, so assume that if all the files are at the same revision
2885 # then we can use that, or it's something more complicated we should
2887 def importP4Labels(self
, stream
, p4Labels
):
2889 print "import p4 labels: " + ' '.join(p4Labels
)
2891 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2892 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2893 if len(validLabelRegexp
) == 0:
2894 validLabelRegexp
= defaultLabelRegexp
2895 m
= re
.compile(validLabelRegexp
)
2897 for name
in p4Labels
:
2900 if not m
.match(name
):
2902 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2905 if name
in ignoredP4Labels
:
2908 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2910 # get the most recent changelist for each file in this label
2911 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2912 for p
in self
.depotPaths
])
2914 if change
.has_key('change'):
2915 # find the corresponding git commit; take the oldest commit
2916 changelist
= int(change
['change'])
2917 if changelist
in self
.committedChanges
:
2918 gitCommit
= ":%d" % changelist
# use a fast-import mark
2921 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2922 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2923 if len(gitCommit
) == 0:
2924 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2927 gitCommit
= gitCommit
.strip()
2930 # Convert from p4 time format
2932 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2934 print "Could not convert label time %s" % labelDetails
['Update']
2937 when
= int(time
.mktime(tmwhen
))
2938 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2940 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2943 print "Label %s has no changelists - possibly deleted?" % name
2946 # We can't import this label; don't try again as it will get very
2947 # expensive repeatedly fetching all the files for labels that will
2948 # never be imported. If the label is moved in the future, the
2949 # ignore will need to be removed manually.
2950 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2952 def guessProjectName(self
):
2953 for p
in self
.depotPaths
:
2956 p
= p
[p
.strip().rfind("/") + 1:]
2957 if not p
.endswith("/"):
2961 def getBranchMapping(self
):
2962 lostAndFoundBranches
= set()
2964 user
= gitConfig("git-p4.branchUser")
2966 command
= "branches -u %s" % user
2968 command
= "branches"
2970 for info
in p4CmdList(command
):
2971 details
= p4Cmd(["branch", "-o", info
["branch"]])
2973 while details
.has_key("View%s" % viewIdx
):
2974 paths
= details
["View%s" % viewIdx
].split(" ")
2975 viewIdx
= viewIdx
+ 1
2976 # require standard //depot/foo/... //depot/bar/... mapping
2977 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2980 destination
= paths
[1]
2982 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2983 source
= source
[len(self
.depotPaths
[0]):-4]
2984 destination
= destination
[len(self
.depotPaths
[0]):-4]
2986 if destination
in self
.knownBranches
:
2988 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2989 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2992 self
.knownBranches
[destination
] = source
2994 lostAndFoundBranches
.discard(destination
)
2996 if source
not in self
.knownBranches
:
2997 lostAndFoundBranches
.add(source
)
2999 # Perforce does not strictly require branches to be defined, so we also
3000 # check git config for a branch list.
3002 # Example of branch definition in git config file:
3004 # branchList=main:branchA
3005 # branchList=main:branchB
3006 # branchList=branchA:branchC
3007 configBranches
= gitConfigList("git-p4.branchList")
3008 for branch
in configBranches
:
3010 (source
, destination
) = branch
.split(":")
3011 self
.knownBranches
[destination
] = source
3013 lostAndFoundBranches
.discard(destination
)
3015 if source
not in self
.knownBranches
:
3016 lostAndFoundBranches
.add(source
)
3019 for branch
in lostAndFoundBranches
:
3020 self
.knownBranches
[branch
] = branch
3022 def getBranchMappingFromGitBranches(self
):
3023 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3024 for branch
in branches
.keys():
3025 if branch
== "master":
3028 branch
= branch
[len(self
.projectName
):]
3029 self
.knownBranches
[branch
] = branch
3031 def updateOptionDict(self
, d
):
3033 if self
.keepRepoPath
:
3034 option_keys
['keepRepoPath'] = 1
3036 d
["options"] = ' '.join(sorted(option_keys
.keys()))
3038 def readOptions(self
, d
):
3039 self
.keepRepoPath
= (d
.has_key('options')
3040 and ('keepRepoPath' in d
['options']))
3042 def gitRefForBranch(self
, branch
):
3043 if branch
== "main":
3044 return self
.refPrefix
+ "master"
3046 if len(branch
) <= 0:
3049 return self
.refPrefix
+ self
.projectName
+ branch
3051 def gitCommitByP4Change(self
, ref
, change
):
3053 print "looking in ref " + ref
+ " for change %s using bisect..." % change
3056 latestCommit
= parseRevision(ref
)
3060 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
3061 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
3066 log
= extractLogMessageFromGitCommit(next
)
3067 settings
= extractSettingsGitLog(log
)
3068 currentChange
= int(settings
['change'])
3070 print "current change %s" % currentChange
3072 if currentChange
== change
:
3074 print "found %s" % next
3077 if currentChange
< change
:
3078 earliestCommit
= "^%s" % next
3080 latestCommit
= "%s" % next
3084 def importNewBranch(self
, branch
, maxChange
):
3085 # make fast-import flush all changes to disk and update the refs using the checkpoint
3086 # command so that we can try to find the branch parent in the git history
3087 self
.gitStream
.write("checkpoint\n\n");
3088 self
.gitStream
.flush();
3089 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3090 range = "@1,%s" % maxChange
3091 #print "prefix" + branchPrefix
3092 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3093 if len(changes
) <= 0:
3095 firstChange
= changes
[0]
3096 #print "first change in branch: %s" % firstChange
3097 sourceBranch
= self
.knownBranches
[branch
]
3098 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3099 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3100 #print "source " + sourceBranch
3102 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3103 #print "branch parent: %s" % branchParentChange
3104 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3105 if len(gitParent
) > 0:
3106 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3107 #print "parent git commit: %s" % gitParent
3109 self
.importChanges(changes
)
3112 def searchParent(self
, parent
, branch
, target
):
3114 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3115 "--no-merges", parent
]):
3117 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3120 print "Found parent of %s in commit %s" % (branch
, blob
)
3127 def importChanges(self
, changes
):
3129 for change
in changes
:
3130 description
= p4_describe(change
)
3131 self
.updateOptionDict(description
)
3134 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3139 if self
.detectBranches
:
3140 branches
= self
.splitFilesIntoBranches(description
)
3141 for branch
in branches
.keys():
3143 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3144 self
.branchPrefixes
= [ branchPrefix
]
3148 filesForCommit
= branches
[branch
]
3151 print "branch is %s" % branch
3153 self
.updatedBranches
.add(branch
)
3155 if branch
not in self
.createdBranches
:
3156 self
.createdBranches
.add(branch
)
3157 parent
= self
.knownBranches
[branch
]
3158 if parent
== branch
:
3161 fullBranch
= self
.projectName
+ branch
3162 if fullBranch
not in self
.p4BranchesInGit
:
3164 print("\n Importing new branch %s" % fullBranch
);
3165 if self
.importNewBranch(branch
, change
- 1):
3167 self
.p4BranchesInGit
.append(fullBranch
)
3169 print("\n Resuming with change %s" % change
);
3172 print "parent determined through known branches: %s" % parent
3174 branch
= self
.gitRefForBranch(branch
)
3175 parent
= self
.gitRefForBranch(parent
)
3178 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3180 if len(parent
) == 0 and branch
in self
.initialParents
:
3181 parent
= self
.initialParents
[branch
]
3182 del self
.initialParents
[branch
]
3186 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3188 print "Creating temporary branch: " + tempBranch
3189 self
.commit(description
, filesForCommit
, tempBranch
)
3190 self
.tempBranches
.append(tempBranch
)
3192 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3194 self
.commit(description
, filesForCommit
, branch
, blob
)
3197 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3198 self
.commit(description
, filesForCommit
, branch
, parent
)
3200 files
= self
.extractFilesFromCommit(description
)
3201 self
.commit(description
, files
, self
.branch
,
3203 # only needed once, to connect to the previous commit
3204 self
.initialParent
= ""
3206 print self
.gitError
.read()
3209 def importHeadRevision(self
, revision
):
3210 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3213 details
["user"] = "git perforce import user"
3214 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3215 % (' '.join(self
.depotPaths
), revision
))
3216 details
["change"] = revision
3220 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3222 for info
in p4CmdList(["files"] + fileArgs
):
3224 if 'code' in info
and info
['code'] == 'error':
3225 sys
.stderr
.write("p4 returned an error: %s\n"
3227 if info
['data'].find("must refer to client") >= 0:
3228 sys
.stderr
.write("This particular p4 error is misleading.\n")
3229 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3230 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3232 if 'p4ExitCode' in info
:
3233 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3237 change
= int(info
["change"])
3238 if change
> newestRevision
:
3239 newestRevision
= change
3241 if info
["action"] in self
.delete_actions
:
3242 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3243 #fileCnt = fileCnt + 1
3246 for prop
in ["depotFile", "rev", "action", "type" ]:
3247 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3249 fileCnt
= fileCnt
+ 1
3251 details
["change"] = newestRevision
3253 # Use time from top-most change so that all git p4 clones of
3254 # the same p4 repo have the same commit SHA1s.
3255 res
= p4_describe(newestRevision
)
3256 details
["time"] = res
["time"]
3258 self
.updateOptionDict(details
)
3260 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3262 print "IO error with git fast-import. Is your git version recent enough?"
3263 print self
.gitError
.read()
3266 def run(self
, args
):
3267 self
.depotPaths
= []
3268 self
.changeRange
= ""
3269 self
.previousDepotPaths
= []
3270 self
.hasOrigin
= False
3272 # map from branch depot path to parent branch
3273 self
.knownBranches
= {}
3274 self
.initialParents
= {}
3276 if self
.importIntoRemotes
:
3277 self
.refPrefix
= "refs/remotes/p4/"
3279 self
.refPrefix
= "refs/heads/p4/"
3281 if self
.syncWithOrigin
:
3282 self
.hasOrigin
= originP4BranchesExist()
3285 print 'Syncing with origin first, using "git fetch origin"'
3286 system("git fetch origin")
3288 branch_arg_given
= bool(self
.branch
)
3289 if len(self
.branch
) == 0:
3290 self
.branch
= self
.refPrefix
+ "master"
3291 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3292 system("git update-ref %s refs/heads/p4" % self
.branch
)
3293 system("git branch -D p4")
3295 # accept either the command-line option, or the configuration variable
3296 if self
.useClientSpec
:
3297 # will use this after clone to set the variable
3298 self
.useClientSpec_from_options
= True
3300 if gitConfigBool("git-p4.useclientspec"):
3301 self
.useClientSpec
= True
3302 if self
.useClientSpec
:
3303 self
.clientSpecDirs
= getClientSpec()
3305 # TODO: should always look at previous commits,
3306 # merge with previous imports, if possible.
3309 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3311 # branches holds mapping from branch name to sha1
3312 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3314 # restrict to just this one, disabling detect-branches
3315 if branch_arg_given
:
3316 short
= self
.branch
.split("/")[-1]
3317 if short
in branches
:
3318 self
.p4BranchesInGit
= [ short
]
3320 self
.p4BranchesInGit
= branches
.keys()
3322 if len(self
.p4BranchesInGit
) > 1:
3324 print "Importing from/into multiple branches"
3325 self
.detectBranches
= True
3326 for branch
in branches
.keys():
3327 self
.initialParents
[self
.refPrefix
+ branch
] = \
3331 print "branches: %s" % self
.p4BranchesInGit
3334 for branch
in self
.p4BranchesInGit
:
3335 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3337 settings
= extractSettingsGitLog(logMsg
)
3339 self
.readOptions(settings
)
3340 if (settings
.has_key('depot-paths')
3341 and settings
.has_key ('change')):
3342 change
= int(settings
['change']) + 1
3343 p4Change
= max(p4Change
, change
)
3345 depotPaths
= sorted(settings
['depot-paths'])
3346 if self
.previousDepotPaths
== []:
3347 self
.previousDepotPaths
= depotPaths
3350 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3351 prev_list
= prev
.split("/")
3352 cur_list
= cur
.split("/")
3353 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3354 if cur_list
[i
] <> prev_list
[i
]:
3358 paths
.append ("/".join(cur_list
[:i
+ 1]))
3360 self
.previousDepotPaths
= paths
3363 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3364 self
.changeRange
= "@%s,#head" % p4Change
3365 if not self
.silent
and not self
.detectBranches
:
3366 print "Performing incremental import into %s git branch" % self
.branch
3368 # accept multiple ref name abbreviations:
3369 # refs/foo/bar/branch -> use it exactly
3370 # p4/branch -> prepend refs/remotes/ or refs/heads/
3371 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3372 if not self
.branch
.startswith("refs/"):
3373 if self
.importIntoRemotes
:
3374 prepend
= "refs/remotes/"
3376 prepend
= "refs/heads/"
3377 if not self
.branch
.startswith("p4/"):
3379 self
.branch
= prepend
+ self
.branch
3381 if len(args
) == 0 and self
.depotPaths
:
3383 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3385 if self
.depotPaths
and self
.depotPaths
!= args
:
3386 print ("previous import used depot path %s and now %s was specified. "
3387 "This doesn't work!" % (' '.join (self
.depotPaths
),
3391 self
.depotPaths
= sorted(args
)
3396 # Make sure no revision specifiers are used when --changesfile
3398 bad_changesfile
= False
3399 if len(self
.changesFile
) > 0:
3400 for p
in self
.depotPaths
:
3401 if p
.find("@") >= 0 or p
.find("#") >= 0:
3402 bad_changesfile
= True
3405 die("Option --changesfile is incompatible with revision specifiers")
3408 for p
in self
.depotPaths
:
3409 if p
.find("@") != -1:
3410 atIdx
= p
.index("@")
3411 self
.changeRange
= p
[atIdx
:]
3412 if self
.changeRange
== "@all":
3413 self
.changeRange
= ""
3414 elif ',' not in self
.changeRange
:
3415 revision
= self
.changeRange
3416 self
.changeRange
= ""
3418 elif p
.find("#") != -1:
3419 hashIdx
= p
.index("#")
3420 revision
= p
[hashIdx
:]
3422 elif self
.previousDepotPaths
== []:
3423 # pay attention to changesfile, if given, else import
3424 # the entire p4 tree at the head revision
3425 if len(self
.changesFile
) == 0:
3428 p
= re
.sub ("\.\.\.$", "", p
)
3429 if not p
.endswith("/"):
3434 self
.depotPaths
= newPaths
3436 # --detect-branches may change this for each branch
3437 self
.branchPrefixes
= self
.depotPaths
3439 self
.loadUserMapFromCache()
3441 if self
.detectLabels
:
3444 if self
.detectBranches
:
3445 ## FIXME - what's a P4 projectName ?
3446 self
.projectName
= self
.guessProjectName()
3449 self
.getBranchMappingFromGitBranches()
3451 self
.getBranchMapping()
3453 print "p4-git branches: %s" % self
.p4BranchesInGit
3454 print "initial parents: %s" % self
.initialParents
3455 for b
in self
.p4BranchesInGit
:
3459 b
= b
[len(self
.projectName
):]
3460 self
.createdBranches
.add(b
)
3462 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3464 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3465 stdin
=subprocess
.PIPE
,
3466 stdout
=subprocess
.PIPE
,
3467 stderr
=subprocess
.PIPE
);
3468 self
.gitOutput
= self
.importProcess
.stdout
3469 self
.gitStream
= self
.importProcess
.stdin
3470 self
.gitError
= self
.importProcess
.stderr
3473 self
.importHeadRevision(revision
)
3477 if len(self
.changesFile
) > 0:
3478 output
= open(self
.changesFile
).readlines()
3481 changeSet
.add(int(line
))
3483 for change
in changeSet
:
3484 changes
.append(change
)
3488 # catch "git p4 sync" with no new branches, in a repo that
3489 # does not have any existing p4 branches
3491 if not self
.p4BranchesInGit
:
3492 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3494 # The default branch is master, unless --branch is used to
3495 # specify something else. Make sure it exists, or complain
3496 # nicely about how to use --branch.
3497 if not self
.detectBranches
:
3498 if not branch_exists(self
.branch
):
3499 if branch_arg_given
:
3500 die("Error: branch %s does not exist." % self
.branch
)
3502 die("Error: no branch %s; perhaps specify one with --branch." %
3506 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3508 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3510 if len(self
.maxChanges
) > 0:
3511 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3513 if len(changes
) == 0:
3515 print "No changes to import!"
3517 if not self
.silent
and not self
.detectBranches
:
3518 print "Import destination: %s" % self
.branch
3520 self
.updatedBranches
= set()
3522 if not self
.detectBranches
:
3524 # start a new branch
3525 self
.initialParent
= ""
3527 # build on a previous revision
3528 self
.initialParent
= parseRevision(self
.branch
)
3530 self
.importChanges(changes
)
3534 if len(self
.updatedBranches
) > 0:
3535 sys
.stdout
.write("Updated branches: ")
3536 for b
in self
.updatedBranches
:
3537 sys
.stdout
.write("%s " % b
)
3538 sys
.stdout
.write("\n")
3540 if gitConfigBool("git-p4.importLabels"):
3541 self
.importLabels
= True
3543 if self
.importLabels
:
3544 p4Labels
= getP4Labels(self
.depotPaths
)
3545 gitTags
= getGitTags()
3547 missingP4Labels
= p4Labels
- gitTags
3548 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3550 self
.gitStream
.close()
3551 if self
.importProcess
.wait() != 0:
3552 die("fast-import failed: %s" % self
.gitError
.read())
3553 self
.gitOutput
.close()
3554 self
.gitError
.close()
3556 # Cleanup temporary branches created during import
3557 if self
.tempBranches
!= []:
3558 for branch
in self
.tempBranches
:
3559 read_pipe("git update-ref -d %s" % branch
)
3560 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3562 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3563 # a convenient shortcut refname "p4".
3564 if self
.importIntoRemotes
:
3565 head_ref
= self
.refPrefix
+ "HEAD"
3566 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3567 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3571 class P4Rebase(Command
):
3573 Command
.__init
__(self
)
3575 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3577 self
.importLabels
= False
3578 self
.description
= ("Fetches the latest revision from perforce and "
3579 + "rebases the current work (branch) against it")
3581 def run(self
, args
):
3583 sync
.importLabels
= self
.importLabels
3586 return self
.rebase()
3589 if os
.system("git update-index --refresh") != 0:
3590 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3591 if len(read_pipe("git diff-index HEAD --")) > 0:
3592 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3594 [upstream
, settings
] = findUpstreamBranchPoint()
3595 if len(upstream
) == 0:
3596 die("Cannot find upstream branchpoint for rebase")
3598 # the branchpoint may be p4/foo~3, so strip off the parent
3599 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3601 print "Rebasing the current branch onto %s" % upstream
3602 oldHead
= read_pipe("git rev-parse HEAD").strip()
3603 system("git rebase %s" % upstream
)
3604 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3607 class P4Clone(P4Sync
):
3609 P4Sync
.__init
__(self
)
3610 self
.description
= "Creates a new git repository and imports from Perforce into it"
3611 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3613 optparse
.make_option("--destination", dest
="cloneDestination",
3614 action
='store', default
=None,
3615 help="where to leave result of the clone"),
3616 optparse
.make_option("--bare", dest
="cloneBare",
3617 action
="store_true", default
=False),
3619 self
.cloneDestination
= None
3620 self
.needsGit
= False
3621 self
.cloneBare
= False
3623 def defaultDestination(self
, args
):
3624 ## TODO: use common prefix of args?
3626 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3627 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3628 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3629 depotDir
= re
.sub(r
"/$", "", depotDir
)
3630 return os
.path
.split(depotDir
)[1]
3632 def run(self
, args
):
3636 if self
.keepRepoPath
and not self
.cloneDestination
:
3637 sys
.stderr
.write("Must specify destination for --keep-path\n")
3642 if not self
.cloneDestination
and len(depotPaths
) > 1:
3643 self
.cloneDestination
= depotPaths
[-1]
3644 depotPaths
= depotPaths
[:-1]
3646 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3647 for p
in depotPaths
:
3648 if not p
.startswith("//"):
3649 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3652 if not self
.cloneDestination
:
3653 self
.cloneDestination
= self
.defaultDestination(args
)
3655 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3657 if not os
.path
.exists(self
.cloneDestination
):
3658 os
.makedirs(self
.cloneDestination
)
3659 chdir(self
.cloneDestination
)
3661 init_cmd
= [ "git", "init" ]
3663 init_cmd
.append("--bare")
3664 retcode
= subprocess
.call(init_cmd
)
3666 raise CalledProcessError(retcode
, init_cmd
)
3668 if not P4Sync
.run(self
, depotPaths
):
3671 # create a master branch and check out a work tree
3672 if gitBranchExists(self
.branch
):
3673 system([ "git", "branch", "master", self
.branch
])
3674 if not self
.cloneBare
:
3675 system([ "git", "checkout", "-f" ])
3677 print 'Not checking out any branch, use ' \
3678 '"git checkout -q -b master <branch>"'
3680 # auto-set this variable if invoked with --use-client-spec
3681 if self
.useClientSpec_from_options
:
3682 system("git config --bool git-p4.useclientspec true")
3686 class P4Branches(Command
):
3688 Command
.__init
__(self
)
3690 self
.description
= ("Shows the git branches that hold imports and their "
3691 + "corresponding perforce depot paths")
3692 self
.verbose
= False
3694 def run(self
, args
):
3695 if originP4BranchesExist():
3696 createOrUpdateBranchesFromOrigin()
3698 cmdline
= "git rev-parse --symbolic "
3699 cmdline
+= " --remotes"
3701 for line
in read_pipe_lines(cmdline
):
3704 if not line
.startswith('p4/') or line
== "p4/HEAD":
3708 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3709 settings
= extractSettingsGitLog(log
)
3711 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3714 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3716 optparse
.IndentedHelpFormatter
.__init
__(self
)
3718 def format_description(self
, description
):
3720 return description
+ "\n"
3724 def printUsage(commands
):
3725 print "usage: %s <command> [options]" % sys
.argv
[0]
3727 print "valid commands: %s" % ", ".join(commands
)
3729 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3734 "submit" : P4Submit
,
3735 "commit" : P4Submit
,
3737 "rebase" : P4Rebase
,
3739 "rollback" : P4RollBack
,
3740 "branches" : P4Branches
3745 if len(sys
.argv
[1:]) == 0:
3746 printUsage(commands
.keys())
3749 cmdName
= sys
.argv
[1]
3751 klass
= commands
[cmdName
]
3754 print "unknown command %s" % cmdName
3756 printUsage(commands
.keys())
3759 options
= cmd
.options
3760 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3764 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3766 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3768 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3770 description
= cmd
.description
,
3771 formatter
= HelpFormatter())
3773 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3775 verbose
= cmd
.verbose
3777 if cmd
.gitdir
== None:
3778 cmd
.gitdir
= os
.path
.abspath(".git")
3779 if not isValidGitDir(cmd
.gitdir
):
3780 # "rev-parse --git-dir" without arguments will try $PWD/.git
3781 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3782 if os
.path
.exists(cmd
.gitdir
):
3783 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3787 if not isValidGitDir(cmd
.gitdir
):
3788 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3789 cmd
.gitdir
+= "/.git"
3791 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3793 # so git commands invoked from the P4 workspace will succeed
3794 os
.environ
["GIT_DIR"] = cmd
.gitdir
3796 if not cmd
.run(args
):
3801 if __name__
== '__main__':