3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess
import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self
, returncode
, cmd
):
39 self
.returncode
= returncode
42 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize
= 512
52 def p4_build_cmd(cmd
):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user
= gitConfig("git-p4.user")
63 real_cmd
+= ["-u",user
]
65 password
= gitConfig("git-p4.password")
67 real_cmd
+= ["-P", password
]
69 port
= gitConfig("git-p4.port")
71 real_cmd
+= ["-p", port
]
73 host
= gitConfig("git-p4.host")
75 real_cmd
+= ["-H", host
]
77 client
= gitConfig("git-p4.client")
79 real_cmd
+= ["-c", client
]
82 if isinstance(cmd
,basestring
):
83 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
89 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
90 This won't automatically add ".git" to a directory.
92 d
= read_pipe(["git", "--git-dir", path
, "rev-parse", "--git-dir"], True).strip()
93 if not d
or len(d
) == 0:
98 def chdir(path
, is_client_path
=False):
99 """Do chdir to the given path, and set the PWD environment
100 variable for use by P4. It does not look at getcwd() output.
101 Since we're not using the shell, it is necessary to set the
102 PWD environment variable explicitly.
104 Normally, expand the path to force it to be absolute. This
105 addresses the use of relative path names inside P4 settings,
106 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
107 as given; it looks for .p4config using PWD.
109 If is_client_path, the path was handed to us directly by p4,
110 and may be a symbolic link. Do not call os.getcwd() in this
111 case, because it will cause p4 to think that PWD is not inside
116 if not is_client_path
:
118 os
.environ
['PWD'] = path
121 """Return free space in bytes on the disk of the given dirname."""
122 if platform
.system() == 'Windows':
123 free_bytes
= ctypes
.c_ulonglong(0)
124 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
125 return free_bytes
.value
127 st
= os
.statvfs(os
.getcwd())
128 return st
.f_bavail
* st
.f_frsize
134 sys
.stderr
.write(msg
+ "\n")
137 def write_pipe(c
, stdin
):
139 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
141 expand
= isinstance(c
,basestring
)
142 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
144 val
= pipe
.write(stdin
)
147 die('Command failed: %s' % str(c
))
151 def p4_write_pipe(c
, stdin
):
152 real_cmd
= p4_build_cmd(c
)
153 return write_pipe(real_cmd
, stdin
)
155 def read_pipe(c
, ignore_error
=False):
157 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
159 expand
= isinstance(c
,basestring
)
160 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
161 (out
, err
) = p
.communicate()
162 if p
.returncode
!= 0 and not ignore_error
:
163 die('Command failed: %s\nError: %s' % (str(c
), err
))
166 def p4_read_pipe(c
, ignore_error
=False):
167 real_cmd
= p4_build_cmd(c
)
168 return read_pipe(real_cmd
, ignore_error
)
170 def read_pipe_lines(c
):
172 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
174 expand
= isinstance(c
, basestring
)
175 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
177 val
= pipe
.readlines()
178 if pipe
.close() or p
.wait():
179 die('Command failed: %s' % str(c
))
183 def p4_read_pipe_lines(c
):
184 """Specifically invoke p4 on the command supplied. """
185 real_cmd
= p4_build_cmd(c
)
186 return read_pipe_lines(real_cmd
)
188 def p4_has_command(cmd
):
189 """Ask p4 for help on this command. If it returns an error, the
190 command does not exist in this version of p4."""
191 real_cmd
= p4_build_cmd(["help", cmd
])
192 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
193 stderr
=subprocess
.PIPE
)
195 return p
.returncode
== 0
197 def p4_has_move_command():
198 """See if the move command exists, that it supports -k, and that
199 it has not been administratively disabled. The arguments
200 must be correct, but the filenames do not have to exist. Use
201 ones with wildcards so even if they exist, it will fail."""
203 if not p4_has_command("move"):
205 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
206 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
207 (out
, err
) = p
.communicate()
208 # return code will be 1 in either case
209 if err
.find("Invalid option") >= 0:
211 if err
.find("disabled") >= 0:
213 # assume it failed because @... was invalid changelist
216 def system(cmd
, ignore_error
=False):
217 expand
= isinstance(cmd
,basestring
)
219 sys
.stderr
.write("executing %s\n" % str(cmd
))
220 retcode
= subprocess
.call(cmd
, shell
=expand
)
221 if retcode
and not ignore_error
:
222 raise CalledProcessError(retcode
, cmd
)
227 """Specifically invoke p4 as the system command. """
228 real_cmd
= p4_build_cmd(cmd
)
229 expand
= isinstance(real_cmd
, basestring
)
230 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
232 raise CalledProcessError(retcode
, real_cmd
)
234 _p4_version_string
= None
235 def p4_version_string():
236 """Read the version string, showing just the last line, which
237 hopefully is the interesting version bit.
240 Perforce - The Fast Software Configuration Management System.
241 Copyright 1995-2011 Perforce Software. All rights reserved.
242 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
244 global _p4_version_string
245 if not _p4_version_string
:
246 a
= p4_read_pipe_lines(["-V"])
247 _p4_version_string
= a
[-1].rstrip()
248 return _p4_version_string
250 def p4_integrate(src
, dest
):
251 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
253 def p4_sync(f
, *options
):
254 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
257 # forcibly add file names with wildcards
258 if wildcard_present(f
):
259 p4_system(["add", "-f", f
])
261 p4_system(["add", f
])
264 p4_system(["delete", wildcard_encode(f
)])
266 def p4_edit(f
, *options
):
267 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
270 p4_system(["revert", wildcard_encode(f
)])
272 def p4_reopen(type, f
):
273 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
275 def p4_move(src
, dest
):
276 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
278 def p4_last_change():
279 results
= p4CmdList(["changes", "-m", "1"])
280 return int(results
[0]['change'])
282 def p4_describe(change
):
283 """Make sure it returns a valid result by checking for
284 the presence of field "time". Return a dict of the
287 ds
= p4CmdList(["describe", "-s", str(change
)])
289 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
293 if "p4ExitCode" in d
:
294 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
297 if d
["code"] == "error":
298 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
301 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
306 # Canonicalize the p4 type and return a tuple of the
307 # base type, plus any modifiers. See "p4 help filetypes"
308 # for a list and explanation.
310 def split_p4_type(p4type
):
312 p4_filetypes_historical
= {
313 "ctempobj": "binary+Sw",
319 "tempobj": "binary+FSw",
320 "ubinary": "binary+F",
321 "uresource": "resource+F",
322 "uxbinary": "binary+Fx",
323 "xbinary": "binary+x",
325 "xtempobj": "binary+Swx",
327 "xunicode": "unicode+x",
330 if p4type
in p4_filetypes_historical
:
331 p4type
= p4_filetypes_historical
[p4type
]
333 s
= p4type
.split("+")
341 # return the raw p4 type of a file (text, text+ko, etc)
344 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
345 return results
[0]['headType']
348 # Given a type base and modifier, return a regexp matching
349 # the keywords that can be expanded in the file
351 def p4_keywords_regexp_for_type(base
, type_mods
):
352 if base
in ("text", "unicode", "binary"):
354 if "ko" in type_mods
:
356 elif "k" in type_mods
:
357 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
361 \$ # Starts with a dollar, followed by...
362 (%s) # one of the keywords, followed by...
363 (:[^$\n]+)? # possibly an old expansion, followed by...
371 # Given a file, return a regexp matching the possible
372 # RCS keywords that will be expanded, or None for files
373 # with kw expansion turned off.
375 def p4_keywords_regexp_for_file(file):
376 if not os
.path
.exists(file):
379 (type_base
, type_mods
) = split_p4_type(p4_type(file))
380 return p4_keywords_regexp_for_type(type_base
, type_mods
)
382 def setP4ExecBit(file, mode
):
383 # Reopens an already open file and changes the execute bit to match
384 # the execute bit setting in the passed in mode.
388 if not isModeExec(mode
):
389 p4Type
= getP4OpenedType(file)
390 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
391 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
392 if p4Type
[-1] == "+":
393 p4Type
= p4Type
[0:-1]
395 p4_reopen(p4Type
, file)
397 def getP4OpenedType(file):
398 # Returns the perforce file type for the given file.
400 result
= p4_read_pipe(["opened", wildcard_encode(file)])
401 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
403 return match
.group(1)
405 die("Could not determine file type for %s (result: '%s')" % (file, result
))
407 # Return the set of all p4 labels
408 def getP4Labels(depotPaths
):
410 if isinstance(depotPaths
,basestring
):
411 depotPaths
= [depotPaths
]
413 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
419 # Return the set of all git tags
422 for line
in read_pipe_lines(["git", "tag"]):
427 def diffTreePattern():
428 # This is a simple generator for the diff tree regex pattern. This could be
429 # a class variable if this and parseDiffTreeEntry were a part of a class.
430 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
434 def parseDiffTreeEntry(entry
):
435 """Parses a single diff tree entry into its component elements.
437 See git-diff-tree(1) manpage for details about the format of the diff
438 output. This method returns a dictionary with the following elements:
440 src_mode - The mode of the source file
441 dst_mode - The mode of the destination file
442 src_sha1 - The sha1 for the source file
443 dst_sha1 - The sha1 fr the destination file
444 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
445 status_score - The score for the status (applicable for 'C' and 'R'
446 statuses). This is None if there is no score.
447 src - The path for the source file.
448 dst - The path for the destination file. This is only present for
449 copy or renames. If it is not present, this is None.
451 If the pattern is not matched, None is returned."""
453 match
= diffTreePattern().next().match(entry
)
456 'src_mode': match
.group(1),
457 'dst_mode': match
.group(2),
458 'src_sha1': match
.group(3),
459 'dst_sha1': match
.group(4),
460 'status': match
.group(5),
461 'status_score': match
.group(6),
462 'src': match
.group(7),
463 'dst': match
.group(10)
467 def isModeExec(mode
):
468 # Returns True if the given git mode represents an executable file,
470 return mode
[-3:] == "755"
472 def isModeExecChanged(src_mode
, dst_mode
):
473 return isModeExec(src_mode
) != isModeExec(dst_mode
)
475 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
477 if isinstance(cmd
,basestring
):
484 cmd
= p4_build_cmd(cmd
)
486 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
488 # Use a temporary file to avoid deadlocks without
489 # subprocess.communicate(), which would put another copy
490 # of stdout into memory.
492 if stdin
is not None:
493 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
494 if isinstance(stdin
,basestring
):
495 stdin_file
.write(stdin
)
498 stdin_file
.write(i
+ '\n')
502 p4
= subprocess
.Popen(cmd
,
505 stdout
=subprocess
.PIPE
)
510 entry
= marshal
.load(p4
.stdout
)
520 entry
["p4ExitCode"] = exitCode
526 list = p4CmdList(cmd
)
532 def p4Where(depotPath
):
533 if not depotPath
.endswith("/"):
535 depotPathLong
= depotPath
+ "..."
536 outputList
= p4CmdList(["where", depotPathLong
])
538 for entry
in outputList
:
539 if "depotFile" in entry
:
540 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
541 # The base path always ends with "/...".
542 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
545 elif "data" in entry
:
546 data
= entry
.get("data")
547 space
= data
.find(" ")
548 if data
[:space
] == depotPath
:
553 if output
["code"] == "error":
557 clientPath
= output
.get("path")
558 elif "data" in output
:
559 data
= output
.get("data")
560 lastSpace
= data
.rfind(" ")
561 clientPath
= data
[lastSpace
+ 1:]
563 if clientPath
.endswith("..."):
564 clientPath
= clientPath
[:-3]
567 def currentGitBranch():
568 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
573 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
575 def isValidGitDir(path
):
576 return git_dir(path
) != None
578 def parseRevision(ref
):
579 return read_pipe("git rev-parse %s" % ref
).strip()
581 def branchExists(ref
):
582 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
586 def extractLogMessageFromGitCommit(commit
):
589 ## fixme: title is first line of commit, not 1st paragraph.
591 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
600 def extractSettingsGitLog(log
):
602 for line
in log
.split("\n"):
604 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
608 assignments
= m
.group(1).split (':')
609 for a
in assignments
:
611 key
= vals
[0].strip()
612 val
= ('='.join (vals
[1:])).strip()
613 if val
.endswith ('\"') and val
.startswith('"'):
618 paths
= values
.get("depot-paths")
620 paths
= values
.get("depot-path")
622 values
['depot-paths'] = paths
.split(',')
625 def gitBranchExists(branch
):
626 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
627 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
628 return proc
.wait() == 0;
632 def gitConfig(key
, typeSpecifier
=None):
633 if not _gitConfig
.has_key(key
):
634 cmd
= [ "git", "config" ]
636 cmd
+= [ typeSpecifier
]
638 s
= read_pipe(cmd
, ignore_error
=True)
639 _gitConfig
[key
] = s
.strip()
640 return _gitConfig
[key
]
642 def gitConfigBool(key
):
643 """Return a bool, using git config --bool. It is True only if the
644 variable is set to true, and False if set to false or not present
647 if not _gitConfig
.has_key(key
):
648 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
649 return _gitConfig
[key
]
651 def gitConfigInt(key
):
652 if not _gitConfig
.has_key(key
):
653 cmd
= [ "git", "config", "--int", key
]
654 s
= read_pipe(cmd
, ignore_error
=True)
657 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
659 _gitConfig
[key
] = None
660 return _gitConfig
[key
]
662 def gitConfigList(key
):
663 if not _gitConfig
.has_key(key
):
664 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
665 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
666 if _gitConfig
[key
] == ['']:
668 return _gitConfig
[key
]
670 def p4BranchesInGit(branchesAreInRemotes
=True):
671 """Find all the branches whose names start with "p4/", looking
672 in remotes or heads as specified by the argument. Return
673 a dictionary of { branch: revision } for each one found.
674 The branch names are the short names, without any
679 cmdline
= "git rev-parse --symbolic "
680 if branchesAreInRemotes
:
681 cmdline
+= "--remotes"
683 cmdline
+= "--branches"
685 for line
in read_pipe_lines(cmdline
):
689 if not line
.startswith('p4/'):
691 # special symbolic ref to p4/master
692 if line
== "p4/HEAD":
695 # strip off p4/ prefix
696 branch
= line
[len("p4/"):]
698 branches
[branch
] = parseRevision(line
)
702 def branch_exists(branch
):
703 """Make sure that the given ref name really exists."""
705 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
706 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
707 out
, _
= p
.communicate()
710 # expect exactly one line of output: the branch name
711 return out
.rstrip() == branch
713 def findUpstreamBranchPoint(head
= "HEAD"):
714 branches
= p4BranchesInGit()
715 # map from depot-path to branch name
716 branchByDepotPath
= {}
717 for branch
in branches
.keys():
718 tip
= branches
[branch
]
719 log
= extractLogMessageFromGitCommit(tip
)
720 settings
= extractSettingsGitLog(log
)
721 if settings
.has_key("depot-paths"):
722 paths
= ",".join(settings
["depot-paths"])
723 branchByDepotPath
[paths
] = "remotes/p4/" + branch
727 while parent
< 65535:
728 commit
= head
+ "~%s" % parent
729 log
= extractLogMessageFromGitCommit(commit
)
730 settings
= extractSettingsGitLog(log
)
731 if settings
.has_key("depot-paths"):
732 paths
= ",".join(settings
["depot-paths"])
733 if branchByDepotPath
.has_key(paths
):
734 return [branchByDepotPath
[paths
], settings
]
738 return ["", settings
]
740 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
742 print ("Creating/updating branch(es) in %s based on origin branch(es)"
745 originPrefix
= "origin/p4/"
747 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
749 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
752 headName
= line
[len(originPrefix
):]
753 remoteHead
= localRefPrefix
+ headName
756 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
757 if (not original
.has_key('depot-paths')
758 or not original
.has_key('change')):
762 if not gitBranchExists(remoteHead
):
764 print "creating %s" % remoteHead
767 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
768 if settings
.has_key('change') > 0:
769 if settings
['depot-paths'] == original
['depot-paths']:
770 originP4Change
= int(original
['change'])
771 p4Change
= int(settings
['change'])
772 if originP4Change
> p4Change
:
773 print ("%s (%s) is newer than %s (%s). "
774 "Updating p4 branch from origin."
775 % (originHead
, originP4Change
,
776 remoteHead
, p4Change
))
779 print ("Ignoring: %s was imported from %s while "
780 "%s was imported from %s"
781 % (originHead
, ','.join(original
['depot-paths']),
782 remoteHead
, ','.join(settings
['depot-paths'])))
785 system("git update-ref %s %s" % (remoteHead
, originHead
))
787 def originP4BranchesExist():
788 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
791 def p4ParseNumericChangeRange(parts
):
792 changeStart
= int(parts
[0][1:])
793 if parts
[1] == '#head':
794 changeEnd
= p4_last_change()
796 changeEnd
= int(parts
[1])
798 return (changeStart
, changeEnd
)
800 def chooseBlockSize(blockSize
):
804 return defaultBlockSize
806 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
809 # Parse the change range into start and end. Try to find integer
810 # revision ranges as these can be broken up into blocks to avoid
811 # hitting server-side limits (maxrows, maxscanresults). But if
812 # that doesn't work, fall back to using the raw revision specifier
813 # strings, without using block mode.
815 if changeRange
is None or changeRange
== '':
817 changeEnd
= p4_last_change()
818 block_size
= chooseBlockSize(requestedBlockSize
)
820 parts
= changeRange
.split(',')
821 assert len(parts
) == 2
823 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
824 block_size
= chooseBlockSize(requestedBlockSize
)
826 changeStart
= parts
[0][1:]
828 if requestedBlockSize
:
829 die("cannot use --changes-block-size with non-numeric revisions")
834 # Retrieve changes a block at a time, to prevent running
835 # into a MaxResults/MaxScanRows error from the server.
841 end
= min(changeEnd
, changeStart
+ block_size
)
842 revisionRange
= "%d,%d" % (changeStart
, end
)
844 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
847 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
849 # Insert changes in chronological order
850 for line
in reversed(p4_read_pipe_lines(cmd
)):
851 changes
.append(int(line
.split(" ")[1]))
859 changeStart
= end
+ 1
861 changes
= sorted(changes
)
864 def p4PathStartsWith(path
, prefix
):
865 # This method tries to remedy a potential mixed-case issue:
867 # If UserA adds //depot/DirA/file1
868 # and UserB adds //depot/dira/file2
870 # we may or may not have a problem. If you have core.ignorecase=true,
871 # we treat DirA and dira as the same directory
872 if gitConfigBool("core.ignorecase"):
873 return path
.lower().startswith(prefix
.lower())
874 return path
.startswith(prefix
)
877 """Look at the p4 client spec, create a View() object that contains
878 all the mappings, and return it."""
880 specList
= p4CmdList("client -o")
881 if len(specList
) != 1:
882 die('Output from "client -o" is %d lines, expecting 1' %
885 # dictionary of all client parameters
889 client_name
= entry
["Client"]
891 # just the keys that start with "View"
892 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
895 view
= View(client_name
)
897 # append the lines, in order, to the view
898 for view_num
in range(len(view_keys
)):
899 k
= "View%d" % view_num
900 if k
not in view_keys
:
901 die("Expected view key %s missing" % k
)
902 view
.append(entry
[k
])
907 """Grab the client directory."""
909 output
= p4CmdList("client -o")
911 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
914 if "Root" not in entry
:
915 die('Client has no "Root"')
920 # P4 wildcards are not allowed in filenames. P4 complains
921 # if you simply add them, but you can force it with "-f", in
922 # which case it translates them into %xx encoding internally.
924 def wildcard_decode(path
):
925 # Search for and fix just these four characters. Do % last so
926 # that fixing it does not inadvertently create new %-escapes.
927 # Cannot have * in a filename in windows; untested as to
928 # what p4 would do in such a case.
929 if not platform
.system() == "Windows":
930 path
= path
.replace("%2A", "*")
931 path
= path
.replace("%23", "#") \
932 .replace("%40", "@") \
936 def wildcard_encode(path
):
937 # do % first to avoid double-encoding the %s introduced here
938 path
= path
.replace("%", "%25") \
939 .replace("*", "%2A") \
940 .replace("#", "%23") \
944 def wildcard_present(path
):
945 m
= re
.search("[*#@%]", path
)
948 class LargeFileSystem(object):
949 """Base class for large file system support."""
951 def __init__(self
, writeToGitStream
):
952 self
.largeFiles
= set()
953 self
.writeToGitStream
= writeToGitStream
955 def generatePointer(self
, cloneDestination
, contentFile
):
956 """Return the content of a pointer file that is stored in Git instead of
957 the actual content."""
958 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
960 def pushFile(self
, localLargeFile
):
961 """Push the actual content which is not stored in the Git repository to
963 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
965 def hasLargeFileExtension(self
, relPath
):
968 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
972 def generateTempFile(self
, contents
):
973 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
977 return contentFile
.name
979 def exceedsLargeFileThreshold(self
, relPath
, contents
):
980 if gitConfigInt('git-p4.largeFileThreshold'):
981 contentsSize
= sum(len(d
) for d
in contents
)
982 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
984 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
985 contentsSize
= sum(len(d
) for d
in contents
)
986 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
988 contentTempFile
= self
.generateTempFile(contents
)
989 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
990 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
991 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
993 compressedContentsSize
= zf
.infolist()[0].compress_size
994 os
.remove(contentTempFile
)
995 os
.remove(compressedContentFile
.name
)
996 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
1000 def addLargeFile(self
, relPath
):
1001 self
.largeFiles
.add(relPath
)
1003 def removeLargeFile(self
, relPath
):
1004 self
.largeFiles
.remove(relPath
)
1006 def isLargeFile(self
, relPath
):
1007 return relPath
in self
.largeFiles
1009 def processContent(self
, git_mode
, relPath
, contents
):
1010 """Processes the content of git fast import. This method decides if a
1011 file is stored in the large file system and handles all necessary
1013 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1014 contentTempFile
= self
.generateTempFile(contents
)
1015 (git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1017 # Move temp file to final location in large file system
1018 largeFileDir
= os
.path
.dirname(localLargeFile
)
1019 if not os
.path
.isdir(largeFileDir
):
1020 os
.makedirs(largeFileDir
)
1021 shutil
.move(contentTempFile
, localLargeFile
)
1022 self
.addLargeFile(relPath
)
1023 if gitConfigBool('git-p4.largeFilePush'):
1024 self
.pushFile(localLargeFile
)
1026 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1027 return (git_mode
, contents
)
1029 class MockLFS(LargeFileSystem
):
1030 """Mock large file system for testing."""
1032 def generatePointer(self
, contentFile
):
1033 """The pointer content is the original content prefixed with "pointer-".
1034 The local filename of the large file storage is derived from the file content.
1036 with
open(contentFile
, 'r') as f
:
1039 pointerContents
= 'pointer-' + content
1040 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1041 return (gitMode
, pointerContents
, localLargeFile
)
1043 def pushFile(self
, localLargeFile
):
1044 """The remote filename of the large file storage is the same as the local
1045 one but in a different directory.
1047 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1048 if not os
.path
.exists(remotePath
):
1049 os
.makedirs(remotePath
)
1050 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1052 class GitLFS(LargeFileSystem
):
1053 """Git LFS as backend for the git-p4 large file system.
1054 See https://git-lfs.github.com/ for details."""
1056 def __init__(self
, *args
):
1057 LargeFileSystem
.__init
__(self
, *args
)
1058 self
.baseGitAttributes
= []
1060 def generatePointer(self
, contentFile
):
1061 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1062 mode and content which is stored in the Git repository instead of
1063 the actual content. Return also the new location of the actual
1066 pointerProcess
= subprocess
.Popen(
1067 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1068 stdout
=subprocess
.PIPE
1070 pointerFile
= pointerProcess
.stdout
.read()
1071 if pointerProcess
.wait():
1072 os
.remove(contentFile
)
1073 die('git-lfs pointer command failed. Did you install the extension?')
1075 # Git LFS removed the preamble in the output of the 'pointer' command
1076 # starting from version 1.2.0. Check for the preamble here to support
1078 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1079 if pointerFile
.startswith('Git LFS pointer for'):
1080 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1082 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1083 localLargeFile
= os
.path
.join(
1085 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1088 # LFS Spec states that pointer files should not have the executable bit set.
1090 return (gitMode
, pointerFile
, localLargeFile
)
1092 def pushFile(self
, localLargeFile
):
1093 uploadProcess
= subprocess
.Popen(
1094 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1096 if uploadProcess
.wait():
1097 die('git-lfs push command failed. Did you define a remote?')
1099 def generateGitAttributes(self
):
1101 self
.baseGitAttributes
+
1105 '# Git LFS (see https://git-lfs.github.com/)\n',
1108 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1109 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1111 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1112 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1116 def addLargeFile(self
, relPath
):
1117 LargeFileSystem
.addLargeFile(self
, relPath
)
1118 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1120 def removeLargeFile(self
, relPath
):
1121 LargeFileSystem
.removeLargeFile(self
, relPath
)
1122 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1124 def processContent(self
, git_mode
, relPath
, contents
):
1125 if relPath
== '.gitattributes':
1126 self
.baseGitAttributes
= contents
1127 return (git_mode
, self
.generateGitAttributes())
1129 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1133 self
.usage
= "usage: %prog [options]"
1134 self
.needsGit
= True
1135 self
.verbose
= False
1139 self
.userMapFromPerforceServer
= False
1140 self
.myP4UserId
= None
1144 return self
.myP4UserId
1146 results
= p4CmdList("user -o")
1148 if r
.has_key('User'):
1149 self
.myP4UserId
= r
['User']
1151 die("Could not find your p4 user id")
1153 def p4UserIsMe(self
, p4User
):
1154 # return True if the given p4 user is actually me
1155 me
= self
.p4UserId()
1156 if not p4User
or p4User
!= me
:
1161 def getUserCacheFilename(self
):
1162 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1163 return home
+ "/.gitp4-usercache.txt"
1165 def getUserMapFromPerforceServer(self
):
1166 if self
.userMapFromPerforceServer
:
1171 for output
in p4CmdList("users"):
1172 if not output
.has_key("User"):
1174 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1175 self
.emails
[output
["Email"]] = output
["User"]
1177 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1178 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1179 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1180 if mapUser
and len(mapUser
[0]) == 3:
1181 user
= mapUser
[0][0]
1182 fullname
= mapUser
[0][1]
1183 email
= mapUser
[0][2]
1184 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1185 self
.emails
[email
] = user
1188 for (key
, val
) in self
.users
.items():
1189 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1191 open(self
.getUserCacheFilename(), "wb").write(s
)
1192 self
.userMapFromPerforceServer
= True
1194 def loadUserMapFromCache(self
):
1196 self
.userMapFromPerforceServer
= False
1198 cache
= open(self
.getUserCacheFilename(), "rb")
1199 lines
= cache
.readlines()
1202 entry
= line
.strip().split("\t")
1203 self
.users
[entry
[0]] = entry
[1]
1205 self
.getUserMapFromPerforceServer()
1207 class P4Debug(Command
):
1209 Command
.__init
__(self
)
1211 self
.description
= "A tool to debug the output of p4 -G."
1212 self
.needsGit
= False
1214 def run(self
, args
):
1216 for output
in p4CmdList(args
):
1217 print 'Element: %d' % j
1222 class P4RollBack(Command
):
1224 Command
.__init
__(self
)
1226 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1228 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1229 self
.rollbackLocalBranches
= False
1231 def run(self
, args
):
1234 maxChange
= int(args
[0])
1236 if "p4ExitCode" in p4Cmd("changes -m 1"):
1237 die("Problems executing p4");
1239 if self
.rollbackLocalBranches
:
1240 refPrefix
= "refs/heads/"
1241 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1243 refPrefix
= "refs/remotes/"
1244 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1247 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1249 ref
= refPrefix
+ line
1250 log
= extractLogMessageFromGitCommit(ref
)
1251 settings
= extractSettingsGitLog(log
)
1253 depotPaths
= settings
['depot-paths']
1254 change
= settings
['change']
1258 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1259 for p
in depotPaths
]))) == 0:
1260 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1261 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1264 while change
and int(change
) > maxChange
:
1267 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1268 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1269 log
= extractLogMessageFromGitCommit(ref
)
1270 settings
= extractSettingsGitLog(log
)
1273 depotPaths
= settings
['depot-paths']
1274 change
= settings
['change']
1277 print "%s rewound to %s" % (ref
, change
)
1281 class P4Submit(Command
, P4UserMap
):
1283 conflict_behavior_choices
= ("ask", "skip", "quit")
1286 Command
.__init
__(self
)
1287 P4UserMap
.__init
__(self
)
1289 optparse
.make_option("--origin", dest
="origin"),
1290 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1291 # preserve the user, requires relevant p4 permissions
1292 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1293 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1294 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1295 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1296 optparse
.make_option("--conflict", dest
="conflict_behavior",
1297 choices
=self
.conflict_behavior_choices
),
1298 optparse
.make_option("--branch", dest
="branch"),
1300 self
.description
= "Submit changes from git to the perforce depot."
1301 self
.usage
+= " [name of git branch to submit into perforce depot]"
1303 self
.detectRenames
= False
1304 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1305 self
.dry_run
= False
1306 self
.prepare_p4_only
= False
1307 self
.conflict_behavior
= None
1308 self
.isWindows
= (platform
.system() == "Windows")
1309 self
.exportLabels
= False
1310 self
.p4HasMoveCommand
= p4_has_move_command()
1313 if gitConfig('git-p4.largeFileSystem'):
1314 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1317 if len(p4CmdList("opened ...")) > 0:
1318 die("You have files opened with perforce! Close them before starting the sync.")
1320 def separate_jobs_from_description(self
, message
):
1321 """Extract and return a possible Jobs field in the commit
1322 message. It goes into a separate section in the p4 change
1325 A jobs line starts with "Jobs:" and looks like a new field
1326 in a form. Values are white-space separated on the same
1327 line or on following lines that start with a tab.
1329 This does not parse and extract the full git commit message
1330 like a p4 form. It just sees the Jobs: line as a marker
1331 to pass everything from then on directly into the p4 form,
1332 but outside the description section.
1334 Return a tuple (stripped log message, jobs string)."""
1336 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1338 return (message
, None)
1340 jobtext
= message
[m
.start():]
1341 stripped_message
= message
[:m
.start()].rstrip()
1342 return (stripped_message
, jobtext
)
1344 def prepareLogMessage(self
, template
, message
, jobs
):
1345 """Edits the template returned from "p4 change -o" to insert
1346 the message in the Description field, and the jobs text in
1350 inDescriptionSection
= False
1352 for line
in template
.split("\n"):
1353 if line
.startswith("#"):
1354 result
+= line
+ "\n"
1357 if inDescriptionSection
:
1358 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1359 inDescriptionSection
= False
1360 # insert Jobs section
1362 result
+= jobs
+ "\n"
1366 if line
.startswith("Description:"):
1367 inDescriptionSection
= True
1369 for messageLine
in message
.split("\n"):
1370 line
+= "\t" + messageLine
+ "\n"
1372 result
+= line
+ "\n"
1376 def patchRCSKeywords(self
, file, pattern
):
1377 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1378 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1380 outFile
= os
.fdopen(handle
, "w+")
1381 inFile
= open(file, "r")
1382 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1383 for line
in inFile
.readlines():
1384 line
= regexp
.sub(r
'$\1$', line
)
1388 # Forcibly overwrite the original file
1390 shutil
.move(outFileName
, file)
1392 # cleanup our temporary file
1393 os
.unlink(outFileName
)
1394 print "Failed to strip RCS keywords in %s" % file
1397 print "Patched up RCS keywords in %s" % file
1399 def p4UserForCommit(self
,id):
1400 # Return the tuple (perforce user,git email) for a given git commit id
1401 self
.getUserMapFromPerforceServer()
1402 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1403 "--format=%ae", id])
1404 gitEmail
= gitEmail
.strip()
1405 if not self
.emails
.has_key(gitEmail
):
1406 return (None,gitEmail
)
1408 return (self
.emails
[gitEmail
],gitEmail
)
1410 def checkValidP4Users(self
,commits
):
1411 # check if any git authors cannot be mapped to p4 users
1413 (user
,email
) = self
.p4UserForCommit(id)
1415 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1416 if gitConfigBool("git-p4.allowMissingP4Users"):
1419 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1421 def lastP4Changelist(self
):
1422 # Get back the last changelist number submitted in this client spec. This
1423 # then gets used to patch up the username in the change. If the same
1424 # client spec is being used by multiple processes then this might go
1426 results
= p4CmdList("client -o") # find the current client
1429 if r
.has_key('Client'):
1430 client
= r
['Client']
1433 die("could not get client spec")
1434 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1436 if r
.has_key('change'):
1438 die("Could not get changelist number for last submit - cannot patch up user details")
1440 def modifyChangelistUser(self
, changelist
, newUser
):
1441 # fixup the user field of a changelist after it has been submitted.
1442 changes
= p4CmdList("change -o %s" % changelist
)
1443 if len(changes
) != 1:
1444 die("Bad output from p4 change modifying %s to user %s" %
1445 (changelist
, newUser
))
1448 if c
['User'] == newUser
: return # nothing to do
1450 input = marshal
.dumps(c
)
1452 result
= p4CmdList("change -f -i", stdin
=input)
1454 if r
.has_key('code'):
1455 if r
['code'] == 'error':
1456 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1457 if r
.has_key('data'):
1458 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1460 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1462 def canChangeChangelists(self
):
1463 # check to see if we have p4 admin or super-user permissions, either of
1464 # which are required to modify changelists.
1465 results
= p4CmdList(["protects", self
.depotPath
])
1467 if r
.has_key('perm'):
1468 if r
['perm'] == 'admin':
1470 if r
['perm'] == 'super':
1474 def prepareSubmitTemplate(self
):
1475 """Run "p4 change -o" to grab a change specification template.
1476 This does not use "p4 -G", as it is nice to keep the submission
1477 template in original order, since a human might edit it.
1479 Remove lines in the Files section that show changes to files
1480 outside the depot path we're committing into."""
1482 [upstream
, settings
] = findUpstreamBranchPoint()
1485 inFilesSection
= False
1486 for line
in p4_read_pipe_lines(['change', '-o']):
1487 if line
.endswith("\r\n"):
1488 line
= line
[:-2] + "\n"
1490 if line
.startswith("\t"):
1491 # path starts and ends with a tab
1493 lastTab
= path
.rfind("\t")
1495 path
= path
[:lastTab
]
1496 if settings
.has_key('depot-paths'):
1497 if not [p
for p
in settings
['depot-paths']
1498 if p4PathStartsWith(path
, p
)]:
1501 if not p4PathStartsWith(path
, self
.depotPath
):
1504 inFilesSection
= False
1506 if line
.startswith("Files:"):
1507 inFilesSection
= True
1513 def edit_template(self
, template_file
):
1514 """Invoke the editor to let the user change the submission
1515 message. Return true if okay to continue with the submit."""
1517 # if configured to skip the editing part, just submit
1518 if gitConfigBool("git-p4.skipSubmitEdit"):
1521 # look at the modification time, to check later if the user saved
1523 mtime
= os
.stat(template_file
).st_mtime
1526 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1527 editor
= os
.environ
.get("P4EDITOR")
1529 editor
= read_pipe("git var GIT_EDITOR").strip()
1530 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1532 # If the file was not saved, prompt to see if this patch should
1533 # be skipped. But skip this verification step if configured so.
1534 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1537 # modification time updated means user saved the file
1538 if os
.stat(template_file
).st_mtime
> mtime
:
1542 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1548 def get_diff_description(self
, editedFiles
, filesToAdd
):
1550 if os
.environ
.has_key("P4DIFF"):
1551 del(os
.environ
["P4DIFF"])
1553 for editedFile
in editedFiles
:
1554 diff
+= p4_read_pipe(['diff', '-du',
1555 wildcard_encode(editedFile
)])
1559 for newFile
in filesToAdd
:
1560 newdiff
+= "==== new file ====\n"
1561 newdiff
+= "--- /dev/null\n"
1562 newdiff
+= "+++ %s\n" % newFile
1563 f
= open(newFile
, "r")
1564 for line
in f
.readlines():
1565 newdiff
+= "+" + line
1568 return (diff
+ newdiff
).replace('\r\n', '\n')
1570 def applyCommit(self
, id):
1571 """Apply one commit, return True if it succeeded."""
1573 print "Applying", read_pipe(["git", "show", "-s",
1574 "--format=format:%h %s", id])
1576 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1578 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1580 filesToChangeType
= set()
1581 filesToDelete
= set()
1583 pureRenameCopy
= set()
1584 filesToChangeExecBit
= {}
1587 diff
= parseDiffTreeEntry(line
)
1588 modifier
= diff
['status']
1592 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1593 filesToChangeExecBit
[path
] = diff
['dst_mode']
1594 editedFiles
.add(path
)
1595 elif modifier
== "A":
1596 filesToAdd
.add(path
)
1597 filesToChangeExecBit
[path
] = diff
['dst_mode']
1598 if path
in filesToDelete
:
1599 filesToDelete
.remove(path
)
1600 elif modifier
== "D":
1601 filesToDelete
.add(path
)
1602 if path
in filesToAdd
:
1603 filesToAdd
.remove(path
)
1604 elif modifier
== "C":
1605 src
, dest
= diff
['src'], diff
['dst']
1606 p4_integrate(src
, dest
)
1607 pureRenameCopy
.add(dest
)
1608 if diff
['src_sha1'] != diff
['dst_sha1']:
1610 pureRenameCopy
.discard(dest
)
1611 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1613 pureRenameCopy
.discard(dest
)
1614 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1616 # turn off read-only attribute
1617 os
.chmod(dest
, stat
.S_IWRITE
)
1619 editedFiles
.add(dest
)
1620 elif modifier
== "R":
1621 src
, dest
= diff
['src'], diff
['dst']
1622 if self
.p4HasMoveCommand
:
1623 p4_edit(src
) # src must be open before move
1624 p4_move(src
, dest
) # opens for (move/delete, move/add)
1626 p4_integrate(src
, dest
)
1627 if diff
['src_sha1'] != diff
['dst_sha1']:
1630 pureRenameCopy
.add(dest
)
1631 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1632 if not self
.p4HasMoveCommand
:
1633 p4_edit(dest
) # with move: already open, writable
1634 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1635 if not self
.p4HasMoveCommand
:
1637 os
.chmod(dest
, stat
.S_IWRITE
)
1639 filesToDelete
.add(src
)
1640 editedFiles
.add(dest
)
1641 elif modifier
== "T":
1642 filesToChangeType
.add(path
)
1644 die("unknown modifier %s for %s" % (modifier
, path
))
1646 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1647 patchcmd
= diffcmd
+ " | git apply "
1648 tryPatchCmd
= patchcmd
+ "--check -"
1649 applyPatchCmd
= patchcmd
+ "--check --apply -"
1650 patch_succeeded
= True
1652 if os
.system(tryPatchCmd
) != 0:
1653 fixed_rcs_keywords
= False
1654 patch_succeeded
= False
1655 print "Unfortunately applying the change failed!"
1657 # Patch failed, maybe it's just RCS keyword woes. Look through
1658 # the patch to see if that's possible.
1659 if gitConfigBool("git-p4.attemptRCSCleanup"):
1663 for file in editedFiles | filesToDelete
:
1664 # did this file's delta contain RCS keywords?
1665 pattern
= p4_keywords_regexp_for_file(file)
1668 # this file is a possibility...look for RCS keywords.
1669 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1670 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1671 if regexp
.search(line
):
1673 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1674 kwfiles
[file] = pattern
1677 for file in kwfiles
:
1679 print "zapping %s with %s" % (line
,pattern
)
1680 # File is being deleted, so not open in p4. Must
1681 # disable the read-only bit on windows.
1682 if self
.isWindows
and file not in editedFiles
:
1683 os
.chmod(file, stat
.S_IWRITE
)
1684 self
.patchRCSKeywords(file, kwfiles
[file])
1685 fixed_rcs_keywords
= True
1687 if fixed_rcs_keywords
:
1688 print "Retrying the patch with RCS keywords cleaned up"
1689 if os
.system(tryPatchCmd
) == 0:
1690 patch_succeeded
= True
1692 if not patch_succeeded
:
1693 for f
in editedFiles
:
1698 # Apply the patch for real, and do add/delete/+x handling.
1700 system(applyPatchCmd
)
1702 for f
in filesToChangeType
:
1703 p4_edit(f
, "-t", "auto")
1704 for f
in filesToAdd
:
1706 for f
in filesToDelete
:
1710 # Set/clear executable bits
1711 for f
in filesToChangeExecBit
.keys():
1712 mode
= filesToChangeExecBit
[f
]
1713 setP4ExecBit(f
, mode
)
1716 # Build p4 change description, starting with the contents
1717 # of the git commit message.
1719 logMessage
= extractLogMessageFromGitCommit(id)
1720 logMessage
= logMessage
.strip()
1721 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1723 template
= self
.prepareSubmitTemplate()
1724 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1726 if self
.preserveUser
:
1727 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1729 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1730 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1731 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1732 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1734 separatorLine
= "######## everything below this line is just the diff #######\n"
1735 if not self
.prepare_p4_only
:
1736 submitTemplate
+= separatorLine
1737 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1739 (handle
, fileName
) = tempfile
.mkstemp()
1740 tmpFile
= os
.fdopen(handle
, "w+b")
1742 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1743 tmpFile
.write(submitTemplate
)
1746 if self
.prepare_p4_only
:
1748 # Leave the p4 tree prepared, and the submit template around
1749 # and let the user decide what to do next
1752 print "P4 workspace prepared for submission."
1753 print "To submit or revert, go to client workspace"
1754 print " " + self
.clientPath
1756 print "To submit, use \"p4 submit\" to write a new description,"
1757 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1758 " \"git p4\"." % fileName
1759 print "You can delete the file \"%s\" when finished." % fileName
1761 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1762 print "To preserve change ownership by user %s, you must\n" \
1763 "do \"p4 change -f <change>\" after submitting and\n" \
1764 "edit the User field."
1766 print "After submitting, renamed files must be re-synced."
1767 print "Invoke \"p4 sync -f\" on each of these files:"
1768 for f
in pureRenameCopy
:
1772 print "To revert the changes, use \"p4 revert ...\", and delete"
1773 print "the submit template file \"%s\"" % fileName
1775 print "Since the commit adds new files, they must be deleted:"
1776 for f
in filesToAdd
:
1782 # Let the user edit the change description, then submit it.
1787 if self
.edit_template(fileName
):
1788 # read the edited message and submit
1789 tmpFile
= open(fileName
, "rb")
1790 message
= tmpFile
.read()
1793 message
= message
.replace("\r\n", "\n")
1794 submitTemplate
= message
[:message
.index(separatorLine
)]
1795 p4_write_pipe(['submit', '-i'], submitTemplate
)
1797 if self
.preserveUser
:
1799 # Get last changelist number. Cannot easily get it from
1800 # the submit command output as the output is
1802 changelist
= self
.lastP4Changelist()
1803 self
.modifyChangelistUser(changelist
, p4User
)
1805 # The rename/copy happened by applying a patch that created a
1806 # new file. This leaves it writable, which confuses p4.
1807 for f
in pureRenameCopy
:
1814 print "Submission cancelled, undoing p4 changes."
1815 for f
in editedFiles
:
1817 for f
in filesToAdd
:
1820 for f
in filesToDelete
:
1826 # Export git tags as p4 labels. Create a p4 label and then tag
1828 def exportGitTags(self
, gitTags
):
1829 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1830 if len(validLabelRegexp
) == 0:
1831 validLabelRegexp
= defaultLabelRegexp
1832 m
= re
.compile(validLabelRegexp
)
1834 for name
in gitTags
:
1836 if not m
.match(name
):
1838 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1841 # Get the p4 commit this corresponds to
1842 logMessage
= extractLogMessageFromGitCommit(name
)
1843 values
= extractSettingsGitLog(logMessage
)
1845 if not values
.has_key('change'):
1846 # a tag pointing to something not sent to p4; ignore
1848 print "git tag %s does not give a p4 commit" % name
1851 changelist
= values
['change']
1853 # Get the tag details.
1857 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1860 if re
.match(r
'tag\s+', l
):
1862 elif re
.match(r
'\s*$', l
):
1869 body
= ["lightweight tag imported by git p4\n"]
1871 # Create the label - use the same view as the client spec we are using
1872 clientSpec
= getClientSpec()
1874 labelTemplate
= "Label: %s\n" % name
1875 labelTemplate
+= "Description:\n"
1877 labelTemplate
+= "\t" + b
+ "\n"
1878 labelTemplate
+= "View:\n"
1879 for depot_side
in clientSpec
.mappings
:
1880 labelTemplate
+= "\t%s\n" % depot_side
1883 print "Would create p4 label %s for tag" % name
1884 elif self
.prepare_p4_only
:
1885 print "Not creating p4 label %s for tag due to option" \
1886 " --prepare-p4-only" % name
1888 p4_write_pipe(["label", "-i"], labelTemplate
)
1891 p4_system(["tag", "-l", name
] +
1892 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1895 print "created p4 label for tag %s" % name
1897 def run(self
, args
):
1899 self
.master
= currentGitBranch()
1900 elif len(args
) == 1:
1901 self
.master
= args
[0]
1902 if not branchExists(self
.master
):
1903 die("Branch %s does not exist" % self
.master
)
1908 allowSubmit
= gitConfig("git-p4.allowSubmit")
1909 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1910 die("%s is not in git-p4.allowSubmit" % self
.master
)
1912 [upstream
, settings
] = findUpstreamBranchPoint()
1913 self
.depotPath
= settings
['depot-paths'][0]
1914 if len(self
.origin
) == 0:
1915 self
.origin
= upstream
1917 if self
.preserveUser
:
1918 if not self
.canChangeChangelists():
1919 die("Cannot preserve user names without p4 super-user or admin permissions")
1921 # if not set from the command line, try the config file
1922 if self
.conflict_behavior
is None:
1923 val
= gitConfig("git-p4.conflict")
1925 if val
not in self
.conflict_behavior_choices
:
1926 die("Invalid value '%s' for config git-p4.conflict" % val
)
1929 self
.conflict_behavior
= val
1932 print "Origin branch is " + self
.origin
1934 if len(self
.depotPath
) == 0:
1935 print "Internal error: cannot locate perforce depot path from existing branches"
1938 self
.useClientSpec
= False
1939 if gitConfigBool("git-p4.useclientspec"):
1940 self
.useClientSpec
= True
1941 if self
.useClientSpec
:
1942 self
.clientSpecDirs
= getClientSpec()
1944 # Check for the existence of P4 branches
1945 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1947 if self
.useClientSpec
and not branchesDetected
:
1948 # all files are relative to the client spec
1949 self
.clientPath
= getClientRoot()
1951 self
.clientPath
= p4Where(self
.depotPath
)
1953 if self
.clientPath
== "":
1954 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1956 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1957 self
.oldWorkingDirectory
= os
.getcwd()
1959 # ensure the clientPath exists
1960 new_client_dir
= False
1961 if not os
.path
.exists(self
.clientPath
):
1962 new_client_dir
= True
1963 os
.makedirs(self
.clientPath
)
1965 chdir(self
.clientPath
, is_client_path
=True)
1967 print "Would synchronize p4 checkout in %s" % self
.clientPath
1969 print "Synchronizing p4 checkout..."
1971 # old one was destroyed, and maybe nobody told p4
1972 p4_sync("...", "-f")
1979 commitish
= self
.master
1983 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
1984 commits
.append(line
.strip())
1987 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
1988 self
.checkAuthorship
= False
1990 self
.checkAuthorship
= True
1992 if self
.preserveUser
:
1993 self
.checkValidP4Users(commits
)
1996 # Build up a set of options to be passed to diff when
1997 # submitting each commit to p4.
1999 if self
.detectRenames
:
2000 # command-line -M arg
2001 self
.diffOpts
= "-M"
2003 # If not explicitly set check the config variable
2004 detectRenames
= gitConfig("git-p4.detectRenames")
2006 if detectRenames
.lower() == "false" or detectRenames
== "":
2008 elif detectRenames
.lower() == "true":
2009 self
.diffOpts
= "-M"
2011 self
.diffOpts
= "-M%s" % detectRenames
2013 # no command-line arg for -C or --find-copies-harder, just
2015 detectCopies
= gitConfig("git-p4.detectCopies")
2016 if detectCopies
.lower() == "false" or detectCopies
== "":
2018 elif detectCopies
.lower() == "true":
2019 self
.diffOpts
+= " -C"
2021 self
.diffOpts
+= " -C%s" % detectCopies
2023 if gitConfigBool("git-p4.detectCopiesHarder"):
2024 self
.diffOpts
+= " --find-copies-harder"
2027 # Apply the commits, one at a time. On failure, ask if should
2028 # continue to try the rest of the patches, or quit.
2033 last
= len(commits
) - 1
2034 for i
, commit
in enumerate(commits
):
2036 print " ", read_pipe(["git", "show", "-s",
2037 "--format=format:%h %s", commit
])
2040 ok
= self
.applyCommit(commit
)
2042 applied
.append(commit
)
2044 if self
.prepare_p4_only
and i
< last
:
2045 print "Processing only the first commit due to option" \
2046 " --prepare-p4-only"
2051 # prompt for what to do, or use the option/variable
2052 if self
.conflict_behavior
== "ask":
2053 print "What do you want to do?"
2054 response
= raw_input("[s]kip this commit but apply"
2055 " the rest, or [q]uit? ")
2058 elif self
.conflict_behavior
== "skip":
2060 elif self
.conflict_behavior
== "quit":
2063 die("Unknown conflict_behavior '%s'" %
2064 self
.conflict_behavior
)
2066 if response
[0] == "s":
2067 print "Skipping this commit, but applying the rest"
2069 if response
[0] == "q":
2076 chdir(self
.oldWorkingDirectory
)
2080 elif self
.prepare_p4_only
:
2082 elif len(commits
) == len(applied
):
2083 print "All commits applied!"
2087 sync
.branch
= self
.branch
2094 if len(applied
) == 0:
2095 print "No commits applied."
2097 print "Applied only the commits marked with '*':"
2103 print star
, read_pipe(["git", "show", "-s",
2104 "--format=format:%h %s", c
])
2105 print "You will have to do 'git p4 sync' and rebase."
2107 if gitConfigBool("git-p4.exportLabels"):
2108 self
.exportLabels
= True
2110 if self
.exportLabels
:
2111 p4Labels
= getP4Labels(self
.depotPath
)
2112 gitTags
= getGitTags()
2114 missingGitTags
= gitTags
- p4Labels
2115 self
.exportGitTags(missingGitTags
)
2117 # exit with error unless everything applied perfectly
2118 if len(commits
) != len(applied
):
2124 """Represent a p4 view ("p4 help views"), and map files in a
2125 repo according to the view."""
2127 def __init__(self
, client_name
):
2129 self
.client_prefix
= "//%s/" % client_name
2130 # cache results of "p4 where" to lookup client file locations
2131 self
.client_spec_path_cache
= {}
2133 def append(self
, view_line
):
2134 """Parse a view line, splitting it into depot and client
2135 sides. Append to self.mappings, preserving order. This
2136 is only needed for tag creation."""
2138 # Split the view line into exactly two words. P4 enforces
2139 # structure on these lines that simplifies this quite a bit.
2141 # Either or both words may be double-quoted.
2142 # Single quotes do not matter.
2143 # Double-quote marks cannot occur inside the words.
2144 # A + or - prefix is also inside the quotes.
2145 # There are no quotes unless they contain a space.
2146 # The line is already white-space stripped.
2147 # The two words are separated by a single space.
2149 if view_line
[0] == '"':
2150 # First word is double quoted. Find its end.
2151 close_quote_index
= view_line
.find('"', 1)
2152 if close_quote_index
<= 0:
2153 die("No first-word closing quote found: %s" % view_line
)
2154 depot_side
= view_line
[1:close_quote_index
]
2155 # skip closing quote and space
2156 rhs_index
= close_quote_index
+ 1 + 1
2158 space_index
= view_line
.find(" ")
2159 if space_index
<= 0:
2160 die("No word-splitting space found: %s" % view_line
)
2161 depot_side
= view_line
[0:space_index
]
2162 rhs_index
= space_index
+ 1
2164 # prefix + means overlay on previous mapping
2165 if depot_side
.startswith("+"):
2166 depot_side
= depot_side
[1:]
2168 # prefix - means exclude this path, leave out of mappings
2170 if depot_side
.startswith("-"):
2172 depot_side
= depot_side
[1:]
2175 self
.mappings
.append(depot_side
)
2177 def convert_client_path(self
, clientFile
):
2178 # chop off //client/ part to make it relative
2179 if not clientFile
.startswith(self
.client_prefix
):
2180 die("No prefix '%s' on clientFile '%s'" %
2181 (self
.client_prefix
, clientFile
))
2182 return clientFile
[len(self
.client_prefix
):]
2184 def update_client_spec_path_cache(self
, files
):
2185 """ Caching file paths by "p4 where" batch query """
2187 # List depot file paths exclude that already cached
2188 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2190 if len(fileArgs
) == 0:
2191 return # All files in cache
2193 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2194 for res
in where_result
:
2195 if "code" in res
and res
["code"] == "error":
2196 # assume error is "... file(s) not in client view"
2198 if "clientFile" not in res
:
2199 die("No clientFile in 'p4 where' output")
2201 # it will list all of them, but only one not unmap-ped
2203 if gitConfigBool("core.ignorecase"):
2204 res
['depotFile'] = res
['depotFile'].lower()
2205 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2207 # not found files or unmap files set to ""
2208 for depotFile
in fileArgs
:
2209 if gitConfigBool("core.ignorecase"):
2210 depotFile
= depotFile
.lower()
2211 if depotFile
not in self
.client_spec_path_cache
:
2212 self
.client_spec_path_cache
[depotFile
] = ""
2214 def map_in_client(self
, depot_path
):
2215 """Return the relative location in the client where this
2216 depot file should live. Returns "" if the file should
2217 not be mapped in the client."""
2219 if gitConfigBool("core.ignorecase"):
2220 depot_path
= depot_path
.lower()
2222 if depot_path
in self
.client_spec_path_cache
:
2223 return self
.client_spec_path_cache
[depot_path
]
2225 die( "Error: %s is not found in client spec path" % depot_path
)
2228 class P4Sync(Command
, P4UserMap
):
2229 delete_actions
= ( "delete", "move/delete", "purge" )
2232 Command
.__init
__(self
)
2233 P4UserMap
.__init
__(self
)
2235 optparse
.make_option("--branch", dest
="branch"),
2236 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2237 optparse
.make_option("--changesfile", dest
="changesFile"),
2238 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2239 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2240 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2241 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2242 help="Import into refs/heads/ , not refs/remotes"),
2243 optparse
.make_option("--max-changes", dest
="maxChanges",
2244 help="Maximum number of changes to import"),
2245 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2246 help="Internal block size to use when iteratively calling p4 changes"),
2247 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2248 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2249 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2250 help="Only sync files that are included in the Perforce Client Spec"),
2251 optparse
.make_option("-/", dest
="cloneExclude",
2252 action
="append", type="string",
2253 help="exclude depot path"),
2255 self
.description
= """Imports from Perforce into a git repository.\n
2257 //depot/my/project/ -- to import the current head
2258 //depot/my/project/@all -- to import everything
2259 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2261 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2263 self
.usage
+= " //depot/path[@revRange]"
2265 self
.createdBranches
= set()
2266 self
.committedChanges
= set()
2268 self
.detectBranches
= False
2269 self
.detectLabels
= False
2270 self
.importLabels
= False
2271 self
.changesFile
= ""
2272 self
.syncWithOrigin
= True
2273 self
.importIntoRemotes
= True
2274 self
.maxChanges
= ""
2275 self
.changes_block_size
= None
2276 self
.keepRepoPath
= False
2277 self
.depotPaths
= None
2278 self
.p4BranchesInGit
= []
2279 self
.cloneExclude
= []
2280 self
.useClientSpec
= False
2281 self
.useClientSpec_from_options
= False
2282 self
.clientSpecDirs
= None
2283 self
.tempBranches
= []
2284 self
.tempBranchLocation
= "refs/git-p4-tmp"
2285 self
.largeFileSystem
= None
2287 if gitConfig('git-p4.largeFileSystem'):
2288 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2289 self
.largeFileSystem
= largeFileSystemConstructor(
2290 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2293 if gitConfig("git-p4.syncFromOrigin") == "false":
2294 self
.syncWithOrigin
= False
2296 # This is required for the "append" cloneExclude action
2297 def ensure_value(self
, attr
, value
):
2298 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2299 setattr(self
, attr
, value
)
2300 return getattr(self
, attr
)
2302 # Force a checkpoint in fast-import and wait for it to finish
2303 def checkpoint(self
):
2304 self
.gitStream
.write("checkpoint\n\n")
2305 self
.gitStream
.write("progress checkpoint\n\n")
2306 out
= self
.gitOutput
.readline()
2308 print "checkpoint finished: " + out
2310 def extractFilesFromCommit(self
, commit
):
2311 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2312 for path
in self
.cloneExclude
]
2315 while commit
.has_key("depotFile%s" % fnum
):
2316 path
= commit
["depotFile%s" % fnum
]
2318 if [p
for p
in self
.cloneExclude
2319 if p4PathStartsWith(path
, p
)]:
2322 found
= [p
for p
in self
.depotPaths
2323 if p4PathStartsWith(path
, p
)]
2330 file["rev"] = commit
["rev%s" % fnum
]
2331 file["action"] = commit
["action%s" % fnum
]
2332 file["type"] = commit
["type%s" % fnum
]
2337 def extractJobsFromCommit(self
, commit
):
2340 while commit
.has_key("job%s" % jnum
):
2341 job
= commit
["job%s" % jnum
]
2346 def stripRepoPath(self
, path
, prefixes
):
2347 """When streaming files, this is called to map a p4 depot path
2348 to where it should go in git. The prefixes are either
2349 self.depotPaths, or self.branchPrefixes in the case of
2350 branch detection."""
2352 if self
.useClientSpec
:
2353 # branch detection moves files up a level (the branch name)
2354 # from what client spec interpretation gives
2355 path
= self
.clientSpecDirs
.map_in_client(path
)
2356 if self
.detectBranches
:
2357 for b
in self
.knownBranches
:
2358 if path
.startswith(b
+ "/"):
2359 path
= path
[len(b
)+1:]
2361 elif self
.keepRepoPath
:
2362 # Preserve everything in relative path name except leading
2363 # //depot/; just look at first prefix as they all should
2364 # be in the same depot.
2365 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2366 if p4PathStartsWith(path
, depot
):
2367 path
= path
[len(depot
):]
2371 if p4PathStartsWith(path
, p
):
2372 path
= path
[len(p
):]
2375 path
= wildcard_decode(path
)
2378 def splitFilesIntoBranches(self
, commit
):
2379 """Look at each depotFile in the commit to figure out to what
2380 branch it belongs."""
2382 if self
.clientSpecDirs
:
2383 files
= self
.extractFilesFromCommit(commit
)
2384 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2388 while commit
.has_key("depotFile%s" % fnum
):
2389 path
= commit
["depotFile%s" % fnum
]
2390 found
= [p
for p
in self
.depotPaths
2391 if p4PathStartsWith(path
, p
)]
2398 file["rev"] = commit
["rev%s" % fnum
]
2399 file["action"] = commit
["action%s" % fnum
]
2400 file["type"] = commit
["type%s" % fnum
]
2403 # start with the full relative path where this file would
2405 if self
.useClientSpec
:
2406 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2408 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2410 for branch
in self
.knownBranches
.keys():
2411 # add a trailing slash so that a commit into qt/4.2foo
2412 # doesn't end up in qt/4.2, e.g.
2413 if relPath
.startswith(branch
+ "/"):
2414 if branch
not in branches
:
2415 branches
[branch
] = []
2416 branches
[branch
].append(file)
2421 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2422 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2423 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2425 self
.gitStream
.write(d
)
2426 self
.gitStream
.write('\n')
2428 # output one file from the P4 stream
2429 # - helper for streamP4Files
2431 def streamOneP4File(self
, file, contents
):
2432 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2434 size
= int(self
.stream_file
['fileSize'])
2435 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2438 (type_base
, type_mods
) = split_p4_type(file["type"])
2441 if "x" in type_mods
:
2443 if type_base
== "symlink":
2445 # p4 print on a symlink sometimes contains "target\n";
2446 # if it does, remove the newline
2447 data
= ''.join(contents
)
2449 # Some version of p4 allowed creating a symlink that pointed
2450 # to nothing. This causes p4 errors when checking out such
2451 # a change, and errors here too. Work around it by ignoring
2452 # the bad symlink; hopefully a future change fixes it.
2453 print "\nIgnoring empty symlink in %s" % file['depotFile']
2455 elif data
[-1] == '\n':
2456 contents
= [data
[:-1]]
2460 if type_base
== "utf16":
2461 # p4 delivers different text in the python output to -G
2462 # than it does when using "print -o", or normal p4 client
2463 # operations. utf16 is converted to ascii or utf8, perhaps.
2464 # But ascii text saved as -t utf16 is completely mangled.
2465 # Invoke print -o to get the real contents.
2467 # On windows, the newlines will always be mangled by print, so put
2468 # them back too. This is not needed to the cygwin windows version,
2469 # just the native "NT" type.
2472 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2473 except Exception as e
:
2474 if 'Translation of file content failed' in str(e
):
2475 type_base
= 'binary'
2479 if p4_version_string().find('/NT') >= 0:
2480 text
= text
.replace('\r\n', '\n')
2483 if type_base
== "apple":
2484 # Apple filetype files will be streamed as a concatenation of
2485 # its appledouble header and the contents. This is useless
2486 # on both macs and non-macs. If using "print -q -o xx", it
2487 # will create "xx" with the data, and "%xx" with the header.
2488 # This is also not very useful.
2490 # Ideally, someday, this script can learn how to generate
2491 # appledouble files directly and import those to git, but
2492 # non-mac machines can never find a use for apple filetype.
2493 print "\nIgnoring apple filetype file %s" % file['depotFile']
2496 # Note that we do not try to de-mangle keywords on utf16 files,
2497 # even though in theory somebody may want that.
2498 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2500 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2501 text
= ''.join(contents
)
2502 text
= regexp
.sub(r
'$\1$', text
)
2506 relPath
.decode('ascii')
2509 if gitConfig('git-p4.pathEncoding'):
2510 encoding
= gitConfig('git-p4.pathEncoding')
2511 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2513 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2515 if self
.largeFileSystem
:
2516 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2518 self
.writeToGitStream(git_mode
, relPath
, contents
)
2520 def streamOneP4Deletion(self
, file):
2521 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2523 sys
.stdout
.write("delete %s\n" % relPath
)
2525 self
.gitStream
.write("D %s\n" % relPath
)
2527 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2528 self
.largeFileSystem
.removeLargeFile(relPath
)
2530 # handle another chunk of streaming data
2531 def streamP4FilesCb(self
, marshalled
):
2533 # catch p4 errors and complain
2535 if "code" in marshalled
:
2536 if marshalled
["code"] == "error":
2537 if "data" in marshalled
:
2538 err
= marshalled
["data"].rstrip()
2540 if not err
and 'fileSize' in self
.stream_file
:
2541 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2542 if required_bytes
> 0:
2543 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2544 os
.getcwd(), required_bytes
/1024/1024
2549 if self
.stream_have_file_info
:
2550 if "depotFile" in self
.stream_file
:
2551 f
= self
.stream_file
["depotFile"]
2552 # force a failure in fast-import, else an empty
2553 # commit will be made
2554 self
.gitStream
.write("\n")
2555 self
.gitStream
.write("die-now\n")
2556 self
.gitStream
.close()
2557 # ignore errors, but make sure it exits first
2558 self
.importProcess
.wait()
2560 die("Error from p4 print for %s: %s" % (f
, err
))
2562 die("Error from p4 print: %s" % err
)
2564 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2565 # start of a new file - output the old one first
2566 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2567 self
.stream_file
= {}
2568 self
.stream_contents
= []
2569 self
.stream_have_file_info
= False
2571 # pick up the new file information... for the
2572 # 'data' field we need to append to our array
2573 for k
in marshalled
.keys():
2575 if 'streamContentSize' not in self
.stream_file
:
2576 self
.stream_file
['streamContentSize'] = 0
2577 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2578 self
.stream_contents
.append(marshalled
['data'])
2580 self
.stream_file
[k
] = marshalled
[k
]
2583 'streamContentSize' in self
.stream_file
and
2584 'fileSize' in self
.stream_file
and
2585 'depotFile' in self
.stream_file
):
2586 size
= int(self
.stream_file
["fileSize"])
2588 progress
= 100*self
.stream_file
['streamContentSize']/size
2589 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2592 self
.stream_have_file_info
= True
2594 # Stream directly from "p4 files" into "git fast-import"
2595 def streamP4Files(self
, files
):
2601 filesForCommit
.append(f
)
2602 if f
['action'] in self
.delete_actions
:
2603 filesToDelete
.append(f
)
2605 filesToRead
.append(f
)
2608 for f
in filesToDelete
:
2609 self
.streamOneP4Deletion(f
)
2611 if len(filesToRead
) > 0:
2612 self
.stream_file
= {}
2613 self
.stream_contents
= []
2614 self
.stream_have_file_info
= False
2616 # curry self argument
2617 def streamP4FilesCbSelf(entry
):
2618 self
.streamP4FilesCb(entry
)
2620 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2622 p4CmdList(["-x", "-", "print"],
2624 cb
=streamP4FilesCbSelf
)
2627 if self
.stream_file
.has_key('depotFile'):
2628 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2630 def make_email(self
, userid
):
2631 if userid
in self
.users
:
2632 return self
.users
[userid
]
2634 return "%s <a@b>" % userid
2636 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2637 """ Stream a p4 tag.
2638 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2642 print "writing tag %s for commit %s" % (labelName
, commit
)
2643 gitStream
.write("tag %s\n" % labelName
)
2644 gitStream
.write("from %s\n" % commit
)
2646 if labelDetails
.has_key('Owner'):
2647 owner
= labelDetails
["Owner"]
2651 # Try to use the owner of the p4 label, or failing that,
2652 # the current p4 user id.
2654 email
= self
.make_email(owner
)
2656 email
= self
.make_email(self
.p4UserId())
2657 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2659 gitStream
.write("tagger %s\n" % tagger
)
2661 print "labelDetails=",labelDetails
2662 if labelDetails
.has_key('Description'):
2663 description
= labelDetails
['Description']
2665 description
= 'Label from git p4'
2667 gitStream
.write("data %d\n" % len(description
))
2668 gitStream
.write(description
)
2669 gitStream
.write("\n")
2671 def inClientSpec(self
, path
):
2672 if not self
.clientSpecDirs
:
2674 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2675 if not inClientSpec
and self
.verbose
:
2676 print('Ignoring file outside of client spec: {0}'.format(path
))
2679 def hasBranchPrefix(self
, path
):
2680 if not self
.branchPrefixes
:
2682 hasPrefix
= [p
for p
in self
.branchPrefixes
2683 if p4PathStartsWith(path
, p
)]
2684 if not hasPrefix
and self
.verbose
:
2685 print('Ignoring file outside of prefix: {0}'.format(path
))
2688 def commit(self
, details
, files
, branch
, parent
= ""):
2689 epoch
= details
["time"]
2690 author
= details
["user"]
2691 jobs
= self
.extractJobsFromCommit(details
)
2694 print('commit into {0}'.format(branch
))
2696 if self
.clientSpecDirs
:
2697 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2699 files
= [f
for f
in files
2700 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
2702 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
2703 print('Ignoring revision {0} as it would produce an empty commit.'
2704 .format(details
['change']))
2707 self
.gitStream
.write("commit %s\n" % branch
)
2708 self
.gitStream
.write("mark :%s\n" % details
["change"])
2709 self
.committedChanges
.add(int(details
["change"]))
2711 if author
not in self
.users
:
2712 self
.getUserMapFromPerforceServer()
2713 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2715 self
.gitStream
.write("committer %s\n" % committer
)
2717 self
.gitStream
.write("data <<EOT\n")
2718 self
.gitStream
.write(details
["desc"])
2720 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
2721 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2722 (','.join(self
.branchPrefixes
), details
["change"]))
2723 if len(details
['options']) > 0:
2724 self
.gitStream
.write(": options = %s" % details
['options'])
2725 self
.gitStream
.write("]\nEOT\n\n")
2729 print "parent %s" % parent
2730 self
.gitStream
.write("from %s\n" % parent
)
2732 self
.streamP4Files(files
)
2733 self
.gitStream
.write("\n")
2735 change
= int(details
["change"])
2737 if self
.labels
.has_key(change
):
2738 label
= self
.labels
[change
]
2739 labelDetails
= label
[0]
2740 labelRevisions
= label
[1]
2742 print "Change %s is labelled %s" % (change
, labelDetails
)
2744 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2745 for p
in self
.branchPrefixes
])
2747 if len(files
) == len(labelRevisions
):
2751 if info
["action"] in self
.delete_actions
:
2753 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2755 if cleanedFiles
== labelRevisions
:
2756 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2760 print ("Tag %s does not match with change %s: files do not match."
2761 % (labelDetails
["label"], change
))
2765 print ("Tag %s does not match with change %s: file count is different."
2766 % (labelDetails
["label"], change
))
2768 # Build a dictionary of changelists and labels, for "detect-labels" option.
2769 def getLabels(self
):
2772 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2773 if len(l
) > 0 and not self
.silent
:
2774 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2777 label
= output
["label"]
2781 print "Querying files for label %s" % label
2782 for file in p4CmdList(["files"] +
2783 ["%s...@%s" % (p
, label
)
2784 for p
in self
.depotPaths
]):
2785 revisions
[file["depotFile"]] = file["rev"]
2786 change
= int(file["change"])
2787 if change
> newestChange
:
2788 newestChange
= change
2790 self
.labels
[newestChange
] = [output
, revisions
]
2793 print "Label changes: %s" % self
.labels
.keys()
2795 # Import p4 labels as git tags. A direct mapping does not
2796 # exist, so assume that if all the files are at the same revision
2797 # then we can use that, or it's something more complicated we should
2799 def importP4Labels(self
, stream
, p4Labels
):
2801 print "import p4 labels: " + ' '.join(p4Labels
)
2803 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2804 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2805 if len(validLabelRegexp
) == 0:
2806 validLabelRegexp
= defaultLabelRegexp
2807 m
= re
.compile(validLabelRegexp
)
2809 for name
in p4Labels
:
2812 if not m
.match(name
):
2814 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2817 if name
in ignoredP4Labels
:
2820 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2822 # get the most recent changelist for each file in this label
2823 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2824 for p
in self
.depotPaths
])
2826 if change
.has_key('change'):
2827 # find the corresponding git commit; take the oldest commit
2828 changelist
= int(change
['change'])
2829 if changelist
in self
.committedChanges
:
2830 gitCommit
= ":%d" % changelist
# use a fast-import mark
2833 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2834 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2835 if len(gitCommit
) == 0:
2836 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2839 gitCommit
= gitCommit
.strip()
2842 # Convert from p4 time format
2844 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2846 print "Could not convert label time %s" % labelDetails
['Update']
2849 when
= int(time
.mktime(tmwhen
))
2850 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2852 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2855 print "Label %s has no changelists - possibly deleted?" % name
2858 # We can't import this label; don't try again as it will get very
2859 # expensive repeatedly fetching all the files for labels that will
2860 # never be imported. If the label is moved in the future, the
2861 # ignore will need to be removed manually.
2862 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2864 def guessProjectName(self
):
2865 for p
in self
.depotPaths
:
2868 p
= p
[p
.strip().rfind("/") + 1:]
2869 if not p
.endswith("/"):
2873 def getBranchMapping(self
):
2874 lostAndFoundBranches
= set()
2876 user
= gitConfig("git-p4.branchUser")
2878 command
= "branches -u %s" % user
2880 command
= "branches"
2882 for info
in p4CmdList(command
):
2883 details
= p4Cmd(["branch", "-o", info
["branch"]])
2885 while details
.has_key("View%s" % viewIdx
):
2886 paths
= details
["View%s" % viewIdx
].split(" ")
2887 viewIdx
= viewIdx
+ 1
2888 # require standard //depot/foo/... //depot/bar/... mapping
2889 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2892 destination
= paths
[1]
2894 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2895 source
= source
[len(self
.depotPaths
[0]):-4]
2896 destination
= destination
[len(self
.depotPaths
[0]):-4]
2898 if destination
in self
.knownBranches
:
2900 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2901 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2904 self
.knownBranches
[destination
] = source
2906 lostAndFoundBranches
.discard(destination
)
2908 if source
not in self
.knownBranches
:
2909 lostAndFoundBranches
.add(source
)
2911 # Perforce does not strictly require branches to be defined, so we also
2912 # check git config for a branch list.
2914 # Example of branch definition in git config file:
2916 # branchList=main:branchA
2917 # branchList=main:branchB
2918 # branchList=branchA:branchC
2919 configBranches
= gitConfigList("git-p4.branchList")
2920 for branch
in configBranches
:
2922 (source
, destination
) = branch
.split(":")
2923 self
.knownBranches
[destination
] = source
2925 lostAndFoundBranches
.discard(destination
)
2927 if source
not in self
.knownBranches
:
2928 lostAndFoundBranches
.add(source
)
2931 for branch
in lostAndFoundBranches
:
2932 self
.knownBranches
[branch
] = branch
2934 def getBranchMappingFromGitBranches(self
):
2935 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2936 for branch
in branches
.keys():
2937 if branch
== "master":
2940 branch
= branch
[len(self
.projectName
):]
2941 self
.knownBranches
[branch
] = branch
2943 def updateOptionDict(self
, d
):
2945 if self
.keepRepoPath
:
2946 option_keys
['keepRepoPath'] = 1
2948 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2950 def readOptions(self
, d
):
2951 self
.keepRepoPath
= (d
.has_key('options')
2952 and ('keepRepoPath' in d
['options']))
2954 def gitRefForBranch(self
, branch
):
2955 if branch
== "main":
2956 return self
.refPrefix
+ "master"
2958 if len(branch
) <= 0:
2961 return self
.refPrefix
+ self
.projectName
+ branch
2963 def gitCommitByP4Change(self
, ref
, change
):
2965 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2968 latestCommit
= parseRevision(ref
)
2972 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2973 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2978 log
= extractLogMessageFromGitCommit(next
)
2979 settings
= extractSettingsGitLog(log
)
2980 currentChange
= int(settings
['change'])
2982 print "current change %s" % currentChange
2984 if currentChange
== change
:
2986 print "found %s" % next
2989 if currentChange
< change
:
2990 earliestCommit
= "^%s" % next
2992 latestCommit
= "%s" % next
2996 def importNewBranch(self
, branch
, maxChange
):
2997 # make fast-import flush all changes to disk and update the refs using the checkpoint
2998 # command so that we can try to find the branch parent in the git history
2999 self
.gitStream
.write("checkpoint\n\n");
3000 self
.gitStream
.flush();
3001 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3002 range = "@1,%s" % maxChange
3003 #print "prefix" + branchPrefix
3004 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3005 if len(changes
) <= 0:
3007 firstChange
= changes
[0]
3008 #print "first change in branch: %s" % firstChange
3009 sourceBranch
= self
.knownBranches
[branch
]
3010 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3011 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3012 #print "source " + sourceBranch
3014 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3015 #print "branch parent: %s" % branchParentChange
3016 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3017 if len(gitParent
) > 0:
3018 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3019 #print "parent git commit: %s" % gitParent
3021 self
.importChanges(changes
)
3024 def searchParent(self
, parent
, branch
, target
):
3026 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3027 "--no-merges", parent
]):
3029 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3032 print "Found parent of %s in commit %s" % (branch
, blob
)
3039 def importChanges(self
, changes
):
3041 for change
in changes
:
3042 description
= p4_describe(change
)
3043 self
.updateOptionDict(description
)
3046 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3051 if self
.detectBranches
:
3052 branches
= self
.splitFilesIntoBranches(description
)
3053 for branch
in branches
.keys():
3055 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3056 self
.branchPrefixes
= [ branchPrefix
]
3060 filesForCommit
= branches
[branch
]
3063 print "branch is %s" % branch
3065 self
.updatedBranches
.add(branch
)
3067 if branch
not in self
.createdBranches
:
3068 self
.createdBranches
.add(branch
)
3069 parent
= self
.knownBranches
[branch
]
3070 if parent
== branch
:
3073 fullBranch
= self
.projectName
+ branch
3074 if fullBranch
not in self
.p4BranchesInGit
:
3076 print("\n Importing new branch %s" % fullBranch
);
3077 if self
.importNewBranch(branch
, change
- 1):
3079 self
.p4BranchesInGit
.append(fullBranch
)
3081 print("\n Resuming with change %s" % change
);
3084 print "parent determined through known branches: %s" % parent
3086 branch
= self
.gitRefForBranch(branch
)
3087 parent
= self
.gitRefForBranch(parent
)
3090 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3092 if len(parent
) == 0 and branch
in self
.initialParents
:
3093 parent
= self
.initialParents
[branch
]
3094 del self
.initialParents
[branch
]
3098 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3100 print "Creating temporary branch: " + tempBranch
3101 self
.commit(description
, filesForCommit
, tempBranch
)
3102 self
.tempBranches
.append(tempBranch
)
3104 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3106 self
.commit(description
, filesForCommit
, branch
, blob
)
3109 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3110 self
.commit(description
, filesForCommit
, branch
, parent
)
3112 files
= self
.extractFilesFromCommit(description
)
3113 self
.commit(description
, files
, self
.branch
,
3115 # only needed once, to connect to the previous commit
3116 self
.initialParent
= ""
3118 print self
.gitError
.read()
3121 def importHeadRevision(self
, revision
):
3122 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3125 details
["user"] = "git perforce import user"
3126 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3127 % (' '.join(self
.depotPaths
), revision
))
3128 details
["change"] = revision
3132 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3134 for info
in p4CmdList(["files"] + fileArgs
):
3136 if 'code' in info
and info
['code'] == 'error':
3137 sys
.stderr
.write("p4 returned an error: %s\n"
3139 if info
['data'].find("must refer to client") >= 0:
3140 sys
.stderr
.write("This particular p4 error is misleading.\n")
3141 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3142 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3144 if 'p4ExitCode' in info
:
3145 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3149 change
= int(info
["change"])
3150 if change
> newestRevision
:
3151 newestRevision
= change
3153 if info
["action"] in self
.delete_actions
:
3154 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3155 #fileCnt = fileCnt + 1
3158 for prop
in ["depotFile", "rev", "action", "type" ]:
3159 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3161 fileCnt
= fileCnt
+ 1
3163 details
["change"] = newestRevision
3165 # Use time from top-most change so that all git p4 clones of
3166 # the same p4 repo have the same commit SHA1s.
3167 res
= p4_describe(newestRevision
)
3168 details
["time"] = res
["time"]
3170 self
.updateOptionDict(details
)
3172 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3174 print "IO error with git fast-import. Is your git version recent enough?"
3175 print self
.gitError
.read()
3178 def run(self
, args
):
3179 self
.depotPaths
= []
3180 self
.changeRange
= ""
3181 self
.previousDepotPaths
= []
3182 self
.hasOrigin
= False
3184 # map from branch depot path to parent branch
3185 self
.knownBranches
= {}
3186 self
.initialParents
= {}
3188 if self
.importIntoRemotes
:
3189 self
.refPrefix
= "refs/remotes/p4/"
3191 self
.refPrefix
= "refs/heads/p4/"
3193 if self
.syncWithOrigin
:
3194 self
.hasOrigin
= originP4BranchesExist()
3197 print 'Syncing with origin first, using "git fetch origin"'
3198 system("git fetch origin")
3200 branch_arg_given
= bool(self
.branch
)
3201 if len(self
.branch
) == 0:
3202 self
.branch
= self
.refPrefix
+ "master"
3203 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3204 system("git update-ref %s refs/heads/p4" % self
.branch
)
3205 system("git branch -D p4")
3207 # accept either the command-line option, or the configuration variable
3208 if self
.useClientSpec
:
3209 # will use this after clone to set the variable
3210 self
.useClientSpec_from_options
= True
3212 if gitConfigBool("git-p4.useclientspec"):
3213 self
.useClientSpec
= True
3214 if self
.useClientSpec
:
3215 self
.clientSpecDirs
= getClientSpec()
3217 # TODO: should always look at previous commits,
3218 # merge with previous imports, if possible.
3221 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3223 # branches holds mapping from branch name to sha1
3224 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3226 # restrict to just this one, disabling detect-branches
3227 if branch_arg_given
:
3228 short
= self
.branch
.split("/")[-1]
3229 if short
in branches
:
3230 self
.p4BranchesInGit
= [ short
]
3232 self
.p4BranchesInGit
= branches
.keys()
3234 if len(self
.p4BranchesInGit
) > 1:
3236 print "Importing from/into multiple branches"
3237 self
.detectBranches
= True
3238 for branch
in branches
.keys():
3239 self
.initialParents
[self
.refPrefix
+ branch
] = \
3243 print "branches: %s" % self
.p4BranchesInGit
3246 for branch
in self
.p4BranchesInGit
:
3247 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3249 settings
= extractSettingsGitLog(logMsg
)
3251 self
.readOptions(settings
)
3252 if (settings
.has_key('depot-paths')
3253 and settings
.has_key ('change')):
3254 change
= int(settings
['change']) + 1
3255 p4Change
= max(p4Change
, change
)
3257 depotPaths
= sorted(settings
['depot-paths'])
3258 if self
.previousDepotPaths
== []:
3259 self
.previousDepotPaths
= depotPaths
3262 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3263 prev_list
= prev
.split("/")
3264 cur_list
= cur
.split("/")
3265 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3266 if cur_list
[i
] <> prev_list
[i
]:
3270 paths
.append ("/".join(cur_list
[:i
+ 1]))
3272 self
.previousDepotPaths
= paths
3275 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3276 self
.changeRange
= "@%s,#head" % p4Change
3277 if not self
.silent
and not self
.detectBranches
:
3278 print "Performing incremental import into %s git branch" % self
.branch
3280 # accept multiple ref name abbreviations:
3281 # refs/foo/bar/branch -> use it exactly
3282 # p4/branch -> prepend refs/remotes/ or refs/heads/
3283 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3284 if not self
.branch
.startswith("refs/"):
3285 if self
.importIntoRemotes
:
3286 prepend
= "refs/remotes/"
3288 prepend
= "refs/heads/"
3289 if not self
.branch
.startswith("p4/"):
3291 self
.branch
= prepend
+ self
.branch
3293 if len(args
) == 0 and self
.depotPaths
:
3295 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3297 if self
.depotPaths
and self
.depotPaths
!= args
:
3298 print ("previous import used depot path %s and now %s was specified. "
3299 "This doesn't work!" % (' '.join (self
.depotPaths
),
3303 self
.depotPaths
= sorted(args
)
3308 # Make sure no revision specifiers are used when --changesfile
3310 bad_changesfile
= False
3311 if len(self
.changesFile
) > 0:
3312 for p
in self
.depotPaths
:
3313 if p
.find("@") >= 0 or p
.find("#") >= 0:
3314 bad_changesfile
= True
3317 die("Option --changesfile is incompatible with revision specifiers")
3320 for p
in self
.depotPaths
:
3321 if p
.find("@") != -1:
3322 atIdx
= p
.index("@")
3323 self
.changeRange
= p
[atIdx
:]
3324 if self
.changeRange
== "@all":
3325 self
.changeRange
= ""
3326 elif ',' not in self
.changeRange
:
3327 revision
= self
.changeRange
3328 self
.changeRange
= ""
3330 elif p
.find("#") != -1:
3331 hashIdx
= p
.index("#")
3332 revision
= p
[hashIdx
:]
3334 elif self
.previousDepotPaths
== []:
3335 # pay attention to changesfile, if given, else import
3336 # the entire p4 tree at the head revision
3337 if len(self
.changesFile
) == 0:
3340 p
= re
.sub ("\.\.\.$", "", p
)
3341 if not p
.endswith("/"):
3346 self
.depotPaths
= newPaths
3348 # --detect-branches may change this for each branch
3349 self
.branchPrefixes
= self
.depotPaths
3351 self
.loadUserMapFromCache()
3353 if self
.detectLabels
:
3356 if self
.detectBranches
:
3357 ## FIXME - what's a P4 projectName ?
3358 self
.projectName
= self
.guessProjectName()
3361 self
.getBranchMappingFromGitBranches()
3363 self
.getBranchMapping()
3365 print "p4-git branches: %s" % self
.p4BranchesInGit
3366 print "initial parents: %s" % self
.initialParents
3367 for b
in self
.p4BranchesInGit
:
3371 b
= b
[len(self
.projectName
):]
3372 self
.createdBranches
.add(b
)
3374 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3376 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3377 stdin
=subprocess
.PIPE
,
3378 stdout
=subprocess
.PIPE
,
3379 stderr
=subprocess
.PIPE
);
3380 self
.gitOutput
= self
.importProcess
.stdout
3381 self
.gitStream
= self
.importProcess
.stdin
3382 self
.gitError
= self
.importProcess
.stderr
3385 self
.importHeadRevision(revision
)
3389 if len(self
.changesFile
) > 0:
3390 output
= open(self
.changesFile
).readlines()
3393 changeSet
.add(int(line
))
3395 for change
in changeSet
:
3396 changes
.append(change
)
3400 # catch "git p4 sync" with no new branches, in a repo that
3401 # does not have any existing p4 branches
3403 if not self
.p4BranchesInGit
:
3404 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3406 # The default branch is master, unless --branch is used to
3407 # specify something else. Make sure it exists, or complain
3408 # nicely about how to use --branch.
3409 if not self
.detectBranches
:
3410 if not branch_exists(self
.branch
):
3411 if branch_arg_given
:
3412 die("Error: branch %s does not exist." % self
.branch
)
3414 die("Error: no branch %s; perhaps specify one with --branch." %
3418 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3420 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3422 if len(self
.maxChanges
) > 0:
3423 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3425 if len(changes
) == 0:
3427 print "No changes to import!"
3429 if not self
.silent
and not self
.detectBranches
:
3430 print "Import destination: %s" % self
.branch
3432 self
.updatedBranches
= set()
3434 if not self
.detectBranches
:
3436 # start a new branch
3437 self
.initialParent
= ""
3439 # build on a previous revision
3440 self
.initialParent
= parseRevision(self
.branch
)
3442 self
.importChanges(changes
)
3446 if len(self
.updatedBranches
) > 0:
3447 sys
.stdout
.write("Updated branches: ")
3448 for b
in self
.updatedBranches
:
3449 sys
.stdout
.write("%s " % b
)
3450 sys
.stdout
.write("\n")
3452 if gitConfigBool("git-p4.importLabels"):
3453 self
.importLabels
= True
3455 if self
.importLabels
:
3456 p4Labels
= getP4Labels(self
.depotPaths
)
3457 gitTags
= getGitTags()
3459 missingP4Labels
= p4Labels
- gitTags
3460 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3462 self
.gitStream
.close()
3463 if self
.importProcess
.wait() != 0:
3464 die("fast-import failed: %s" % self
.gitError
.read())
3465 self
.gitOutput
.close()
3466 self
.gitError
.close()
3468 # Cleanup temporary branches created during import
3469 if self
.tempBranches
!= []:
3470 for branch
in self
.tempBranches
:
3471 read_pipe("git update-ref -d %s" % branch
)
3472 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3474 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3475 # a convenient shortcut refname "p4".
3476 if self
.importIntoRemotes
:
3477 head_ref
= self
.refPrefix
+ "HEAD"
3478 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3479 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3483 class P4Rebase(Command
):
3485 Command
.__init
__(self
)
3487 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3489 self
.importLabels
= False
3490 self
.description
= ("Fetches the latest revision from perforce and "
3491 + "rebases the current work (branch) against it")
3493 def run(self
, args
):
3495 sync
.importLabels
= self
.importLabels
3498 return self
.rebase()
3501 if os
.system("git update-index --refresh") != 0:
3502 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3503 if len(read_pipe("git diff-index HEAD --")) > 0:
3504 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3506 [upstream
, settings
] = findUpstreamBranchPoint()
3507 if len(upstream
) == 0:
3508 die("Cannot find upstream branchpoint for rebase")
3510 # the branchpoint may be p4/foo~3, so strip off the parent
3511 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3513 print "Rebasing the current branch onto %s" % upstream
3514 oldHead
= read_pipe("git rev-parse HEAD").strip()
3515 system("git rebase %s" % upstream
)
3516 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3519 class P4Clone(P4Sync
):
3521 P4Sync
.__init
__(self
)
3522 self
.description
= "Creates a new git repository and imports from Perforce into it"
3523 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3525 optparse
.make_option("--destination", dest
="cloneDestination",
3526 action
='store', default
=None,
3527 help="where to leave result of the clone"),
3528 optparse
.make_option("--bare", dest
="cloneBare",
3529 action
="store_true", default
=False),
3531 self
.cloneDestination
= None
3532 self
.needsGit
= False
3533 self
.cloneBare
= False
3535 def defaultDestination(self
, args
):
3536 ## TODO: use common prefix of args?
3538 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3539 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3540 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3541 depotDir
= re
.sub(r
"/$", "", depotDir
)
3542 return os
.path
.split(depotDir
)[1]
3544 def run(self
, args
):
3548 if self
.keepRepoPath
and not self
.cloneDestination
:
3549 sys
.stderr
.write("Must specify destination for --keep-path\n")
3554 if not self
.cloneDestination
and len(depotPaths
) > 1:
3555 self
.cloneDestination
= depotPaths
[-1]
3556 depotPaths
= depotPaths
[:-1]
3558 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3559 for p
in depotPaths
:
3560 if not p
.startswith("//"):
3561 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3564 if not self
.cloneDestination
:
3565 self
.cloneDestination
= self
.defaultDestination(args
)
3567 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3569 if not os
.path
.exists(self
.cloneDestination
):
3570 os
.makedirs(self
.cloneDestination
)
3571 chdir(self
.cloneDestination
)
3573 init_cmd
= [ "git", "init" ]
3575 init_cmd
.append("--bare")
3576 retcode
= subprocess
.call(init_cmd
)
3578 raise CalledProcessError(retcode
, init_cmd
)
3580 if not P4Sync
.run(self
, depotPaths
):
3583 # create a master branch and check out a work tree
3584 if gitBranchExists(self
.branch
):
3585 system([ "git", "branch", "master", self
.branch
])
3586 if not self
.cloneBare
:
3587 system([ "git", "checkout", "-f" ])
3589 print 'Not checking out any branch, use ' \
3590 '"git checkout -q -b master <branch>"'
3592 # auto-set this variable if invoked with --use-client-spec
3593 if self
.useClientSpec_from_options
:
3594 system("git config --bool git-p4.useclientspec true")
3598 class P4Branches(Command
):
3600 Command
.__init
__(self
)
3602 self
.description
= ("Shows the git branches that hold imports and their "
3603 + "corresponding perforce depot paths")
3604 self
.verbose
= False
3606 def run(self
, args
):
3607 if originP4BranchesExist():
3608 createOrUpdateBranchesFromOrigin()
3610 cmdline
= "git rev-parse --symbolic "
3611 cmdline
+= " --remotes"
3613 for line
in read_pipe_lines(cmdline
):
3616 if not line
.startswith('p4/') or line
== "p4/HEAD":
3620 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3621 settings
= extractSettingsGitLog(log
)
3623 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3626 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3628 optparse
.IndentedHelpFormatter
.__init
__(self
)
3630 def format_description(self
, description
):
3632 return description
+ "\n"
3636 def printUsage(commands
):
3637 print "usage: %s <command> [options]" % sys
.argv
[0]
3639 print "valid commands: %s" % ", ".join(commands
)
3641 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3646 "submit" : P4Submit
,
3647 "commit" : P4Submit
,
3649 "rebase" : P4Rebase
,
3651 "rollback" : P4RollBack
,
3652 "branches" : P4Branches
3657 if len(sys
.argv
[1:]) == 0:
3658 printUsage(commands
.keys())
3661 cmdName
= sys
.argv
[1]
3663 klass
= commands
[cmdName
]
3666 print "unknown command %s" % cmdName
3668 printUsage(commands
.keys())
3671 options
= cmd
.options
3672 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3676 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3678 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3680 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3682 description
= cmd
.description
,
3683 formatter
= HelpFormatter())
3685 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3687 verbose
= cmd
.verbose
3689 if cmd
.gitdir
== None:
3690 cmd
.gitdir
= os
.path
.abspath(".git")
3691 if not isValidGitDir(cmd
.gitdir
):
3692 # "rev-parse --git-dir" without arguments will try $PWD/.git
3693 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3694 if os
.path
.exists(cmd
.gitdir
):
3695 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3699 if not isValidGitDir(cmd
.gitdir
):
3700 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3701 cmd
.gitdir
+= "/.git"
3703 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3705 # so git commands invoked from the P4 workspace will succeed
3706 os
.environ
["GIT_DIR"] = cmd
.gitdir
3708 if not cmd
.run(args
):
3713 if __name__
== '__main__':