3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
30 from subprocess
import CalledProcessError
32 # from python2.7:subprocess.py
33 # Exception classes used by this module.
34 class CalledProcessError(Exception):
35 """This exception is raised when a process run by check_call() returns
36 a non-zero exit status. The exit status will be stored in the
37 returncode attribute."""
38 def __init__(self
, returncode
, cmd
):
39 self
.returncode
= returncode
42 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
46 # Only labels/tags matching this will be imported/exported
47 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
49 # Grab changes in blocks of this many revisions, unless otherwise requested
50 defaultBlockSize
= 512
52 def p4_build_cmd(cmd
):
53 """Build a suitable p4 command line.
55 This consolidates building and returning a p4 command line into one
56 location. It means that hooking into the environment, or other configuration
57 can be done more easily.
61 user
= gitConfig("git-p4.user")
63 real_cmd
+= ["-u",user
]
65 password
= gitConfig("git-p4.password")
67 real_cmd
+= ["-P", password
]
69 port
= gitConfig("git-p4.port")
71 real_cmd
+= ["-p", port
]
73 host
= gitConfig("git-p4.host")
75 real_cmd
+= ["-H", host
]
77 client
= gitConfig("git-p4.client")
79 real_cmd
+= ["-c", client
]
82 if isinstance(cmd
,basestring
):
83 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
88 def chdir(path
, is_client_path
=False):
89 """Do chdir to the given path, and set the PWD environment
90 variable for use by P4. It does not look at getcwd() output.
91 Since we're not using the shell, it is necessary to set the
92 PWD environment variable explicitly.
94 Normally, expand the path to force it to be absolute. This
95 addresses the use of relative path names inside P4 settings,
96 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
97 as given; it looks for .p4config using PWD.
99 If is_client_path, the path was handed to us directly by p4,
100 and may be a symbolic link. Do not call os.getcwd() in this
101 case, because it will cause p4 to think that PWD is not inside
106 if not is_client_path
:
108 os
.environ
['PWD'] = path
111 """Return free space in bytes on the disk of the given dirname."""
112 if platform
.system() == 'Windows':
113 free_bytes
= ctypes
.c_ulonglong(0)
114 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
115 return free_bytes
.value
117 st
= os
.statvfs(os
.getcwd())
118 return st
.f_bavail
* st
.f_frsize
124 sys
.stderr
.write(msg
+ "\n")
127 def write_pipe(c
, stdin
):
129 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
131 expand
= isinstance(c
,basestring
)
132 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
134 val
= pipe
.write(stdin
)
137 die('Command failed: %s' % str(c
))
141 def p4_write_pipe(c
, stdin
):
142 real_cmd
= p4_build_cmd(c
)
143 return write_pipe(real_cmd
, stdin
)
145 def read_pipe(c
, ignore_error
=False):
147 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
149 expand
= isinstance(c
,basestring
)
150 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
, shell
=expand
)
151 (out
, err
) = p
.communicate()
152 if p
.returncode
!= 0 and not ignore_error
:
153 die('Command failed: %s\nError: %s' % (str(c
), err
))
156 def p4_read_pipe(c
, ignore_error
=False):
157 real_cmd
= p4_build_cmd(c
)
158 return read_pipe(real_cmd
, ignore_error
)
160 def read_pipe_lines(c
):
162 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
164 expand
= isinstance(c
, basestring
)
165 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
167 val
= pipe
.readlines()
168 if pipe
.close() or p
.wait():
169 die('Command failed: %s' % str(c
))
173 def p4_read_pipe_lines(c
):
174 """Specifically invoke p4 on the command supplied. """
175 real_cmd
= p4_build_cmd(c
)
176 return read_pipe_lines(real_cmd
)
178 def p4_has_command(cmd
):
179 """Ask p4 for help on this command. If it returns an error, the
180 command does not exist in this version of p4."""
181 real_cmd
= p4_build_cmd(["help", cmd
])
182 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
183 stderr
=subprocess
.PIPE
)
185 return p
.returncode
== 0
187 def p4_has_move_command():
188 """See if the move command exists, that it supports -k, and that
189 it has not been administratively disabled. The arguments
190 must be correct, but the filenames do not have to exist. Use
191 ones with wildcards so even if they exist, it will fail."""
193 if not p4_has_command("move"):
195 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
196 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
197 (out
, err
) = p
.communicate()
198 # return code will be 1 in either case
199 if err
.find("Invalid option") >= 0:
201 if err
.find("disabled") >= 0:
203 # assume it failed because @... was invalid changelist
206 def system(cmd
, ignore_error
=False):
207 expand
= isinstance(cmd
,basestring
)
209 sys
.stderr
.write("executing %s\n" % str(cmd
))
210 retcode
= subprocess
.call(cmd
, shell
=expand
)
211 if retcode
and not ignore_error
:
212 raise CalledProcessError(retcode
, cmd
)
217 """Specifically invoke p4 as the system command. """
218 real_cmd
= p4_build_cmd(cmd
)
219 expand
= isinstance(real_cmd
, basestring
)
220 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
222 raise CalledProcessError(retcode
, real_cmd
)
224 _p4_version_string
= None
225 def p4_version_string():
226 """Read the version string, showing just the last line, which
227 hopefully is the interesting version bit.
230 Perforce - The Fast Software Configuration Management System.
231 Copyright 1995-2011 Perforce Software. All rights reserved.
232 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
234 global _p4_version_string
235 if not _p4_version_string
:
236 a
= p4_read_pipe_lines(["-V"])
237 _p4_version_string
= a
[-1].rstrip()
238 return _p4_version_string
240 def p4_integrate(src
, dest
):
241 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
243 def p4_sync(f
, *options
):
244 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
247 # forcibly add file names with wildcards
248 if wildcard_present(f
):
249 p4_system(["add", "-f", f
])
251 p4_system(["add", f
])
254 p4_system(["delete", wildcard_encode(f
)])
256 def p4_edit(f
, *options
):
257 p4_system(["edit"] + list(options
) + [wildcard_encode(f
)])
260 p4_system(["revert", wildcard_encode(f
)])
262 def p4_reopen(type, f
):
263 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
265 def p4_move(src
, dest
):
266 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
268 def p4_last_change():
269 results
= p4CmdList(["changes", "-m", "1"])
270 return int(results
[0]['change'])
272 def p4_describe(change
):
273 """Make sure it returns a valid result by checking for
274 the presence of field "time". Return a dict of the
277 ds
= p4CmdList(["describe", "-s", str(change
)])
279 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
283 if "p4ExitCode" in d
:
284 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
287 if d
["code"] == "error":
288 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
291 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
296 # Canonicalize the p4 type and return a tuple of the
297 # base type, plus any modifiers. See "p4 help filetypes"
298 # for a list and explanation.
300 def split_p4_type(p4type
):
302 p4_filetypes_historical
= {
303 "ctempobj": "binary+Sw",
309 "tempobj": "binary+FSw",
310 "ubinary": "binary+F",
311 "uresource": "resource+F",
312 "uxbinary": "binary+Fx",
313 "xbinary": "binary+x",
315 "xtempobj": "binary+Swx",
317 "xunicode": "unicode+x",
320 if p4type
in p4_filetypes_historical
:
321 p4type
= p4_filetypes_historical
[p4type
]
323 s
= p4type
.split("+")
331 # return the raw p4 type of a file (text, text+ko, etc)
334 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
335 return results
[0]['headType']
338 # Given a type base and modifier, return a regexp matching
339 # the keywords that can be expanded in the file
341 def p4_keywords_regexp_for_type(base
, type_mods
):
342 if base
in ("text", "unicode", "binary"):
344 if "ko" in type_mods
:
346 elif "k" in type_mods
:
347 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
351 \$ # Starts with a dollar, followed by...
352 (%s) # one of the keywords, followed by...
353 (:[^$\n]+)? # possibly an old expansion, followed by...
361 # Given a file, return a regexp matching the possible
362 # RCS keywords that will be expanded, or None for files
363 # with kw expansion turned off.
365 def p4_keywords_regexp_for_file(file):
366 if not os
.path
.exists(file):
369 (type_base
, type_mods
) = split_p4_type(p4_type(file))
370 return p4_keywords_regexp_for_type(type_base
, type_mods
)
372 def setP4ExecBit(file, mode
):
373 # Reopens an already open file and changes the execute bit to match
374 # the execute bit setting in the passed in mode.
378 if not isModeExec(mode
):
379 p4Type
= getP4OpenedType(file)
380 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
381 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
382 if p4Type
[-1] == "+":
383 p4Type
= p4Type
[0:-1]
385 p4_reopen(p4Type
, file)
387 def getP4OpenedType(file):
388 # Returns the perforce file type for the given file.
390 result
= p4_read_pipe(["opened", wildcard_encode(file)])
391 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
393 return match
.group(1)
395 die("Could not determine file type for %s (result: '%s')" % (file, result
))
397 # Return the set of all p4 labels
398 def getP4Labels(depotPaths
):
400 if isinstance(depotPaths
,basestring
):
401 depotPaths
= [depotPaths
]
403 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
409 # Return the set of all git tags
412 for line
in read_pipe_lines(["git", "tag"]):
417 def diffTreePattern():
418 # This is a simple generator for the diff tree regex pattern. This could be
419 # a class variable if this and parseDiffTreeEntry were a part of a class.
420 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
424 def parseDiffTreeEntry(entry
):
425 """Parses a single diff tree entry into its component elements.
427 See git-diff-tree(1) manpage for details about the format of the diff
428 output. This method returns a dictionary with the following elements:
430 src_mode - The mode of the source file
431 dst_mode - The mode of the destination file
432 src_sha1 - The sha1 for the source file
433 dst_sha1 - The sha1 fr the destination file
434 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
435 status_score - The score for the status (applicable for 'C' and 'R'
436 statuses). This is None if there is no score.
437 src - The path for the source file.
438 dst - The path for the destination file. This is only present for
439 copy or renames. If it is not present, this is None.
441 If the pattern is not matched, None is returned."""
443 match
= diffTreePattern().next().match(entry
)
446 'src_mode': match
.group(1),
447 'dst_mode': match
.group(2),
448 'src_sha1': match
.group(3),
449 'dst_sha1': match
.group(4),
450 'status': match
.group(5),
451 'status_score': match
.group(6),
452 'src': match
.group(7),
453 'dst': match
.group(10)
457 def isModeExec(mode
):
458 # Returns True if the given git mode represents an executable file,
460 return mode
[-3:] == "755"
462 def isModeExecChanged(src_mode
, dst_mode
):
463 return isModeExec(src_mode
) != isModeExec(dst_mode
)
465 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
467 if isinstance(cmd
,basestring
):
474 cmd
= p4_build_cmd(cmd
)
476 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
478 # Use a temporary file to avoid deadlocks without
479 # subprocess.communicate(), which would put another copy
480 # of stdout into memory.
482 if stdin
is not None:
483 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
484 if isinstance(stdin
,basestring
):
485 stdin_file
.write(stdin
)
488 stdin_file
.write(i
+ '\n')
492 p4
= subprocess
.Popen(cmd
,
495 stdout
=subprocess
.PIPE
)
500 entry
= marshal
.load(p4
.stdout
)
510 entry
["p4ExitCode"] = exitCode
516 list = p4CmdList(cmd
)
522 def p4Where(depotPath
):
523 if not depotPath
.endswith("/"):
525 depotPathLong
= depotPath
+ "..."
526 outputList
= p4CmdList(["where", depotPathLong
])
528 for entry
in outputList
:
529 if "depotFile" in entry
:
530 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
531 # The base path always ends with "/...".
532 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
535 elif "data" in entry
:
536 data
= entry
.get("data")
537 space
= data
.find(" ")
538 if data
[:space
] == depotPath
:
543 if output
["code"] == "error":
547 clientPath
= output
.get("path")
548 elif "data" in output
:
549 data
= output
.get("data")
550 lastSpace
= data
.rfind(" ")
551 clientPath
= data
[lastSpace
+ 1:]
553 if clientPath
.endswith("..."):
554 clientPath
= clientPath
[:-3]
557 def currentGitBranch():
558 retcode
= system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error
=True)
563 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
565 def isValidGitDir(path
):
566 if (os
.path
.exists(path
+ "/HEAD")
567 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
571 def parseRevision(ref
):
572 return read_pipe("git rev-parse %s" % ref
).strip()
574 def branchExists(ref
):
575 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
579 def extractLogMessageFromGitCommit(commit
):
582 ## fixme: title is first line of commit, not 1st paragraph.
584 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
593 def extractSettingsGitLog(log
):
595 for line
in log
.split("\n"):
597 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
601 assignments
= m
.group(1).split (':')
602 for a
in assignments
:
604 key
= vals
[0].strip()
605 val
= ('='.join (vals
[1:])).strip()
606 if val
.endswith ('\"') and val
.startswith('"'):
611 paths
= values
.get("depot-paths")
613 paths
= values
.get("depot-path")
615 values
['depot-paths'] = paths
.split(',')
618 def gitBranchExists(branch
):
619 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
620 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
621 return proc
.wait() == 0;
625 def gitConfig(key
, typeSpecifier
=None):
626 if not _gitConfig
.has_key(key
):
627 cmd
= [ "git", "config" ]
629 cmd
+= [ typeSpecifier
]
631 s
= read_pipe(cmd
, ignore_error
=True)
632 _gitConfig
[key
] = s
.strip()
633 return _gitConfig
[key
]
635 def gitConfigBool(key
):
636 """Return a bool, using git config --bool. It is True only if the
637 variable is set to true, and False if set to false or not present
640 if not _gitConfig
.has_key(key
):
641 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
642 return _gitConfig
[key
]
644 def gitConfigInt(key
):
645 if not _gitConfig
.has_key(key
):
646 cmd
= [ "git", "config", "--int", key
]
647 s
= read_pipe(cmd
, ignore_error
=True)
650 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
652 _gitConfig
[key
] = None
653 return _gitConfig
[key
]
655 def gitConfigList(key
):
656 if not _gitConfig
.has_key(key
):
657 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
658 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
659 if _gitConfig
[key
] == ['']:
661 return _gitConfig
[key
]
663 def p4BranchesInGit(branchesAreInRemotes
=True):
664 """Find all the branches whose names start with "p4/", looking
665 in remotes or heads as specified by the argument. Return
666 a dictionary of { branch: revision } for each one found.
667 The branch names are the short names, without any
672 cmdline
= "git rev-parse --symbolic "
673 if branchesAreInRemotes
:
674 cmdline
+= "--remotes"
676 cmdline
+= "--branches"
678 for line
in read_pipe_lines(cmdline
):
682 if not line
.startswith('p4/'):
684 # special symbolic ref to p4/master
685 if line
== "p4/HEAD":
688 # strip off p4/ prefix
689 branch
= line
[len("p4/"):]
691 branches
[branch
] = parseRevision(line
)
695 def branch_exists(branch
):
696 """Make sure that the given ref name really exists."""
698 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
699 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
700 out
, _
= p
.communicate()
703 # expect exactly one line of output: the branch name
704 return out
.rstrip() == branch
706 def findUpstreamBranchPoint(head
= "HEAD"):
707 branches
= p4BranchesInGit()
708 # map from depot-path to branch name
709 branchByDepotPath
= {}
710 for branch
in branches
.keys():
711 tip
= branches
[branch
]
712 log
= extractLogMessageFromGitCommit(tip
)
713 settings
= extractSettingsGitLog(log
)
714 if settings
.has_key("depot-paths"):
715 paths
= ",".join(settings
["depot-paths"])
716 branchByDepotPath
[paths
] = "remotes/p4/" + branch
720 while parent
< 65535:
721 commit
= head
+ "~%s" % parent
722 log
= extractLogMessageFromGitCommit(commit
)
723 settings
= extractSettingsGitLog(log
)
724 if settings
.has_key("depot-paths"):
725 paths
= ",".join(settings
["depot-paths"])
726 if branchByDepotPath
.has_key(paths
):
727 return [branchByDepotPath
[paths
], settings
]
731 return ["", settings
]
733 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
735 print ("Creating/updating branch(es) in %s based on origin branch(es)"
738 originPrefix
= "origin/p4/"
740 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
742 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
745 headName
= line
[len(originPrefix
):]
746 remoteHead
= localRefPrefix
+ headName
749 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
750 if (not original
.has_key('depot-paths')
751 or not original
.has_key('change')):
755 if not gitBranchExists(remoteHead
):
757 print "creating %s" % remoteHead
760 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
761 if settings
.has_key('change') > 0:
762 if settings
['depot-paths'] == original
['depot-paths']:
763 originP4Change
= int(original
['change'])
764 p4Change
= int(settings
['change'])
765 if originP4Change
> p4Change
:
766 print ("%s (%s) is newer than %s (%s). "
767 "Updating p4 branch from origin."
768 % (originHead
, originP4Change
,
769 remoteHead
, p4Change
))
772 print ("Ignoring: %s was imported from %s while "
773 "%s was imported from %s"
774 % (originHead
, ','.join(original
['depot-paths']),
775 remoteHead
, ','.join(settings
['depot-paths'])))
778 system("git update-ref %s %s" % (remoteHead
, originHead
))
780 def originP4BranchesExist():
781 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
784 def p4ParseNumericChangeRange(parts
):
785 changeStart
= int(parts
[0][1:])
786 if parts
[1] == '#head':
787 changeEnd
= p4_last_change()
789 changeEnd
= int(parts
[1])
791 return (changeStart
, changeEnd
)
793 def chooseBlockSize(blockSize
):
797 return defaultBlockSize
799 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
802 # Parse the change range into start and end. Try to find integer
803 # revision ranges as these can be broken up into blocks to avoid
804 # hitting server-side limits (maxrows, maxscanresults). But if
805 # that doesn't work, fall back to using the raw revision specifier
806 # strings, without using block mode.
808 if changeRange
is None or changeRange
== '':
810 changeEnd
= p4_last_change()
811 block_size
= chooseBlockSize(requestedBlockSize
)
813 parts
= changeRange
.split(',')
814 assert len(parts
) == 2
816 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
817 block_size
= chooseBlockSize(requestedBlockSize
)
819 changeStart
= parts
[0][1:]
821 if requestedBlockSize
:
822 die("cannot use --changes-block-size with non-numeric revisions")
827 # Retrieve changes a block at a time, to prevent running
828 # into a MaxResults/MaxScanRows error from the server.
834 end
= min(changeEnd
, changeStart
+ block_size
)
835 revisionRange
= "%d,%d" % (changeStart
, end
)
837 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
840 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
842 # Insert changes in chronological order
843 for line
in reversed(p4_read_pipe_lines(cmd
)):
844 changes
.append(int(line
.split(" ")[1]))
852 changeStart
= end
+ 1
854 changes
= sorted(changes
)
857 def p4PathStartsWith(path
, prefix
):
858 # This method tries to remedy a potential mixed-case issue:
860 # If UserA adds //depot/DirA/file1
861 # and UserB adds //depot/dira/file2
863 # we may or may not have a problem. If you have core.ignorecase=true,
864 # we treat DirA and dira as the same directory
865 if gitConfigBool("core.ignorecase"):
866 return path
.lower().startswith(prefix
.lower())
867 return path
.startswith(prefix
)
870 """Look at the p4 client spec, create a View() object that contains
871 all the mappings, and return it."""
873 specList
= p4CmdList("client -o")
874 if len(specList
) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' %
878 # dictionary of all client parameters
882 client_name
= entry
["Client"]
884 # just the keys that start with "View"
885 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
888 view
= View(client_name
)
890 # append the lines, in order, to the view
891 for view_num
in range(len(view_keys
)):
892 k
= "View%d" % view_num
893 if k
not in view_keys
:
894 die("Expected view key %s missing" % k
)
895 view
.append(entry
[k
])
900 """Grab the client directory."""
902 output
= p4CmdList("client -o")
904 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
907 if "Root" not in entry
:
908 die('Client has no "Root"')
913 # P4 wildcards are not allowed in filenames. P4 complains
914 # if you simply add them, but you can force it with "-f", in
915 # which case it translates them into %xx encoding internally.
917 def wildcard_decode(path
):
918 # Search for and fix just these four characters. Do % last so
919 # that fixing it does not inadvertently create new %-escapes.
920 # Cannot have * in a filename in windows; untested as to
921 # what p4 would do in such a case.
922 if not platform
.system() == "Windows":
923 path
= path
.replace("%2A", "*")
924 path
= path
.replace("%23", "#") \
925 .replace("%40", "@") \
929 def wildcard_encode(path
):
930 # do % first to avoid double-encoding the %s introduced here
931 path
= path
.replace("%", "%25") \
932 .replace("*", "%2A") \
933 .replace("#", "%23") \
937 def wildcard_present(path
):
938 m
= re
.search("[*#@%]", path
)
941 class LargeFileSystem(object):
942 """Base class for large file system support."""
944 def __init__(self
, writeToGitStream
):
945 self
.largeFiles
= set()
946 self
.writeToGitStream
= writeToGitStream
948 def generatePointer(self
, cloneDestination
, contentFile
):
949 """Return the content of a pointer file that is stored in Git instead of
950 the actual content."""
951 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
953 def pushFile(self
, localLargeFile
):
954 """Push the actual content which is not stored in the Git repository to
956 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
958 def hasLargeFileExtension(self
, relPath
):
961 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
965 def generateTempFile(self
, contents
):
966 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
970 return contentFile
.name
972 def exceedsLargeFileThreshold(self
, relPath
, contents
):
973 if gitConfigInt('git-p4.largeFileThreshold'):
974 contentsSize
= sum(len(d
) for d
in contents
)
975 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
977 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
978 contentsSize
= sum(len(d
) for d
in contents
)
979 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
981 contentTempFile
= self
.generateTempFile(contents
)
982 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
983 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
984 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
986 compressedContentsSize
= zf
.infolist()[0].compress_size
987 os
.remove(contentTempFile
)
988 os
.remove(compressedContentFile
.name
)
989 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
993 def addLargeFile(self
, relPath
):
994 self
.largeFiles
.add(relPath
)
996 def removeLargeFile(self
, relPath
):
997 self
.largeFiles
.remove(relPath
)
999 def isLargeFile(self
, relPath
):
1000 return relPath
in self
.largeFiles
1002 def processContent(self
, git_mode
, relPath
, contents
):
1003 """Processes the content of git fast import. This method decides if a
1004 file is stored in the large file system and handles all necessary
1006 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1007 contentTempFile
= self
.generateTempFile(contents
)
1008 (pointer_git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1009 if pointer_git_mode
:
1010 git_mode
= pointer_git_mode
1012 # Move temp file to final location in large file system
1013 largeFileDir
= os
.path
.dirname(localLargeFile
)
1014 if not os
.path
.isdir(largeFileDir
):
1015 os
.makedirs(largeFileDir
)
1016 shutil
.move(contentTempFile
, localLargeFile
)
1017 self
.addLargeFile(relPath
)
1018 if gitConfigBool('git-p4.largeFilePush'):
1019 self
.pushFile(localLargeFile
)
1021 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1022 return (git_mode
, contents
)
1024 class MockLFS(LargeFileSystem
):
1025 """Mock large file system for testing."""
1027 def generatePointer(self
, contentFile
):
1028 """The pointer content is the original content prefixed with "pointer-".
1029 The local filename of the large file storage is derived from the file content.
1031 with
open(contentFile
, 'r') as f
:
1034 pointerContents
= 'pointer-' + content
1035 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1036 return (gitMode
, pointerContents
, localLargeFile
)
1038 def pushFile(self
, localLargeFile
):
1039 """The remote filename of the large file storage is the same as the local
1040 one but in a different directory.
1042 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1043 if not os
.path
.exists(remotePath
):
1044 os
.makedirs(remotePath
)
1045 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1047 class GitLFS(LargeFileSystem
):
1048 """Git LFS as backend for the git-p4 large file system.
1049 See https://git-lfs.github.com/ for details."""
1051 def __init__(self
, *args
):
1052 LargeFileSystem
.__init
__(self
, *args
)
1053 self
.baseGitAttributes
= []
1055 def generatePointer(self
, contentFile
):
1056 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1057 mode and content which is stored in the Git repository instead of
1058 the actual content. Return also the new location of the actual
1061 if os
.path
.getsize(contentFile
) == 0:
1062 return (None, '', None)
1064 pointerProcess
= subprocess
.Popen(
1065 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1066 stdout
=subprocess
.PIPE
1068 pointerFile
= pointerProcess
.stdout
.read()
1069 if pointerProcess
.wait():
1070 os
.remove(contentFile
)
1071 die('git-lfs pointer command failed. Did you install the extension?')
1073 # Git LFS removed the preamble in the output of the 'pointer' command
1074 # starting from version 1.2.0. Check for the preamble here to support
1076 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1077 if pointerFile
.startswith('Git LFS pointer for'):
1078 pointerFile
= re
.sub(r
'Git LFS pointer for.*\n\n', '', pointerFile
)
1080 oid
= re
.search(r
'^oid \w+:(\w+)', pointerFile
, re
.MULTILINE
).group(1)
1081 localLargeFile
= os
.path
.join(
1083 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1086 # LFS Spec states that pointer files should not have the executable bit set.
1088 return (gitMode
, pointerFile
, localLargeFile
)
1090 def pushFile(self
, localLargeFile
):
1091 uploadProcess
= subprocess
.Popen(
1092 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1094 if uploadProcess
.wait():
1095 die('git-lfs push command failed. Did you define a remote?')
1097 def generateGitAttributes(self
):
1099 self
.baseGitAttributes
+
1103 '# Git LFS (see https://git-lfs.github.com/)\n',
1106 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1107 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1109 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1110 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1114 def addLargeFile(self
, relPath
):
1115 LargeFileSystem
.addLargeFile(self
, relPath
)
1116 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1118 def removeLargeFile(self
, relPath
):
1119 LargeFileSystem
.removeLargeFile(self
, relPath
)
1120 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1122 def processContent(self
, git_mode
, relPath
, contents
):
1123 if relPath
== '.gitattributes':
1124 self
.baseGitAttributes
= contents
1125 return (git_mode
, self
.generateGitAttributes())
1127 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1131 self
.usage
= "usage: %prog [options]"
1132 self
.needsGit
= True
1133 self
.verbose
= False
1137 self
.userMapFromPerforceServer
= False
1138 self
.myP4UserId
= None
1142 return self
.myP4UserId
1144 results
= p4CmdList("user -o")
1146 if r
.has_key('User'):
1147 self
.myP4UserId
= r
['User']
1149 die("Could not find your p4 user id")
1151 def p4UserIsMe(self
, p4User
):
1152 # return True if the given p4 user is actually me
1153 me
= self
.p4UserId()
1154 if not p4User
or p4User
!= me
:
1159 def getUserCacheFilename(self
):
1160 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1161 return home
+ "/.gitp4-usercache.txt"
1163 def getUserMapFromPerforceServer(self
):
1164 if self
.userMapFromPerforceServer
:
1169 for output
in p4CmdList("users"):
1170 if not output
.has_key("User"):
1172 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1173 self
.emails
[output
["Email"]] = output
["User"]
1175 mapUserConfigRegex
= re
.compile(r
"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re
.VERBOSE
)
1176 for mapUserConfig
in gitConfigList("git-p4.mapUser"):
1177 mapUser
= mapUserConfigRegex
.findall(mapUserConfig
)
1178 if mapUser
and len(mapUser
[0]) == 3:
1179 user
= mapUser
[0][0]
1180 fullname
= mapUser
[0][1]
1181 email
= mapUser
[0][2]
1182 self
.users
[user
] = fullname
+ " <" + email
+ ">"
1183 self
.emails
[email
] = user
1186 for (key
, val
) in self
.users
.items():
1187 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1189 open(self
.getUserCacheFilename(), "wb").write(s
)
1190 self
.userMapFromPerforceServer
= True
1192 def loadUserMapFromCache(self
):
1194 self
.userMapFromPerforceServer
= False
1196 cache
= open(self
.getUserCacheFilename(), "rb")
1197 lines
= cache
.readlines()
1200 entry
= line
.strip().split("\t")
1201 self
.users
[entry
[0]] = entry
[1]
1203 self
.getUserMapFromPerforceServer()
1205 class P4Debug(Command
):
1207 Command
.__init
__(self
)
1209 self
.description
= "A tool to debug the output of p4 -G."
1210 self
.needsGit
= False
1212 def run(self
, args
):
1214 for output
in p4CmdList(args
):
1215 print 'Element: %d' % j
1220 class P4RollBack(Command
):
1222 Command
.__init
__(self
)
1224 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1226 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1227 self
.rollbackLocalBranches
= False
1229 def run(self
, args
):
1232 maxChange
= int(args
[0])
1234 if "p4ExitCode" in p4Cmd("changes -m 1"):
1235 die("Problems executing p4");
1237 if self
.rollbackLocalBranches
:
1238 refPrefix
= "refs/heads/"
1239 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1241 refPrefix
= "refs/remotes/"
1242 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1245 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1247 ref
= refPrefix
+ line
1248 log
= extractLogMessageFromGitCommit(ref
)
1249 settings
= extractSettingsGitLog(log
)
1251 depotPaths
= settings
['depot-paths']
1252 change
= settings
['change']
1256 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1257 for p
in depotPaths
]))) == 0:
1258 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1259 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1262 while change
and int(change
) > maxChange
:
1265 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1266 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1267 log
= extractLogMessageFromGitCommit(ref
)
1268 settings
= extractSettingsGitLog(log
)
1271 depotPaths
= settings
['depot-paths']
1272 change
= settings
['change']
1275 print "%s rewound to %s" % (ref
, change
)
1279 class P4Submit(Command
, P4UserMap
):
1281 conflict_behavior_choices
= ("ask", "skip", "quit")
1284 Command
.__init
__(self
)
1285 P4UserMap
.__init
__(self
)
1287 optparse
.make_option("--origin", dest
="origin"),
1288 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1289 # preserve the user, requires relevant p4 permissions
1290 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1291 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1292 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1293 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1294 optparse
.make_option("--conflict", dest
="conflict_behavior",
1295 choices
=self
.conflict_behavior_choices
),
1296 optparse
.make_option("--branch", dest
="branch"),
1298 self
.description
= "Submit changes from git to the perforce depot."
1299 self
.usage
+= " [name of git branch to submit into perforce depot]"
1301 self
.detectRenames
= False
1302 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1303 self
.dry_run
= False
1304 self
.prepare_p4_only
= False
1305 self
.conflict_behavior
= None
1306 self
.isWindows
= (platform
.system() == "Windows")
1307 self
.exportLabels
= False
1308 self
.p4HasMoveCommand
= p4_has_move_command()
1311 if gitConfig('git-p4.largeFileSystem'):
1312 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1315 if len(p4CmdList("opened ...")) > 0:
1316 die("You have files opened with perforce! Close them before starting the sync.")
1318 def separate_jobs_from_description(self
, message
):
1319 """Extract and return a possible Jobs field in the commit
1320 message. It goes into a separate section in the p4 change
1323 A jobs line starts with "Jobs:" and looks like a new field
1324 in a form. Values are white-space separated on the same
1325 line or on following lines that start with a tab.
1327 This does not parse and extract the full git commit message
1328 like a p4 form. It just sees the Jobs: line as a marker
1329 to pass everything from then on directly into the p4 form,
1330 but outside the description section.
1332 Return a tuple (stripped log message, jobs string)."""
1334 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1336 return (message
, None)
1338 jobtext
= message
[m
.start():]
1339 stripped_message
= message
[:m
.start()].rstrip()
1340 return (stripped_message
, jobtext
)
1342 def prepareLogMessage(self
, template
, message
, jobs
):
1343 """Edits the template returned from "p4 change -o" to insert
1344 the message in the Description field, and the jobs text in
1348 inDescriptionSection
= False
1350 for line
in template
.split("\n"):
1351 if line
.startswith("#"):
1352 result
+= line
+ "\n"
1355 if inDescriptionSection
:
1356 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1357 inDescriptionSection
= False
1358 # insert Jobs section
1360 result
+= jobs
+ "\n"
1364 if line
.startswith("Description:"):
1365 inDescriptionSection
= True
1367 for messageLine
in message
.split("\n"):
1368 line
+= "\t" + messageLine
+ "\n"
1370 result
+= line
+ "\n"
1374 def patchRCSKeywords(self
, file, pattern
):
1375 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1376 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1378 outFile
= os
.fdopen(handle
, "w+")
1379 inFile
= open(file, "r")
1380 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1381 for line
in inFile
.readlines():
1382 line
= regexp
.sub(r
'$\1$', line
)
1386 # Forcibly overwrite the original file
1388 shutil
.move(outFileName
, file)
1390 # cleanup our temporary file
1391 os
.unlink(outFileName
)
1392 print "Failed to strip RCS keywords in %s" % file
1395 print "Patched up RCS keywords in %s" % file
1397 def p4UserForCommit(self
,id):
1398 # Return the tuple (perforce user,git email) for a given git commit id
1399 self
.getUserMapFromPerforceServer()
1400 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1401 "--format=%ae", id])
1402 gitEmail
= gitEmail
.strip()
1403 if not self
.emails
.has_key(gitEmail
):
1404 return (None,gitEmail
)
1406 return (self
.emails
[gitEmail
],gitEmail
)
1408 def checkValidP4Users(self
,commits
):
1409 # check if any git authors cannot be mapped to p4 users
1411 (user
,email
) = self
.p4UserForCommit(id)
1413 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1414 if gitConfigBool("git-p4.allowMissingP4Users"):
1417 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1419 def lastP4Changelist(self
):
1420 # Get back the last changelist number submitted in this client spec. This
1421 # then gets used to patch up the username in the change. If the same
1422 # client spec is being used by multiple processes then this might go
1424 results
= p4CmdList("client -o") # find the current client
1427 if r
.has_key('Client'):
1428 client
= r
['Client']
1431 die("could not get client spec")
1432 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1434 if r
.has_key('change'):
1436 die("Could not get changelist number for last submit - cannot patch up user details")
1438 def modifyChangelistUser(self
, changelist
, newUser
):
1439 # fixup the user field of a changelist after it has been submitted.
1440 changes
= p4CmdList("change -o %s" % changelist
)
1441 if len(changes
) != 1:
1442 die("Bad output from p4 change modifying %s to user %s" %
1443 (changelist
, newUser
))
1446 if c
['User'] == newUser
: return # nothing to do
1448 input = marshal
.dumps(c
)
1450 result
= p4CmdList("change -f -i", stdin
=input)
1452 if r
.has_key('code'):
1453 if r
['code'] == 'error':
1454 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1455 if r
.has_key('data'):
1456 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1458 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1460 def canChangeChangelists(self
):
1461 # check to see if we have p4 admin or super-user permissions, either of
1462 # which are required to modify changelists.
1463 results
= p4CmdList(["protects", self
.depotPath
])
1465 if r
.has_key('perm'):
1466 if r
['perm'] == 'admin':
1468 if r
['perm'] == 'super':
1472 def prepareSubmitTemplate(self
):
1473 """Run "p4 change -o" to grab a change specification template.
1474 This does not use "p4 -G", as it is nice to keep the submission
1475 template in original order, since a human might edit it.
1477 Remove lines in the Files section that show changes to files
1478 outside the depot path we're committing into."""
1480 [upstream
, settings
] = findUpstreamBranchPoint()
1483 inFilesSection
= False
1484 for line
in p4_read_pipe_lines(['change', '-o']):
1485 if line
.endswith("\r\n"):
1486 line
= line
[:-2] + "\n"
1488 if line
.startswith("\t"):
1489 # path starts and ends with a tab
1491 lastTab
= path
.rfind("\t")
1493 path
= path
[:lastTab
]
1494 if settings
.has_key('depot-paths'):
1495 if not [p
for p
in settings
['depot-paths']
1496 if p4PathStartsWith(path
, p
)]:
1499 if not p4PathStartsWith(path
, self
.depotPath
):
1502 inFilesSection
= False
1504 if line
.startswith("Files:"):
1505 inFilesSection
= True
1511 def edit_template(self
, template_file
):
1512 """Invoke the editor to let the user change the submission
1513 message. Return true if okay to continue with the submit."""
1515 # if configured to skip the editing part, just submit
1516 if gitConfigBool("git-p4.skipSubmitEdit"):
1519 # look at the modification time, to check later if the user saved
1521 mtime
= os
.stat(template_file
).st_mtime
1524 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1525 editor
= os
.environ
.get("P4EDITOR")
1527 editor
= read_pipe("git var GIT_EDITOR").strip()
1528 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1530 # If the file was not saved, prompt to see if this patch should
1531 # be skipped. But skip this verification step if configured so.
1532 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1535 # modification time updated means user saved the file
1536 if os
.stat(template_file
).st_mtime
> mtime
:
1540 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1546 def get_diff_description(self
, editedFiles
, filesToAdd
):
1548 if os
.environ
.has_key("P4DIFF"):
1549 del(os
.environ
["P4DIFF"])
1551 for editedFile
in editedFiles
:
1552 diff
+= p4_read_pipe(['diff', '-du',
1553 wildcard_encode(editedFile
)])
1557 for newFile
in filesToAdd
:
1558 newdiff
+= "==== new file ====\n"
1559 newdiff
+= "--- /dev/null\n"
1560 newdiff
+= "+++ %s\n" % newFile
1561 f
= open(newFile
, "r")
1562 for line
in f
.readlines():
1563 newdiff
+= "+" + line
1566 return (diff
+ newdiff
).replace('\r\n', '\n')
1568 def applyCommit(self
, id):
1569 """Apply one commit, return True if it succeeded."""
1571 print "Applying", read_pipe(["git", "show", "-s",
1572 "--format=format:%h %s", id])
1574 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1576 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1578 filesToChangeType
= set()
1579 filesToDelete
= set()
1581 pureRenameCopy
= set()
1582 filesToChangeExecBit
= {}
1585 diff
= parseDiffTreeEntry(line
)
1586 modifier
= diff
['status']
1590 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1591 filesToChangeExecBit
[path
] = diff
['dst_mode']
1592 editedFiles
.add(path
)
1593 elif modifier
== "A":
1594 filesToAdd
.add(path
)
1595 filesToChangeExecBit
[path
] = diff
['dst_mode']
1596 if path
in filesToDelete
:
1597 filesToDelete
.remove(path
)
1598 elif modifier
== "D":
1599 filesToDelete
.add(path
)
1600 if path
in filesToAdd
:
1601 filesToAdd
.remove(path
)
1602 elif modifier
== "C":
1603 src
, dest
= diff
['src'], diff
['dst']
1604 p4_integrate(src
, dest
)
1605 pureRenameCopy
.add(dest
)
1606 if diff
['src_sha1'] != diff
['dst_sha1']:
1608 pureRenameCopy
.discard(dest
)
1609 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1611 pureRenameCopy
.discard(dest
)
1612 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1614 # turn off read-only attribute
1615 os
.chmod(dest
, stat
.S_IWRITE
)
1617 editedFiles
.add(dest
)
1618 elif modifier
== "R":
1619 src
, dest
= diff
['src'], diff
['dst']
1620 if self
.p4HasMoveCommand
:
1621 p4_edit(src
) # src must be open before move
1622 p4_move(src
, dest
) # opens for (move/delete, move/add)
1624 p4_integrate(src
, dest
)
1625 if diff
['src_sha1'] != diff
['dst_sha1']:
1628 pureRenameCopy
.add(dest
)
1629 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1630 if not self
.p4HasMoveCommand
:
1631 p4_edit(dest
) # with move: already open, writable
1632 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1633 if not self
.p4HasMoveCommand
:
1635 os
.chmod(dest
, stat
.S_IWRITE
)
1637 filesToDelete
.add(src
)
1638 editedFiles
.add(dest
)
1639 elif modifier
== "T":
1640 filesToChangeType
.add(path
)
1642 die("unknown modifier %s for %s" % (modifier
, path
))
1644 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1645 patchcmd
= diffcmd
+ " | git apply "
1646 tryPatchCmd
= patchcmd
+ "--check -"
1647 applyPatchCmd
= patchcmd
+ "--check --apply -"
1648 patch_succeeded
= True
1650 if os
.system(tryPatchCmd
) != 0:
1651 fixed_rcs_keywords
= False
1652 patch_succeeded
= False
1653 print "Unfortunately applying the change failed!"
1655 # Patch failed, maybe it's just RCS keyword woes. Look through
1656 # the patch to see if that's possible.
1657 if gitConfigBool("git-p4.attemptRCSCleanup"):
1661 for file in editedFiles | filesToDelete
:
1662 # did this file's delta contain RCS keywords?
1663 pattern
= p4_keywords_regexp_for_file(file)
1666 # this file is a possibility...look for RCS keywords.
1667 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1668 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1669 if regexp
.search(line
):
1671 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1672 kwfiles
[file] = pattern
1675 for file in kwfiles
:
1677 print "zapping %s with %s" % (line
,pattern
)
1678 # File is being deleted, so not open in p4. Must
1679 # disable the read-only bit on windows.
1680 if self
.isWindows
and file not in editedFiles
:
1681 os
.chmod(file, stat
.S_IWRITE
)
1682 self
.patchRCSKeywords(file, kwfiles
[file])
1683 fixed_rcs_keywords
= True
1685 if fixed_rcs_keywords
:
1686 print "Retrying the patch with RCS keywords cleaned up"
1687 if os
.system(tryPatchCmd
) == 0:
1688 patch_succeeded
= True
1690 if not patch_succeeded
:
1691 for f
in editedFiles
:
1696 # Apply the patch for real, and do add/delete/+x handling.
1698 system(applyPatchCmd
)
1700 for f
in filesToChangeType
:
1701 p4_edit(f
, "-t", "auto")
1702 for f
in filesToAdd
:
1704 for f
in filesToDelete
:
1708 # Set/clear executable bits
1709 for f
in filesToChangeExecBit
.keys():
1710 mode
= filesToChangeExecBit
[f
]
1711 setP4ExecBit(f
, mode
)
1714 # Build p4 change description, starting with the contents
1715 # of the git commit message.
1717 logMessage
= extractLogMessageFromGitCommit(id)
1718 logMessage
= logMessage
.strip()
1719 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1721 template
= self
.prepareSubmitTemplate()
1722 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1724 if self
.preserveUser
:
1725 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1727 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1728 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1729 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1730 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1732 separatorLine
= "######## everything below this line is just the diff #######\n"
1733 if not self
.prepare_p4_only
:
1734 submitTemplate
+= separatorLine
1735 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1737 (handle
, fileName
) = tempfile
.mkstemp()
1738 tmpFile
= os
.fdopen(handle
, "w+b")
1740 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1741 tmpFile
.write(submitTemplate
)
1744 if self
.prepare_p4_only
:
1746 # Leave the p4 tree prepared, and the submit template around
1747 # and let the user decide what to do next
1750 print "P4 workspace prepared for submission."
1751 print "To submit or revert, go to client workspace"
1752 print " " + self
.clientPath
1754 print "To submit, use \"p4 submit\" to write a new description,"
1755 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1756 " \"git p4\"." % fileName
1757 print "You can delete the file \"%s\" when finished." % fileName
1759 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1760 print "To preserve change ownership by user %s, you must\n" \
1761 "do \"p4 change -f <change>\" after submitting and\n" \
1762 "edit the User field."
1764 print "After submitting, renamed files must be re-synced."
1765 print "Invoke \"p4 sync -f\" on each of these files:"
1766 for f
in pureRenameCopy
:
1770 print "To revert the changes, use \"p4 revert ...\", and delete"
1771 print "the submit template file \"%s\"" % fileName
1773 print "Since the commit adds new files, they must be deleted:"
1774 for f
in filesToAdd
:
1780 # Let the user edit the change description, then submit it.
1785 if self
.edit_template(fileName
):
1786 # read the edited message and submit
1787 tmpFile
= open(fileName
, "rb")
1788 message
= tmpFile
.read()
1791 message
= message
.replace("\r\n", "\n")
1792 submitTemplate
= message
[:message
.index(separatorLine
)]
1793 p4_write_pipe(['submit', '-i'], submitTemplate
)
1795 if self
.preserveUser
:
1797 # Get last changelist number. Cannot easily get it from
1798 # the submit command output as the output is
1800 changelist
= self
.lastP4Changelist()
1801 self
.modifyChangelistUser(changelist
, p4User
)
1803 # The rename/copy happened by applying a patch that created a
1804 # new file. This leaves it writable, which confuses p4.
1805 for f
in pureRenameCopy
:
1812 print "Submission cancelled, undoing p4 changes."
1813 for f
in editedFiles
:
1815 for f
in filesToAdd
:
1818 for f
in filesToDelete
:
1824 # Export git tags as p4 labels. Create a p4 label and then tag
1826 def exportGitTags(self
, gitTags
):
1827 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1828 if len(validLabelRegexp
) == 0:
1829 validLabelRegexp
= defaultLabelRegexp
1830 m
= re
.compile(validLabelRegexp
)
1832 for name
in gitTags
:
1834 if not m
.match(name
):
1836 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1839 # Get the p4 commit this corresponds to
1840 logMessage
= extractLogMessageFromGitCommit(name
)
1841 values
= extractSettingsGitLog(logMessage
)
1843 if not values
.has_key('change'):
1844 # a tag pointing to something not sent to p4; ignore
1846 print "git tag %s does not give a p4 commit" % name
1849 changelist
= values
['change']
1851 # Get the tag details.
1855 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1858 if re
.match(r
'tag\s+', l
):
1860 elif re
.match(r
'\s*$', l
):
1867 body
= ["lightweight tag imported by git p4\n"]
1869 # Create the label - use the same view as the client spec we are using
1870 clientSpec
= getClientSpec()
1872 labelTemplate
= "Label: %s\n" % name
1873 labelTemplate
+= "Description:\n"
1875 labelTemplate
+= "\t" + b
+ "\n"
1876 labelTemplate
+= "View:\n"
1877 for depot_side
in clientSpec
.mappings
:
1878 labelTemplate
+= "\t%s\n" % depot_side
1881 print "Would create p4 label %s for tag" % name
1882 elif self
.prepare_p4_only
:
1883 print "Not creating p4 label %s for tag due to option" \
1884 " --prepare-p4-only" % name
1886 p4_write_pipe(["label", "-i"], labelTemplate
)
1889 p4_system(["tag", "-l", name
] +
1890 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1893 print "created p4 label for tag %s" % name
1895 def run(self
, args
):
1897 self
.master
= currentGitBranch()
1898 elif len(args
) == 1:
1899 self
.master
= args
[0]
1900 if not branchExists(self
.master
):
1901 die("Branch %s does not exist" % self
.master
)
1906 allowSubmit
= gitConfig("git-p4.allowSubmit")
1907 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1908 die("%s is not in git-p4.allowSubmit" % self
.master
)
1910 [upstream
, settings
] = findUpstreamBranchPoint()
1911 self
.depotPath
= settings
['depot-paths'][0]
1912 if len(self
.origin
) == 0:
1913 self
.origin
= upstream
1915 if self
.preserveUser
:
1916 if not self
.canChangeChangelists():
1917 die("Cannot preserve user names without p4 super-user or admin permissions")
1919 # if not set from the command line, try the config file
1920 if self
.conflict_behavior
is None:
1921 val
= gitConfig("git-p4.conflict")
1923 if val
not in self
.conflict_behavior_choices
:
1924 die("Invalid value '%s' for config git-p4.conflict" % val
)
1927 self
.conflict_behavior
= val
1930 print "Origin branch is " + self
.origin
1932 if len(self
.depotPath
) == 0:
1933 print "Internal error: cannot locate perforce depot path from existing branches"
1936 self
.useClientSpec
= False
1937 if gitConfigBool("git-p4.useclientspec"):
1938 self
.useClientSpec
= True
1939 if self
.useClientSpec
:
1940 self
.clientSpecDirs
= getClientSpec()
1942 # Check for the existence of P4 branches
1943 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1945 if self
.useClientSpec
and not branchesDetected
:
1946 # all files are relative to the client spec
1947 self
.clientPath
= getClientRoot()
1949 self
.clientPath
= p4Where(self
.depotPath
)
1951 if self
.clientPath
== "":
1952 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1954 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1955 self
.oldWorkingDirectory
= os
.getcwd()
1957 # ensure the clientPath exists
1958 new_client_dir
= False
1959 if not os
.path
.exists(self
.clientPath
):
1960 new_client_dir
= True
1961 os
.makedirs(self
.clientPath
)
1963 chdir(self
.clientPath
, is_client_path
=True)
1965 print "Would synchronize p4 checkout in %s" % self
.clientPath
1967 print "Synchronizing p4 checkout..."
1969 # old one was destroyed, and maybe nobody told p4
1970 p4_sync("...", "-f")
1977 commitish
= self
.master
1981 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, commitish
)]):
1982 commits
.append(line
.strip())
1985 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
1986 self
.checkAuthorship
= False
1988 self
.checkAuthorship
= True
1990 if self
.preserveUser
:
1991 self
.checkValidP4Users(commits
)
1994 # Build up a set of options to be passed to diff when
1995 # submitting each commit to p4.
1997 if self
.detectRenames
:
1998 # command-line -M arg
1999 self
.diffOpts
= "-M"
2001 # If not explicitly set check the config variable
2002 detectRenames
= gitConfig("git-p4.detectRenames")
2004 if detectRenames
.lower() == "false" or detectRenames
== "":
2006 elif detectRenames
.lower() == "true":
2007 self
.diffOpts
= "-M"
2009 self
.diffOpts
= "-M%s" % detectRenames
2011 # no command-line arg for -C or --find-copies-harder, just
2013 detectCopies
= gitConfig("git-p4.detectCopies")
2014 if detectCopies
.lower() == "false" or detectCopies
== "":
2016 elif detectCopies
.lower() == "true":
2017 self
.diffOpts
+= " -C"
2019 self
.diffOpts
+= " -C%s" % detectCopies
2021 if gitConfigBool("git-p4.detectCopiesHarder"):
2022 self
.diffOpts
+= " --find-copies-harder"
2025 # Apply the commits, one at a time. On failure, ask if should
2026 # continue to try the rest of the patches, or quit.
2031 last
= len(commits
) - 1
2032 for i
, commit
in enumerate(commits
):
2034 print " ", read_pipe(["git", "show", "-s",
2035 "--format=format:%h %s", commit
])
2038 ok
= self
.applyCommit(commit
)
2040 applied
.append(commit
)
2042 if self
.prepare_p4_only
and i
< last
:
2043 print "Processing only the first commit due to option" \
2044 " --prepare-p4-only"
2049 # prompt for what to do, or use the option/variable
2050 if self
.conflict_behavior
== "ask":
2051 print "What do you want to do?"
2052 response
= raw_input("[s]kip this commit but apply"
2053 " the rest, or [q]uit? ")
2056 elif self
.conflict_behavior
== "skip":
2058 elif self
.conflict_behavior
== "quit":
2061 die("Unknown conflict_behavior '%s'" %
2062 self
.conflict_behavior
)
2064 if response
[0] == "s":
2065 print "Skipping this commit, but applying the rest"
2067 if response
[0] == "q":
2074 chdir(self
.oldWorkingDirectory
)
2078 elif self
.prepare_p4_only
:
2080 elif len(commits
) == len(applied
):
2081 print "All commits applied!"
2085 sync
.branch
= self
.branch
2092 if len(applied
) == 0:
2093 print "No commits applied."
2095 print "Applied only the commits marked with '*':"
2101 print star
, read_pipe(["git", "show", "-s",
2102 "--format=format:%h %s", c
])
2103 print "You will have to do 'git p4 sync' and rebase."
2105 if gitConfigBool("git-p4.exportLabels"):
2106 self
.exportLabels
= True
2108 if self
.exportLabels
:
2109 p4Labels
= getP4Labels(self
.depotPath
)
2110 gitTags
= getGitTags()
2112 missingGitTags
= gitTags
- p4Labels
2113 self
.exportGitTags(missingGitTags
)
2115 # exit with error unless everything applied perfectly
2116 if len(commits
) != len(applied
):
2122 """Represent a p4 view ("p4 help views"), and map files in a
2123 repo according to the view."""
2125 def __init__(self
, client_name
):
2127 self
.client_prefix
= "//%s/" % client_name
2128 # cache results of "p4 where" to lookup client file locations
2129 self
.client_spec_path_cache
= {}
2131 def append(self
, view_line
):
2132 """Parse a view line, splitting it into depot and client
2133 sides. Append to self.mappings, preserving order. This
2134 is only needed for tag creation."""
2136 # Split the view line into exactly two words. P4 enforces
2137 # structure on these lines that simplifies this quite a bit.
2139 # Either or both words may be double-quoted.
2140 # Single quotes do not matter.
2141 # Double-quote marks cannot occur inside the words.
2142 # A + or - prefix is also inside the quotes.
2143 # There are no quotes unless they contain a space.
2144 # The line is already white-space stripped.
2145 # The two words are separated by a single space.
2147 if view_line
[0] == '"':
2148 # First word is double quoted. Find its end.
2149 close_quote_index
= view_line
.find('"', 1)
2150 if close_quote_index
<= 0:
2151 die("No first-word closing quote found: %s" % view_line
)
2152 depot_side
= view_line
[1:close_quote_index
]
2153 # skip closing quote and space
2154 rhs_index
= close_quote_index
+ 1 + 1
2156 space_index
= view_line
.find(" ")
2157 if space_index
<= 0:
2158 die("No word-splitting space found: %s" % view_line
)
2159 depot_side
= view_line
[0:space_index
]
2160 rhs_index
= space_index
+ 1
2162 # prefix + means overlay on previous mapping
2163 if depot_side
.startswith("+"):
2164 depot_side
= depot_side
[1:]
2166 # prefix - means exclude this path, leave out of mappings
2168 if depot_side
.startswith("-"):
2170 depot_side
= depot_side
[1:]
2173 self
.mappings
.append(depot_side
)
2175 def convert_client_path(self
, clientFile
):
2176 # chop off //client/ part to make it relative
2177 if not clientFile
.startswith(self
.client_prefix
):
2178 die("No prefix '%s' on clientFile '%s'" %
2179 (self
.client_prefix
, clientFile
))
2180 return clientFile
[len(self
.client_prefix
):]
2182 def update_client_spec_path_cache(self
, files
):
2183 """ Caching file paths by "p4 where" batch query """
2185 # List depot file paths exclude that already cached
2186 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2188 if len(fileArgs
) == 0:
2189 return # All files in cache
2191 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2192 for res
in where_result
:
2193 if "code" in res
and res
["code"] == "error":
2194 # assume error is "... file(s) not in client view"
2196 if "clientFile" not in res
:
2197 die("No clientFile in 'p4 where' output")
2199 # it will list all of them, but only one not unmap-ped
2201 if gitConfigBool("core.ignorecase"):
2202 res
['depotFile'] = res
['depotFile'].lower()
2203 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2205 # not found files or unmap files set to ""
2206 for depotFile
in fileArgs
:
2207 if gitConfigBool("core.ignorecase"):
2208 depotFile
= depotFile
.lower()
2209 if depotFile
not in self
.client_spec_path_cache
:
2210 self
.client_spec_path_cache
[depotFile
] = ""
2212 def map_in_client(self
, depot_path
):
2213 """Return the relative location in the client where this
2214 depot file should live. Returns "" if the file should
2215 not be mapped in the client."""
2217 if gitConfigBool("core.ignorecase"):
2218 depot_path
= depot_path
.lower()
2220 if depot_path
in self
.client_spec_path_cache
:
2221 return self
.client_spec_path_cache
[depot_path
]
2223 die( "Error: %s is not found in client spec path" % depot_path
)
2226 class P4Sync(Command
, P4UserMap
):
2227 delete_actions
= ( "delete", "move/delete", "purge" )
2230 Command
.__init
__(self
)
2231 P4UserMap
.__init
__(self
)
2233 optparse
.make_option("--branch", dest
="branch"),
2234 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2235 optparse
.make_option("--changesfile", dest
="changesFile"),
2236 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2237 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2238 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2239 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2240 help="Import into refs/heads/ , not refs/remotes"),
2241 optparse
.make_option("--max-changes", dest
="maxChanges",
2242 help="Maximum number of changes to import"),
2243 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2244 help="Internal block size to use when iteratively calling p4 changes"),
2245 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2246 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2247 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2248 help="Only sync files that are included in the Perforce Client Spec"),
2249 optparse
.make_option("-/", dest
="cloneExclude",
2250 action
="append", type="string",
2251 help="exclude depot path"),
2253 self
.description
= """Imports from Perforce into a git repository.\n
2255 //depot/my/project/ -- to import the current head
2256 //depot/my/project/@all -- to import everything
2257 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2259 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2261 self
.usage
+= " //depot/path[@revRange]"
2263 self
.createdBranches
= set()
2264 self
.committedChanges
= set()
2266 self
.detectBranches
= False
2267 self
.detectLabels
= False
2268 self
.importLabels
= False
2269 self
.changesFile
= ""
2270 self
.syncWithOrigin
= True
2271 self
.importIntoRemotes
= True
2272 self
.maxChanges
= ""
2273 self
.changes_block_size
= None
2274 self
.keepRepoPath
= False
2275 self
.depotPaths
= None
2276 self
.p4BranchesInGit
= []
2277 self
.cloneExclude
= []
2278 self
.useClientSpec
= False
2279 self
.useClientSpec_from_options
= False
2280 self
.clientSpecDirs
= None
2281 self
.tempBranches
= []
2282 self
.tempBranchLocation
= "refs/git-p4-tmp"
2283 self
.largeFileSystem
= None
2285 if gitConfig('git-p4.largeFileSystem'):
2286 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2287 self
.largeFileSystem
= largeFileSystemConstructor(
2288 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2291 if gitConfig("git-p4.syncFromOrigin") == "false":
2292 self
.syncWithOrigin
= False
2294 # This is required for the "append" cloneExclude action
2295 def ensure_value(self
, attr
, value
):
2296 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2297 setattr(self
, attr
, value
)
2298 return getattr(self
, attr
)
2300 # Force a checkpoint in fast-import and wait for it to finish
2301 def checkpoint(self
):
2302 self
.gitStream
.write("checkpoint\n\n")
2303 self
.gitStream
.write("progress checkpoint\n\n")
2304 out
= self
.gitOutput
.readline()
2306 print "checkpoint finished: " + out
2308 def extractFilesFromCommit(self
, commit
):
2309 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2310 for path
in self
.cloneExclude
]
2313 while commit
.has_key("depotFile%s" % fnum
):
2314 path
= commit
["depotFile%s" % fnum
]
2316 if [p
for p
in self
.cloneExclude
2317 if p4PathStartsWith(path
, p
)]:
2320 found
= [p
for p
in self
.depotPaths
2321 if p4PathStartsWith(path
, p
)]
2328 file["rev"] = commit
["rev%s" % fnum
]
2329 file["action"] = commit
["action%s" % fnum
]
2330 file["type"] = commit
["type%s" % fnum
]
2335 def extractJobsFromCommit(self
, commit
):
2338 while commit
.has_key("job%s" % jnum
):
2339 job
= commit
["job%s" % jnum
]
2344 def stripRepoPath(self
, path
, prefixes
):
2345 """When streaming files, this is called to map a p4 depot path
2346 to where it should go in git. The prefixes are either
2347 self.depotPaths, or self.branchPrefixes in the case of
2348 branch detection."""
2350 if self
.useClientSpec
:
2351 # branch detection moves files up a level (the branch name)
2352 # from what client spec interpretation gives
2353 path
= self
.clientSpecDirs
.map_in_client(path
)
2354 if self
.detectBranches
:
2355 for b
in self
.knownBranches
:
2356 if path
.startswith(b
+ "/"):
2357 path
= path
[len(b
)+1:]
2359 elif self
.keepRepoPath
:
2360 # Preserve everything in relative path name except leading
2361 # //depot/; just look at first prefix as they all should
2362 # be in the same depot.
2363 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2364 if p4PathStartsWith(path
, depot
):
2365 path
= path
[len(depot
):]
2369 if p4PathStartsWith(path
, p
):
2370 path
= path
[len(p
):]
2373 path
= wildcard_decode(path
)
2376 def splitFilesIntoBranches(self
, commit
):
2377 """Look at each depotFile in the commit to figure out to what
2378 branch it belongs."""
2380 if self
.clientSpecDirs
:
2381 files
= self
.extractFilesFromCommit(commit
)
2382 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2386 while commit
.has_key("depotFile%s" % fnum
):
2387 path
= commit
["depotFile%s" % fnum
]
2388 found
= [p
for p
in self
.depotPaths
2389 if p4PathStartsWith(path
, p
)]
2396 file["rev"] = commit
["rev%s" % fnum
]
2397 file["action"] = commit
["action%s" % fnum
]
2398 file["type"] = commit
["type%s" % fnum
]
2401 # start with the full relative path where this file would
2403 if self
.useClientSpec
:
2404 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2406 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2408 for branch
in self
.knownBranches
.keys():
2409 # add a trailing slash so that a commit into qt/4.2foo
2410 # doesn't end up in qt/4.2, e.g.
2411 if relPath
.startswith(branch
+ "/"):
2412 if branch
not in branches
:
2413 branches
[branch
] = []
2414 branches
[branch
].append(file)
2419 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2420 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2421 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2423 self
.gitStream
.write(d
)
2424 self
.gitStream
.write('\n')
2426 # output one file from the P4 stream
2427 # - helper for streamP4Files
2429 def streamOneP4File(self
, file, contents
):
2430 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2432 size
= int(self
.stream_file
['fileSize'])
2433 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2436 (type_base
, type_mods
) = split_p4_type(file["type"])
2439 if "x" in type_mods
:
2441 if type_base
== "symlink":
2443 # p4 print on a symlink sometimes contains "target\n";
2444 # if it does, remove the newline
2445 data
= ''.join(contents
)
2447 # Some version of p4 allowed creating a symlink that pointed
2448 # to nothing. This causes p4 errors when checking out such
2449 # a change, and errors here too. Work around it by ignoring
2450 # the bad symlink; hopefully a future change fixes it.
2451 print "\nIgnoring empty symlink in %s" % file['depotFile']
2453 elif data
[-1] == '\n':
2454 contents
= [data
[:-1]]
2458 if type_base
== "utf16":
2459 # p4 delivers different text in the python output to -G
2460 # than it does when using "print -o", or normal p4 client
2461 # operations. utf16 is converted to ascii or utf8, perhaps.
2462 # But ascii text saved as -t utf16 is completely mangled.
2463 # Invoke print -o to get the real contents.
2465 # On windows, the newlines will always be mangled by print, so put
2466 # them back too. This is not needed to the cygwin windows version,
2467 # just the native "NT" type.
2470 text
= p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2471 except Exception as e
:
2472 if 'Translation of file content failed' in str(e
):
2473 type_base
= 'binary'
2477 if p4_version_string().find('/NT') >= 0:
2478 text
= text
.replace('\r\n', '\n')
2481 if type_base
== "apple":
2482 # Apple filetype files will be streamed as a concatenation of
2483 # its appledouble header and the contents. This is useless
2484 # on both macs and non-macs. If using "print -q -o xx", it
2485 # will create "xx" with the data, and "%xx" with the header.
2486 # This is also not very useful.
2488 # Ideally, someday, this script can learn how to generate
2489 # appledouble files directly and import those to git, but
2490 # non-mac machines can never find a use for apple filetype.
2491 print "\nIgnoring apple filetype file %s" % file['depotFile']
2494 # Note that we do not try to de-mangle keywords on utf16 files,
2495 # even though in theory somebody may want that.
2496 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2498 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2499 text
= ''.join(contents
)
2500 text
= regexp
.sub(r
'$\1$', text
)
2504 relPath
.decode('ascii')
2507 if gitConfig('git-p4.pathEncoding'):
2508 encoding
= gitConfig('git-p4.pathEncoding')
2509 relPath
= relPath
.decode(encoding
, 'replace').encode('utf8', 'replace')
2511 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding
, relPath
)
2513 if self
.largeFileSystem
:
2514 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2516 self
.writeToGitStream(git_mode
, relPath
, contents
)
2518 def streamOneP4Deletion(self
, file):
2519 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2521 sys
.stdout
.write("delete %s\n" % relPath
)
2523 self
.gitStream
.write("D %s\n" % relPath
)
2525 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2526 self
.largeFileSystem
.removeLargeFile(relPath
)
2528 # handle another chunk of streaming data
2529 def streamP4FilesCb(self
, marshalled
):
2531 # catch p4 errors and complain
2533 if "code" in marshalled
:
2534 if marshalled
["code"] == "error":
2535 if "data" in marshalled
:
2536 err
= marshalled
["data"].rstrip()
2538 if not err
and 'fileSize' in self
.stream_file
:
2539 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2540 if required_bytes
> 0:
2541 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2542 os
.getcwd(), required_bytes
/1024/1024
2547 if self
.stream_have_file_info
:
2548 if "depotFile" in self
.stream_file
:
2549 f
= self
.stream_file
["depotFile"]
2550 # force a failure in fast-import, else an empty
2551 # commit will be made
2552 self
.gitStream
.write("\n")
2553 self
.gitStream
.write("die-now\n")
2554 self
.gitStream
.close()
2555 # ignore errors, but make sure it exits first
2556 self
.importProcess
.wait()
2558 die("Error from p4 print for %s: %s" % (f
, err
))
2560 die("Error from p4 print: %s" % err
)
2562 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2563 # start of a new file - output the old one first
2564 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2565 self
.stream_file
= {}
2566 self
.stream_contents
= []
2567 self
.stream_have_file_info
= False
2569 # pick up the new file information... for the
2570 # 'data' field we need to append to our array
2571 for k
in marshalled
.keys():
2573 if 'streamContentSize' not in self
.stream_file
:
2574 self
.stream_file
['streamContentSize'] = 0
2575 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2576 self
.stream_contents
.append(marshalled
['data'])
2578 self
.stream_file
[k
] = marshalled
[k
]
2581 'streamContentSize' in self
.stream_file
and
2582 'fileSize' in self
.stream_file
and
2583 'depotFile' in self
.stream_file
):
2584 size
= int(self
.stream_file
["fileSize"])
2586 progress
= 100*self
.stream_file
['streamContentSize']/size
2587 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2590 self
.stream_have_file_info
= True
2592 # Stream directly from "p4 files" into "git fast-import"
2593 def streamP4Files(self
, files
):
2599 filesForCommit
.append(f
)
2600 if f
['action'] in self
.delete_actions
:
2601 filesToDelete
.append(f
)
2603 filesToRead
.append(f
)
2606 for f
in filesToDelete
:
2607 self
.streamOneP4Deletion(f
)
2609 if len(filesToRead
) > 0:
2610 self
.stream_file
= {}
2611 self
.stream_contents
= []
2612 self
.stream_have_file_info
= False
2614 # curry self argument
2615 def streamP4FilesCbSelf(entry
):
2616 self
.streamP4FilesCb(entry
)
2618 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2620 p4CmdList(["-x", "-", "print"],
2622 cb
=streamP4FilesCbSelf
)
2625 if self
.stream_file
.has_key('depotFile'):
2626 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2628 def make_email(self
, userid
):
2629 if userid
in self
.users
:
2630 return self
.users
[userid
]
2632 return "%s <a@b>" % userid
2634 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2635 """ Stream a p4 tag.
2636 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2640 print "writing tag %s for commit %s" % (labelName
, commit
)
2641 gitStream
.write("tag %s\n" % labelName
)
2642 gitStream
.write("from %s\n" % commit
)
2644 if labelDetails
.has_key('Owner'):
2645 owner
= labelDetails
["Owner"]
2649 # Try to use the owner of the p4 label, or failing that,
2650 # the current p4 user id.
2652 email
= self
.make_email(owner
)
2654 email
= self
.make_email(self
.p4UserId())
2655 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2657 gitStream
.write("tagger %s\n" % tagger
)
2659 print "labelDetails=",labelDetails
2660 if labelDetails
.has_key('Description'):
2661 description
= labelDetails
['Description']
2663 description
= 'Label from git p4'
2665 gitStream
.write("data %d\n" % len(description
))
2666 gitStream
.write(description
)
2667 gitStream
.write("\n")
2669 def inClientSpec(self
, path
):
2670 if not self
.clientSpecDirs
:
2672 inClientSpec
= self
.clientSpecDirs
.map_in_client(path
)
2673 if not inClientSpec
and self
.verbose
:
2674 print('Ignoring file outside of client spec: {0}'.format(path
))
2677 def hasBranchPrefix(self
, path
):
2678 if not self
.branchPrefixes
:
2680 hasPrefix
= [p
for p
in self
.branchPrefixes
2681 if p4PathStartsWith(path
, p
)]
2682 if not hasPrefix
and self
.verbose
:
2683 print('Ignoring file outside of prefix: {0}'.format(path
))
2686 def commit(self
, details
, files
, branch
, parent
= ""):
2687 epoch
= details
["time"]
2688 author
= details
["user"]
2689 jobs
= self
.extractJobsFromCommit(details
)
2692 print('commit into {0}'.format(branch
))
2694 if self
.clientSpecDirs
:
2695 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2697 files
= [f
for f
in files
2698 if self
.inClientSpec(f
['path']) and self
.hasBranchPrefix(f
['path'])]
2700 if not files
and not gitConfigBool('git-p4.keepEmptyCommits'):
2701 print('Ignoring revision {0} as it would produce an empty commit.'
2702 .format(details
['change']))
2705 self
.gitStream
.write("commit %s\n" % branch
)
2706 self
.gitStream
.write("mark :%s\n" % details
["change"])
2707 self
.committedChanges
.add(int(details
["change"]))
2709 if author
not in self
.users
:
2710 self
.getUserMapFromPerforceServer()
2711 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2713 self
.gitStream
.write("committer %s\n" % committer
)
2715 self
.gitStream
.write("data <<EOT\n")
2716 self
.gitStream
.write(details
["desc"])
2718 self
.gitStream
.write("\nJobs: %s" % (' '.join(jobs
)))
2719 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2720 (','.join(self
.branchPrefixes
), details
["change"]))
2721 if len(details
['options']) > 0:
2722 self
.gitStream
.write(": options = %s" % details
['options'])
2723 self
.gitStream
.write("]\nEOT\n\n")
2727 print "parent %s" % parent
2728 self
.gitStream
.write("from %s\n" % parent
)
2730 self
.streamP4Files(files
)
2731 self
.gitStream
.write("\n")
2733 change
= int(details
["change"])
2735 if self
.labels
.has_key(change
):
2736 label
= self
.labels
[change
]
2737 labelDetails
= label
[0]
2738 labelRevisions
= label
[1]
2740 print "Change %s is labelled %s" % (change
, labelDetails
)
2742 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2743 for p
in self
.branchPrefixes
])
2745 if len(files
) == len(labelRevisions
):
2749 if info
["action"] in self
.delete_actions
:
2751 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2753 if cleanedFiles
== labelRevisions
:
2754 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2758 print ("Tag %s does not match with change %s: files do not match."
2759 % (labelDetails
["label"], change
))
2763 print ("Tag %s does not match with change %s: file count is different."
2764 % (labelDetails
["label"], change
))
2766 # Build a dictionary of changelists and labels, for "detect-labels" option.
2767 def getLabels(self
):
2770 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2771 if len(l
) > 0 and not self
.silent
:
2772 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2775 label
= output
["label"]
2779 print "Querying files for label %s" % label
2780 for file in p4CmdList(["files"] +
2781 ["%s...@%s" % (p
, label
)
2782 for p
in self
.depotPaths
]):
2783 revisions
[file["depotFile"]] = file["rev"]
2784 change
= int(file["change"])
2785 if change
> newestChange
:
2786 newestChange
= change
2788 self
.labels
[newestChange
] = [output
, revisions
]
2791 print "Label changes: %s" % self
.labels
.keys()
2793 # Import p4 labels as git tags. A direct mapping does not
2794 # exist, so assume that if all the files are at the same revision
2795 # then we can use that, or it's something more complicated we should
2797 def importP4Labels(self
, stream
, p4Labels
):
2799 print "import p4 labels: " + ' '.join(p4Labels
)
2801 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2802 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2803 if len(validLabelRegexp
) == 0:
2804 validLabelRegexp
= defaultLabelRegexp
2805 m
= re
.compile(validLabelRegexp
)
2807 for name
in p4Labels
:
2810 if not m
.match(name
):
2812 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2815 if name
in ignoredP4Labels
:
2818 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2820 # get the most recent changelist for each file in this label
2821 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2822 for p
in self
.depotPaths
])
2824 if change
.has_key('change'):
2825 # find the corresponding git commit; take the oldest commit
2826 changelist
= int(change
['change'])
2827 if changelist
in self
.committedChanges
:
2828 gitCommit
= ":%d" % changelist
# use a fast-import mark
2831 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2832 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
], ignore_error
=True)
2833 if len(gitCommit
) == 0:
2834 print "importing label %s: could not find git commit for changelist %d" % (name
, changelist
)
2837 gitCommit
= gitCommit
.strip()
2840 # Convert from p4 time format
2842 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2844 print "Could not convert label time %s" % labelDetails
['Update']
2847 when
= int(time
.mktime(tmwhen
))
2848 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2850 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2853 print "Label %s has no changelists - possibly deleted?" % name
2856 # We can't import this label; don't try again as it will get very
2857 # expensive repeatedly fetching all the files for labels that will
2858 # never be imported. If the label is moved in the future, the
2859 # ignore will need to be removed manually.
2860 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2862 def guessProjectName(self
):
2863 for p
in self
.depotPaths
:
2866 p
= p
[p
.strip().rfind("/") + 1:]
2867 if not p
.endswith("/"):
2871 def getBranchMapping(self
):
2872 lostAndFoundBranches
= set()
2874 user
= gitConfig("git-p4.branchUser")
2876 command
= "branches -u %s" % user
2878 command
= "branches"
2880 for info
in p4CmdList(command
):
2881 details
= p4Cmd(["branch", "-o", info
["branch"]])
2883 while details
.has_key("View%s" % viewIdx
):
2884 paths
= details
["View%s" % viewIdx
].split(" ")
2885 viewIdx
= viewIdx
+ 1
2886 # require standard //depot/foo/... //depot/bar/... mapping
2887 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2890 destination
= paths
[1]
2892 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2893 source
= source
[len(self
.depotPaths
[0]):-4]
2894 destination
= destination
[len(self
.depotPaths
[0]):-4]
2896 if destination
in self
.knownBranches
:
2898 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2899 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2902 self
.knownBranches
[destination
] = source
2904 lostAndFoundBranches
.discard(destination
)
2906 if source
not in self
.knownBranches
:
2907 lostAndFoundBranches
.add(source
)
2909 # Perforce does not strictly require branches to be defined, so we also
2910 # check git config for a branch list.
2912 # Example of branch definition in git config file:
2914 # branchList=main:branchA
2915 # branchList=main:branchB
2916 # branchList=branchA:branchC
2917 configBranches
= gitConfigList("git-p4.branchList")
2918 for branch
in configBranches
:
2920 (source
, destination
) = branch
.split(":")
2921 self
.knownBranches
[destination
] = source
2923 lostAndFoundBranches
.discard(destination
)
2925 if source
not in self
.knownBranches
:
2926 lostAndFoundBranches
.add(source
)
2929 for branch
in lostAndFoundBranches
:
2930 self
.knownBranches
[branch
] = branch
2932 def getBranchMappingFromGitBranches(self
):
2933 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2934 for branch
in branches
.keys():
2935 if branch
== "master":
2938 branch
= branch
[len(self
.projectName
):]
2939 self
.knownBranches
[branch
] = branch
2941 def updateOptionDict(self
, d
):
2943 if self
.keepRepoPath
:
2944 option_keys
['keepRepoPath'] = 1
2946 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2948 def readOptions(self
, d
):
2949 self
.keepRepoPath
= (d
.has_key('options')
2950 and ('keepRepoPath' in d
['options']))
2952 def gitRefForBranch(self
, branch
):
2953 if branch
== "main":
2954 return self
.refPrefix
+ "master"
2956 if len(branch
) <= 0:
2959 return self
.refPrefix
+ self
.projectName
+ branch
2961 def gitCommitByP4Change(self
, ref
, change
):
2963 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2966 latestCommit
= parseRevision(ref
)
2970 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2971 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2976 log
= extractLogMessageFromGitCommit(next
)
2977 settings
= extractSettingsGitLog(log
)
2978 currentChange
= int(settings
['change'])
2980 print "current change %s" % currentChange
2982 if currentChange
== change
:
2984 print "found %s" % next
2987 if currentChange
< change
:
2988 earliestCommit
= "^%s" % next
2990 latestCommit
= "%s" % next
2994 def importNewBranch(self
, branch
, maxChange
):
2995 # make fast-import flush all changes to disk and update the refs using the checkpoint
2996 # command so that we can try to find the branch parent in the git history
2997 self
.gitStream
.write("checkpoint\n\n");
2998 self
.gitStream
.flush();
2999 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3000 range = "@1,%s" % maxChange
3001 #print "prefix" + branchPrefix
3002 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
3003 if len(changes
) <= 0:
3005 firstChange
= changes
[0]
3006 #print "first change in branch: %s" % firstChange
3007 sourceBranch
= self
.knownBranches
[branch
]
3008 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
3009 sourceRef
= self
.gitRefForBranch(sourceBranch
)
3010 #print "source " + sourceBranch
3012 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
3013 #print "branch parent: %s" % branchParentChange
3014 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
3015 if len(gitParent
) > 0:
3016 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
3017 #print "parent git commit: %s" % gitParent
3019 self
.importChanges(changes
)
3022 def searchParent(self
, parent
, branch
, target
):
3024 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
3025 "--no-merges", parent
]):
3027 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
3030 print "Found parent of %s in commit %s" % (branch
, blob
)
3037 def importChanges(self
, changes
):
3039 for change
in changes
:
3040 description
= p4_describe(change
)
3041 self
.updateOptionDict(description
)
3044 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
3049 if self
.detectBranches
:
3050 branches
= self
.splitFilesIntoBranches(description
)
3051 for branch
in branches
.keys():
3053 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
3054 self
.branchPrefixes
= [ branchPrefix
]
3058 filesForCommit
= branches
[branch
]
3061 print "branch is %s" % branch
3063 self
.updatedBranches
.add(branch
)
3065 if branch
not in self
.createdBranches
:
3066 self
.createdBranches
.add(branch
)
3067 parent
= self
.knownBranches
[branch
]
3068 if parent
== branch
:
3071 fullBranch
= self
.projectName
+ branch
3072 if fullBranch
not in self
.p4BranchesInGit
:
3074 print("\n Importing new branch %s" % fullBranch
);
3075 if self
.importNewBranch(branch
, change
- 1):
3077 self
.p4BranchesInGit
.append(fullBranch
)
3079 print("\n Resuming with change %s" % change
);
3082 print "parent determined through known branches: %s" % parent
3084 branch
= self
.gitRefForBranch(branch
)
3085 parent
= self
.gitRefForBranch(parent
)
3088 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
3090 if len(parent
) == 0 and branch
in self
.initialParents
:
3091 parent
= self
.initialParents
[branch
]
3092 del self
.initialParents
[branch
]
3096 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3098 print "Creating temporary branch: " + tempBranch
3099 self
.commit(description
, filesForCommit
, tempBranch
)
3100 self
.tempBranches
.append(tempBranch
)
3102 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3104 self
.commit(description
, filesForCommit
, branch
, blob
)
3107 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3108 self
.commit(description
, filesForCommit
, branch
, parent
)
3110 files
= self
.extractFilesFromCommit(description
)
3111 self
.commit(description
, files
, self
.branch
,
3113 # only needed once, to connect to the previous commit
3114 self
.initialParent
= ""
3116 print self
.gitError
.read()
3119 def importHeadRevision(self
, revision
):
3120 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3123 details
["user"] = "git perforce import user"
3124 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3125 % (' '.join(self
.depotPaths
), revision
))
3126 details
["change"] = revision
3130 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3132 for info
in p4CmdList(["files"] + fileArgs
):
3134 if 'code' in info
and info
['code'] == 'error':
3135 sys
.stderr
.write("p4 returned an error: %s\n"
3137 if info
['data'].find("must refer to client") >= 0:
3138 sys
.stderr
.write("This particular p4 error is misleading.\n")
3139 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3140 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3142 if 'p4ExitCode' in info
:
3143 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3147 change
= int(info
["change"])
3148 if change
> newestRevision
:
3149 newestRevision
= change
3151 if info
["action"] in self
.delete_actions
:
3152 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3153 #fileCnt = fileCnt + 1
3156 for prop
in ["depotFile", "rev", "action", "type" ]:
3157 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3159 fileCnt
= fileCnt
+ 1
3161 details
["change"] = newestRevision
3163 # Use time from top-most change so that all git p4 clones of
3164 # the same p4 repo have the same commit SHA1s.
3165 res
= p4_describe(newestRevision
)
3166 details
["time"] = res
["time"]
3168 self
.updateOptionDict(details
)
3170 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3172 print "IO error with git fast-import. Is your git version recent enough?"
3173 print self
.gitError
.read()
3176 def run(self
, args
):
3177 self
.depotPaths
= []
3178 self
.changeRange
= ""
3179 self
.previousDepotPaths
= []
3180 self
.hasOrigin
= False
3182 # map from branch depot path to parent branch
3183 self
.knownBranches
= {}
3184 self
.initialParents
= {}
3186 if self
.importIntoRemotes
:
3187 self
.refPrefix
= "refs/remotes/p4/"
3189 self
.refPrefix
= "refs/heads/p4/"
3191 if self
.syncWithOrigin
:
3192 self
.hasOrigin
= originP4BranchesExist()
3195 print 'Syncing with origin first, using "git fetch origin"'
3196 system("git fetch origin")
3198 branch_arg_given
= bool(self
.branch
)
3199 if len(self
.branch
) == 0:
3200 self
.branch
= self
.refPrefix
+ "master"
3201 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3202 system("git update-ref %s refs/heads/p4" % self
.branch
)
3203 system("git branch -D p4")
3205 # accept either the command-line option, or the configuration variable
3206 if self
.useClientSpec
:
3207 # will use this after clone to set the variable
3208 self
.useClientSpec_from_options
= True
3210 if gitConfigBool("git-p4.useclientspec"):
3211 self
.useClientSpec
= True
3212 if self
.useClientSpec
:
3213 self
.clientSpecDirs
= getClientSpec()
3215 # TODO: should always look at previous commits,
3216 # merge with previous imports, if possible.
3219 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3221 # branches holds mapping from branch name to sha1
3222 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3224 # restrict to just this one, disabling detect-branches
3225 if branch_arg_given
:
3226 short
= self
.branch
.split("/")[-1]
3227 if short
in branches
:
3228 self
.p4BranchesInGit
= [ short
]
3230 self
.p4BranchesInGit
= branches
.keys()
3232 if len(self
.p4BranchesInGit
) > 1:
3234 print "Importing from/into multiple branches"
3235 self
.detectBranches
= True
3236 for branch
in branches
.keys():
3237 self
.initialParents
[self
.refPrefix
+ branch
] = \
3241 print "branches: %s" % self
.p4BranchesInGit
3244 for branch
in self
.p4BranchesInGit
:
3245 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3247 settings
= extractSettingsGitLog(logMsg
)
3249 self
.readOptions(settings
)
3250 if (settings
.has_key('depot-paths')
3251 and settings
.has_key ('change')):
3252 change
= int(settings
['change']) + 1
3253 p4Change
= max(p4Change
, change
)
3255 depotPaths
= sorted(settings
['depot-paths'])
3256 if self
.previousDepotPaths
== []:
3257 self
.previousDepotPaths
= depotPaths
3260 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3261 prev_list
= prev
.split("/")
3262 cur_list
= cur
.split("/")
3263 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3264 if cur_list
[i
] <> prev_list
[i
]:
3268 paths
.append ("/".join(cur_list
[:i
+ 1]))
3270 self
.previousDepotPaths
= paths
3273 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3274 self
.changeRange
= "@%s,#head" % p4Change
3275 if not self
.silent
and not self
.detectBranches
:
3276 print "Performing incremental import into %s git branch" % self
.branch
3278 # accept multiple ref name abbreviations:
3279 # refs/foo/bar/branch -> use it exactly
3280 # p4/branch -> prepend refs/remotes/ or refs/heads/
3281 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3282 if not self
.branch
.startswith("refs/"):
3283 if self
.importIntoRemotes
:
3284 prepend
= "refs/remotes/"
3286 prepend
= "refs/heads/"
3287 if not self
.branch
.startswith("p4/"):
3289 self
.branch
= prepend
+ self
.branch
3291 if len(args
) == 0 and self
.depotPaths
:
3293 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3295 if self
.depotPaths
and self
.depotPaths
!= args
:
3296 print ("previous import used depot path %s and now %s was specified. "
3297 "This doesn't work!" % (' '.join (self
.depotPaths
),
3301 self
.depotPaths
= sorted(args
)
3306 # Make sure no revision specifiers are used when --changesfile
3308 bad_changesfile
= False
3309 if len(self
.changesFile
) > 0:
3310 for p
in self
.depotPaths
:
3311 if p
.find("@") >= 0 or p
.find("#") >= 0:
3312 bad_changesfile
= True
3315 die("Option --changesfile is incompatible with revision specifiers")
3318 for p
in self
.depotPaths
:
3319 if p
.find("@") != -1:
3320 atIdx
= p
.index("@")
3321 self
.changeRange
= p
[atIdx
:]
3322 if self
.changeRange
== "@all":
3323 self
.changeRange
= ""
3324 elif ',' not in self
.changeRange
:
3325 revision
= self
.changeRange
3326 self
.changeRange
= ""
3328 elif p
.find("#") != -1:
3329 hashIdx
= p
.index("#")
3330 revision
= p
[hashIdx
:]
3332 elif self
.previousDepotPaths
== []:
3333 # pay attention to changesfile, if given, else import
3334 # the entire p4 tree at the head revision
3335 if len(self
.changesFile
) == 0:
3338 p
= re
.sub ("\.\.\.$", "", p
)
3339 if not p
.endswith("/"):
3344 self
.depotPaths
= newPaths
3346 # --detect-branches may change this for each branch
3347 self
.branchPrefixes
= self
.depotPaths
3349 self
.loadUserMapFromCache()
3351 if self
.detectLabels
:
3354 if self
.detectBranches
:
3355 ## FIXME - what's a P4 projectName ?
3356 self
.projectName
= self
.guessProjectName()
3359 self
.getBranchMappingFromGitBranches()
3361 self
.getBranchMapping()
3363 print "p4-git branches: %s" % self
.p4BranchesInGit
3364 print "initial parents: %s" % self
.initialParents
3365 for b
in self
.p4BranchesInGit
:
3369 b
= b
[len(self
.projectName
):]
3370 self
.createdBranches
.add(b
)
3372 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3374 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3375 stdin
=subprocess
.PIPE
,
3376 stdout
=subprocess
.PIPE
,
3377 stderr
=subprocess
.PIPE
);
3378 self
.gitOutput
= self
.importProcess
.stdout
3379 self
.gitStream
= self
.importProcess
.stdin
3380 self
.gitError
= self
.importProcess
.stderr
3383 self
.importHeadRevision(revision
)
3387 if len(self
.changesFile
) > 0:
3388 output
= open(self
.changesFile
).readlines()
3391 changeSet
.add(int(line
))
3393 for change
in changeSet
:
3394 changes
.append(change
)
3398 # catch "git p4 sync" with no new branches, in a repo that
3399 # does not have any existing p4 branches
3401 if not self
.p4BranchesInGit
:
3402 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3404 # The default branch is master, unless --branch is used to
3405 # specify something else. Make sure it exists, or complain
3406 # nicely about how to use --branch.
3407 if not self
.detectBranches
:
3408 if not branch_exists(self
.branch
):
3409 if branch_arg_given
:
3410 die("Error: branch %s does not exist." % self
.branch
)
3412 die("Error: no branch %s; perhaps specify one with --branch." %
3416 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3418 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3420 if len(self
.maxChanges
) > 0:
3421 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3423 if len(changes
) == 0:
3425 print "No changes to import!"
3427 if not self
.silent
and not self
.detectBranches
:
3428 print "Import destination: %s" % self
.branch
3430 self
.updatedBranches
= set()
3432 if not self
.detectBranches
:
3434 # start a new branch
3435 self
.initialParent
= ""
3437 # build on a previous revision
3438 self
.initialParent
= parseRevision(self
.branch
)
3440 self
.importChanges(changes
)
3444 if len(self
.updatedBranches
) > 0:
3445 sys
.stdout
.write("Updated branches: ")
3446 for b
in self
.updatedBranches
:
3447 sys
.stdout
.write("%s " % b
)
3448 sys
.stdout
.write("\n")
3450 if gitConfigBool("git-p4.importLabels"):
3451 self
.importLabels
= True
3453 if self
.importLabels
:
3454 p4Labels
= getP4Labels(self
.depotPaths
)
3455 gitTags
= getGitTags()
3457 missingP4Labels
= p4Labels
- gitTags
3458 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3460 self
.gitStream
.close()
3461 if self
.importProcess
.wait() != 0:
3462 die("fast-import failed: %s" % self
.gitError
.read())
3463 self
.gitOutput
.close()
3464 self
.gitError
.close()
3466 # Cleanup temporary branches created during import
3467 if self
.tempBranches
!= []:
3468 for branch
in self
.tempBranches
:
3469 read_pipe("git update-ref -d %s" % branch
)
3470 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3472 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3473 # a convenient shortcut refname "p4".
3474 if self
.importIntoRemotes
:
3475 head_ref
= self
.refPrefix
+ "HEAD"
3476 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3477 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3481 class P4Rebase(Command
):
3483 Command
.__init
__(self
)
3485 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3487 self
.importLabels
= False
3488 self
.description
= ("Fetches the latest revision from perforce and "
3489 + "rebases the current work (branch) against it")
3491 def run(self
, args
):
3493 sync
.importLabels
= self
.importLabels
3496 return self
.rebase()
3499 if os
.system("git update-index --refresh") != 0:
3500 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3501 if len(read_pipe("git diff-index HEAD --")) > 0:
3502 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3504 [upstream
, settings
] = findUpstreamBranchPoint()
3505 if len(upstream
) == 0:
3506 die("Cannot find upstream branchpoint for rebase")
3508 # the branchpoint may be p4/foo~3, so strip off the parent
3509 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3511 print "Rebasing the current branch onto %s" % upstream
3512 oldHead
= read_pipe("git rev-parse HEAD").strip()
3513 system("git rebase %s" % upstream
)
3514 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3517 class P4Clone(P4Sync
):
3519 P4Sync
.__init
__(self
)
3520 self
.description
= "Creates a new git repository and imports from Perforce into it"
3521 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3523 optparse
.make_option("--destination", dest
="cloneDestination",
3524 action
='store', default
=None,
3525 help="where to leave result of the clone"),
3526 optparse
.make_option("--bare", dest
="cloneBare",
3527 action
="store_true", default
=False),
3529 self
.cloneDestination
= None
3530 self
.needsGit
= False
3531 self
.cloneBare
= False
3533 def defaultDestination(self
, args
):
3534 ## TODO: use common prefix of args?
3536 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3537 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3538 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3539 depotDir
= re
.sub(r
"/$", "", depotDir
)
3540 return os
.path
.split(depotDir
)[1]
3542 def run(self
, args
):
3546 if self
.keepRepoPath
and not self
.cloneDestination
:
3547 sys
.stderr
.write("Must specify destination for --keep-path\n")
3552 if not self
.cloneDestination
and len(depotPaths
) > 1:
3553 self
.cloneDestination
= depotPaths
[-1]
3554 depotPaths
= depotPaths
[:-1]
3556 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3557 for p
in depotPaths
:
3558 if not p
.startswith("//"):
3559 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3562 if not self
.cloneDestination
:
3563 self
.cloneDestination
= self
.defaultDestination(args
)
3565 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3567 if not os
.path
.exists(self
.cloneDestination
):
3568 os
.makedirs(self
.cloneDestination
)
3569 chdir(self
.cloneDestination
)
3571 init_cmd
= [ "git", "init" ]
3573 init_cmd
.append("--bare")
3574 retcode
= subprocess
.call(init_cmd
)
3576 raise CalledProcessError(retcode
, init_cmd
)
3578 if not P4Sync
.run(self
, depotPaths
):
3581 # create a master branch and check out a work tree
3582 if gitBranchExists(self
.branch
):
3583 system([ "git", "branch", "master", self
.branch
])
3584 if not self
.cloneBare
:
3585 system([ "git", "checkout", "-f" ])
3587 print 'Not checking out any branch, use ' \
3588 '"git checkout -q -b master <branch>"'
3590 # auto-set this variable if invoked with --use-client-spec
3591 if self
.useClientSpec_from_options
:
3592 system("git config --bool git-p4.useclientspec true")
3596 class P4Branches(Command
):
3598 Command
.__init
__(self
)
3600 self
.description
= ("Shows the git branches that hold imports and their "
3601 + "corresponding perforce depot paths")
3602 self
.verbose
= False
3604 def run(self
, args
):
3605 if originP4BranchesExist():
3606 createOrUpdateBranchesFromOrigin()
3608 cmdline
= "git rev-parse --symbolic "
3609 cmdline
+= " --remotes"
3611 for line
in read_pipe_lines(cmdline
):
3614 if not line
.startswith('p4/') or line
== "p4/HEAD":
3618 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3619 settings
= extractSettingsGitLog(log
)
3621 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3624 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3626 optparse
.IndentedHelpFormatter
.__init
__(self
)
3628 def format_description(self
, description
):
3630 return description
+ "\n"
3634 def printUsage(commands
):
3635 print "usage: %s <command> [options]" % sys
.argv
[0]
3637 print "valid commands: %s" % ", ".join(commands
)
3639 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3644 "submit" : P4Submit
,
3645 "commit" : P4Submit
,
3647 "rebase" : P4Rebase
,
3649 "rollback" : P4RollBack
,
3650 "branches" : P4Branches
3655 if len(sys
.argv
[1:]) == 0:
3656 printUsage(commands
.keys())
3659 cmdName
= sys
.argv
[1]
3661 klass
= commands
[cmdName
]
3664 print "unknown command %s" % cmdName
3666 printUsage(commands
.keys())
3669 options
= cmd
.options
3670 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3674 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3676 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3678 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3680 description
= cmd
.description
,
3681 formatter
= HelpFormatter())
3683 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3685 verbose
= cmd
.verbose
3687 if cmd
.gitdir
== None:
3688 cmd
.gitdir
= os
.path
.abspath(".git")
3689 if not isValidGitDir(cmd
.gitdir
):
3690 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3691 if os
.path
.exists(cmd
.gitdir
):
3692 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3696 if not isValidGitDir(cmd
.gitdir
):
3697 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3698 cmd
.gitdir
+= "/.git"
3700 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3702 os
.environ
["GIT_DIR"] = cmd
.gitdir
3704 if not cmd
.run(args
):
3709 if __name__
== '__main__':