3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 if sys
.hexversion
< 0x02040000:
12 # The limiter is the subprocess module
13 sys
.stderr
.write("git-p4: requires Python 2.4 or later.\n")
29 from subprocess
import CalledProcessError
31 # from python2.7:subprocess.py
32 # Exception classes used by this module.
33 class CalledProcessError(Exception):
34 """This exception is raised when a process run by check_call() returns
35 a non-zero exit status. The exit status will be stored in the
36 returncode attribute."""
37 def __init__(self
, returncode
, cmd
):
38 self
.returncode
= returncode
41 return "Command '%s' returned non-zero exit status %d" % (self
.cmd
, self
.returncode
)
45 # Only labels/tags matching this will be imported/exported
46 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
48 # Grab changes in blocks of this many revisions, unless otherwise requested
49 defaultBlockSize
= 512
51 def p4_build_cmd(cmd
):
52 """Build a suitable p4 command line.
54 This consolidates building and returning a p4 command line into one
55 location. It means that hooking into the environment, or other configuration
56 can be done more easily.
60 user
= gitConfig("git-p4.user")
62 real_cmd
+= ["-u",user
]
64 password
= gitConfig("git-p4.password")
66 real_cmd
+= ["-P", password
]
68 port
= gitConfig("git-p4.port")
70 real_cmd
+= ["-p", port
]
72 host
= gitConfig("git-p4.host")
74 real_cmd
+= ["-H", host
]
76 client
= gitConfig("git-p4.client")
78 real_cmd
+= ["-c", client
]
81 if isinstance(cmd
,basestring
):
82 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
87 def chdir(path
, is_client_path
=False):
88 """Do chdir to the given path, and set the PWD environment
89 variable for use by P4. It does not look at getcwd() output.
90 Since we're not using the shell, it is necessary to set the
91 PWD environment variable explicitly.
93 Normally, expand the path to force it to be absolute. This
94 addresses the use of relative path names inside P4 settings,
95 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
96 as given; it looks for .p4config using PWD.
98 If is_client_path, the path was handed to us directly by p4,
99 and may be a symbolic link. Do not call os.getcwd() in this
100 case, because it will cause p4 to think that PWD is not inside
105 if not is_client_path
:
107 os
.environ
['PWD'] = path
110 """Return free space in bytes on the disk of the given dirname."""
111 if platform
.system() == 'Windows':
112 free_bytes
= ctypes
.c_ulonglong(0)
113 ctypes
.windll
.kernel32
.GetDiskFreeSpaceExW(ctypes
.c_wchar_p(os
.getcwd()), None, None, ctypes
.pointer(free_bytes
))
114 return free_bytes
.value
116 st
= os
.statvfs(os
.getcwd())
117 return st
.f_bavail
* st
.f_frsize
123 sys
.stderr
.write(msg
+ "\n")
126 def write_pipe(c
, stdin
):
128 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
130 expand
= isinstance(c
,basestring
)
131 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
133 val
= pipe
.write(stdin
)
136 die('Command failed: %s' % str(c
))
140 def p4_write_pipe(c
, stdin
):
141 real_cmd
= p4_build_cmd(c
)
142 return write_pipe(real_cmd
, stdin
)
144 def read_pipe(c
, ignore_error
=False):
146 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
148 expand
= isinstance(c
,basestring
)
149 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
152 if p
.wait() and not ignore_error
:
153 die('Command failed: %s' % str(c
))
157 def p4_read_pipe(c
, ignore_error
=False):
158 real_cmd
= p4_build_cmd(c
)
159 return read_pipe(real_cmd
, ignore_error
)
161 def read_pipe_lines(c
):
163 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
165 expand
= isinstance(c
, basestring
)
166 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
168 val
= pipe
.readlines()
169 if pipe
.close() or p
.wait():
170 die('Command failed: %s' % str(c
))
174 def p4_read_pipe_lines(c
):
175 """Specifically invoke p4 on the command supplied. """
176 real_cmd
= p4_build_cmd(c
)
177 return read_pipe_lines(real_cmd
)
179 def p4_has_command(cmd
):
180 """Ask p4 for help on this command. If it returns an error, the
181 command does not exist in this version of p4."""
182 real_cmd
= p4_build_cmd(["help", cmd
])
183 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
184 stderr
=subprocess
.PIPE
)
186 return p
.returncode
== 0
188 def p4_has_move_command():
189 """See if the move command exists, that it supports -k, and that
190 it has not been administratively disabled. The arguments
191 must be correct, but the filenames do not have to exist. Use
192 ones with wildcards so even if they exist, it will fail."""
194 if not p4_has_command("move"):
196 cmd
= p4_build_cmd(["move", "-k", "@from", "@to"])
197 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
198 (out
, err
) = p
.communicate()
199 # return code will be 1 in either case
200 if err
.find("Invalid option") >= 0:
202 if err
.find("disabled") >= 0:
204 # assume it failed because @... was invalid changelist
208 expand
= isinstance(cmd
,basestring
)
210 sys
.stderr
.write("executing %s\n" % str(cmd
))
211 retcode
= subprocess
.call(cmd
, shell
=expand
)
213 raise CalledProcessError(retcode
, cmd
)
216 """Specifically invoke p4 as the system command. """
217 real_cmd
= p4_build_cmd(cmd
)
218 expand
= isinstance(real_cmd
, basestring
)
219 retcode
= subprocess
.call(real_cmd
, shell
=expand
)
221 raise CalledProcessError(retcode
, real_cmd
)
223 _p4_version_string
= None
224 def p4_version_string():
225 """Read the version string, showing just the last line, which
226 hopefully is the interesting version bit.
229 Perforce - The Fast Software Configuration Management System.
230 Copyright 1995-2011 Perforce Software. All rights reserved.
231 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
233 global _p4_version_string
234 if not _p4_version_string
:
235 a
= p4_read_pipe_lines(["-V"])
236 _p4_version_string
= a
[-1].rstrip()
237 return _p4_version_string
239 def p4_integrate(src
, dest
):
240 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
242 def p4_sync(f
, *options
):
243 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
246 # forcibly add file names with wildcards
247 if wildcard_present(f
):
248 p4_system(["add", "-f", f
])
250 p4_system(["add", f
])
253 p4_system(["delete", wildcard_encode(f
)])
256 p4_system(["edit", wildcard_encode(f
)])
259 p4_system(["revert", wildcard_encode(f
)])
261 def p4_reopen(type, f
):
262 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
264 def p4_move(src
, dest
):
265 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
267 def p4_last_change():
268 results
= p4CmdList(["changes", "-m", "1"])
269 return int(results
[0]['change'])
271 def p4_describe(change
):
272 """Make sure it returns a valid result by checking for
273 the presence of field "time". Return a dict of the
276 ds
= p4CmdList(["describe", "-s", str(change
)])
278 die("p4 describe -s %d did not return 1 result: %s" % (change
, str(ds
)))
282 if "p4ExitCode" in d
:
283 die("p4 describe -s %d exited with %d: %s" % (change
, d
["p4ExitCode"],
286 if d
["code"] == "error":
287 die("p4 describe -s %d returned error code: %s" % (change
, str(d
)))
290 die("p4 describe -s %d returned no \"time\": %s" % (change
, str(d
)))
295 # Canonicalize the p4 type and return a tuple of the
296 # base type, plus any modifiers. See "p4 help filetypes"
297 # for a list and explanation.
299 def split_p4_type(p4type
):
301 p4_filetypes_historical
= {
302 "ctempobj": "binary+Sw",
308 "tempobj": "binary+FSw",
309 "ubinary": "binary+F",
310 "uresource": "resource+F",
311 "uxbinary": "binary+Fx",
312 "xbinary": "binary+x",
314 "xtempobj": "binary+Swx",
316 "xunicode": "unicode+x",
319 if p4type
in p4_filetypes_historical
:
320 p4type
= p4_filetypes_historical
[p4type
]
322 s
= p4type
.split("+")
330 # return the raw p4 type of a file (text, text+ko, etc)
333 results
= p4CmdList(["fstat", "-T", "headType", wildcard_encode(f
)])
334 return results
[0]['headType']
337 # Given a type base and modifier, return a regexp matching
338 # the keywords that can be expanded in the file
340 def p4_keywords_regexp_for_type(base
, type_mods
):
341 if base
in ("text", "unicode", "binary"):
343 if "ko" in type_mods
:
345 elif "k" in type_mods
:
346 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
350 \$ # Starts with a dollar, followed by...
351 (%s) # one of the keywords, followed by...
352 (:[^$\n]+)? # possibly an old expansion, followed by...
360 # Given a file, return a regexp matching the possible
361 # RCS keywords that will be expanded, or None for files
362 # with kw expansion turned off.
364 def p4_keywords_regexp_for_file(file):
365 if not os
.path
.exists(file):
368 (type_base
, type_mods
) = split_p4_type(p4_type(file))
369 return p4_keywords_regexp_for_type(type_base
, type_mods
)
371 def setP4ExecBit(file, mode
):
372 # Reopens an already open file and changes the execute bit to match
373 # the execute bit setting in the passed in mode.
377 if not isModeExec(mode
):
378 p4Type
= getP4OpenedType(file)
379 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
380 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
381 if p4Type
[-1] == "+":
382 p4Type
= p4Type
[0:-1]
384 p4_reopen(p4Type
, file)
386 def getP4OpenedType(file):
387 # Returns the perforce file type for the given file.
389 result
= p4_read_pipe(["opened", wildcard_encode(file)])
390 match
= re
.match(".*\((.+)\)( \*exclusive\*)?\r?$", result
)
392 return match
.group(1)
394 die("Could not determine file type for %s (result: '%s')" % (file, result
))
396 # Return the set of all p4 labels
397 def getP4Labels(depotPaths
):
399 if isinstance(depotPaths
,basestring
):
400 depotPaths
= [depotPaths
]
402 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
408 # Return the set of all git tags
411 for line
in read_pipe_lines(["git", "tag"]):
416 def diffTreePattern():
417 # This is a simple generator for the diff tree regex pattern. This could be
418 # a class variable if this and parseDiffTreeEntry were a part of a class.
419 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
423 def parseDiffTreeEntry(entry
):
424 """Parses a single diff tree entry into its component elements.
426 See git-diff-tree(1) manpage for details about the format of the diff
427 output. This method returns a dictionary with the following elements:
429 src_mode - The mode of the source file
430 dst_mode - The mode of the destination file
431 src_sha1 - The sha1 for the source file
432 dst_sha1 - The sha1 fr the destination file
433 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
434 status_score - The score for the status (applicable for 'C' and 'R'
435 statuses). This is None if there is no score.
436 src - The path for the source file.
437 dst - The path for the destination file. This is only present for
438 copy or renames. If it is not present, this is None.
440 If the pattern is not matched, None is returned."""
442 match
= diffTreePattern().next().match(entry
)
445 'src_mode': match
.group(1),
446 'dst_mode': match
.group(2),
447 'src_sha1': match
.group(3),
448 'dst_sha1': match
.group(4),
449 'status': match
.group(5),
450 'status_score': match
.group(6),
451 'src': match
.group(7),
452 'dst': match
.group(10)
456 def isModeExec(mode
):
457 # Returns True if the given git mode represents an executable file,
459 return mode
[-3:] == "755"
461 def isModeExecChanged(src_mode
, dst_mode
):
462 return isModeExec(src_mode
) != isModeExec(dst_mode
)
464 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
466 if isinstance(cmd
,basestring
):
473 cmd
= p4_build_cmd(cmd
)
475 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
477 # Use a temporary file to avoid deadlocks without
478 # subprocess.communicate(), which would put another copy
479 # of stdout into memory.
481 if stdin
is not None:
482 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
483 if isinstance(stdin
,basestring
):
484 stdin_file
.write(stdin
)
487 stdin_file
.write(i
+ '\n')
491 p4
= subprocess
.Popen(cmd
,
494 stdout
=subprocess
.PIPE
)
499 entry
= marshal
.load(p4
.stdout
)
509 entry
["p4ExitCode"] = exitCode
515 list = p4CmdList(cmd
)
521 def p4Where(depotPath
):
522 if not depotPath
.endswith("/"):
524 depotPathLong
= depotPath
+ "..."
525 outputList
= p4CmdList(["where", depotPathLong
])
527 for entry
in outputList
:
528 if "depotFile" in entry
:
529 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
530 # The base path always ends with "/...".
531 if entry
["depotFile"].find(depotPath
) == 0 and entry
["depotFile"][-4:] == "/...":
534 elif "data" in entry
:
535 data
= entry
.get("data")
536 space
= data
.find(" ")
537 if data
[:space
] == depotPath
:
542 if output
["code"] == "error":
546 clientPath
= output
.get("path")
547 elif "data" in output
:
548 data
= output
.get("data")
549 lastSpace
= data
.rfind(" ")
550 clientPath
= data
[lastSpace
+ 1:]
552 if clientPath
.endswith("..."):
553 clientPath
= clientPath
[:-3]
556 def currentGitBranch():
557 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
559 def isValidGitDir(path
):
560 if (os
.path
.exists(path
+ "/HEAD")
561 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
565 def parseRevision(ref
):
566 return read_pipe("git rev-parse %s" % ref
).strip()
568 def branchExists(ref
):
569 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
573 def extractLogMessageFromGitCommit(commit
):
576 ## fixme: title is first line of commit, not 1st paragraph.
578 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
587 def extractSettingsGitLog(log
):
589 for line
in log
.split("\n"):
591 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
595 assignments
= m
.group(1).split (':')
596 for a
in assignments
:
598 key
= vals
[0].strip()
599 val
= ('='.join (vals
[1:])).strip()
600 if val
.endswith ('\"') and val
.startswith('"'):
605 paths
= values
.get("depot-paths")
607 paths
= values
.get("depot-path")
609 values
['depot-paths'] = paths
.split(',')
612 def gitBranchExists(branch
):
613 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
614 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
615 return proc
.wait() == 0;
619 def gitConfig(key
, typeSpecifier
=None):
620 if not _gitConfig
.has_key(key
):
621 cmd
= [ "git", "config" ]
623 cmd
+= [ typeSpecifier
]
625 s
= read_pipe(cmd
, ignore_error
=True)
626 _gitConfig
[key
] = s
.strip()
627 return _gitConfig
[key
]
629 def gitConfigBool(key
):
630 """Return a bool, using git config --bool. It is True only if the
631 variable is set to true, and False if set to false or not present
634 if not _gitConfig
.has_key(key
):
635 _gitConfig
[key
] = gitConfig(key
, '--bool') == "true"
636 return _gitConfig
[key
]
638 def gitConfigInt(key
):
639 if not _gitConfig
.has_key(key
):
640 cmd
= [ "git", "config", "--int", key
]
641 s
= read_pipe(cmd
, ignore_error
=True)
644 _gitConfig
[key
] = int(gitConfig(key
, '--int'))
646 _gitConfig
[key
] = None
647 return _gitConfig
[key
]
649 def gitConfigList(key
):
650 if not _gitConfig
.has_key(key
):
651 s
= read_pipe(["git", "config", "--get-all", key
], ignore_error
=True)
652 _gitConfig
[key
] = s
.strip().split(os
.linesep
)
653 if _gitConfig
[key
] == ['']:
655 return _gitConfig
[key
]
657 def p4BranchesInGit(branchesAreInRemotes
=True):
658 """Find all the branches whose names start with "p4/", looking
659 in remotes or heads as specified by the argument. Return
660 a dictionary of { branch: revision } for each one found.
661 The branch names are the short names, without any
666 cmdline
= "git rev-parse --symbolic "
667 if branchesAreInRemotes
:
668 cmdline
+= "--remotes"
670 cmdline
+= "--branches"
672 for line
in read_pipe_lines(cmdline
):
676 if not line
.startswith('p4/'):
678 # special symbolic ref to p4/master
679 if line
== "p4/HEAD":
682 # strip off p4/ prefix
683 branch
= line
[len("p4/"):]
685 branches
[branch
] = parseRevision(line
)
689 def branch_exists(branch
):
690 """Make sure that the given ref name really exists."""
692 cmd
= [ "git", "rev-parse", "--symbolic", "--verify", branch
]
693 p
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
694 out
, _
= p
.communicate()
697 # expect exactly one line of output: the branch name
698 return out
.rstrip() == branch
700 def findUpstreamBranchPoint(head
= "HEAD"):
701 branches
= p4BranchesInGit()
702 # map from depot-path to branch name
703 branchByDepotPath
= {}
704 for branch
in branches
.keys():
705 tip
= branches
[branch
]
706 log
= extractLogMessageFromGitCommit(tip
)
707 settings
= extractSettingsGitLog(log
)
708 if settings
.has_key("depot-paths"):
709 paths
= ",".join(settings
["depot-paths"])
710 branchByDepotPath
[paths
] = "remotes/p4/" + branch
714 while parent
< 65535:
715 commit
= head
+ "~%s" % parent
716 log
= extractLogMessageFromGitCommit(commit
)
717 settings
= extractSettingsGitLog(log
)
718 if settings
.has_key("depot-paths"):
719 paths
= ",".join(settings
["depot-paths"])
720 if branchByDepotPath
.has_key(paths
):
721 return [branchByDepotPath
[paths
], settings
]
725 return ["", settings
]
727 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
729 print ("Creating/updating branch(es) in %s based on origin branch(es)"
732 originPrefix
= "origin/p4/"
734 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
736 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
739 headName
= line
[len(originPrefix
):]
740 remoteHead
= localRefPrefix
+ headName
743 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
744 if (not original
.has_key('depot-paths')
745 or not original
.has_key('change')):
749 if not gitBranchExists(remoteHead
):
751 print "creating %s" % remoteHead
754 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
755 if settings
.has_key('change') > 0:
756 if settings
['depot-paths'] == original
['depot-paths']:
757 originP4Change
= int(original
['change'])
758 p4Change
= int(settings
['change'])
759 if originP4Change
> p4Change
:
760 print ("%s (%s) is newer than %s (%s). "
761 "Updating p4 branch from origin."
762 % (originHead
, originP4Change
,
763 remoteHead
, p4Change
))
766 print ("Ignoring: %s was imported from %s while "
767 "%s was imported from %s"
768 % (originHead
, ','.join(original
['depot-paths']),
769 remoteHead
, ','.join(settings
['depot-paths'])))
772 system("git update-ref %s %s" % (remoteHead
, originHead
))
774 def originP4BranchesExist():
775 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
778 def p4ParseNumericChangeRange(parts
):
779 changeStart
= int(parts
[0][1:])
780 if parts
[1] == '#head':
781 changeEnd
= p4_last_change()
783 changeEnd
= int(parts
[1])
785 return (changeStart
, changeEnd
)
787 def chooseBlockSize(blockSize
):
791 return defaultBlockSize
793 def p4ChangesForPaths(depotPaths
, changeRange
, requestedBlockSize
):
796 # Parse the change range into start and end. Try to find integer
797 # revision ranges as these can be broken up into blocks to avoid
798 # hitting server-side limits (maxrows, maxscanresults). But if
799 # that doesn't work, fall back to using the raw revision specifier
800 # strings, without using block mode.
802 if changeRange
is None or changeRange
== '':
804 changeEnd
= p4_last_change()
805 block_size
= chooseBlockSize(requestedBlockSize
)
807 parts
= changeRange
.split(',')
808 assert len(parts
) == 2
810 (changeStart
, changeEnd
) = p4ParseNumericChangeRange(parts
)
811 block_size
= chooseBlockSize(requestedBlockSize
)
813 changeStart
= parts
[0][1:]
815 if requestedBlockSize
:
816 die("cannot use --changes-block-size with non-numeric revisions")
819 # Accumulate change numbers in a dictionary to avoid duplicates
823 # Retrieve changes a block at a time, to prevent running
824 # into a MaxResults/MaxScanRows error from the server.
830 end
= min(changeEnd
, changeStart
+ block_size
)
831 revisionRange
= "%d,%d" % (changeStart
, end
)
833 revisionRange
= "%s,%s" % (changeStart
, changeEnd
)
835 cmd
+= ["%s...@%s" % (p
, revisionRange
)]
837 for line
in p4_read_pipe_lines(cmd
):
838 changeNum
= int(line
.split(" ")[1])
839 changes
[changeNum
] = True
847 changeStart
= end
+ 1
849 changelist
= changes
.keys()
853 def p4PathStartsWith(path
, prefix
):
854 # This method tries to remedy a potential mixed-case issue:
856 # If UserA adds //depot/DirA/file1
857 # and UserB adds //depot/dira/file2
859 # we may or may not have a problem. If you have core.ignorecase=true,
860 # we treat DirA and dira as the same directory
861 if gitConfigBool("core.ignorecase"):
862 return path
.lower().startswith(prefix
.lower())
863 return path
.startswith(prefix
)
866 """Look at the p4 client spec, create a View() object that contains
867 all the mappings, and return it."""
869 specList
= p4CmdList("client -o")
870 if len(specList
) != 1:
871 die('Output from "client -o" is %d lines, expecting 1' %
874 # dictionary of all client parameters
878 client_name
= entry
["Client"]
880 # just the keys that start with "View"
881 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
884 view
= View(client_name
)
886 # append the lines, in order, to the view
887 for view_num
in range(len(view_keys
)):
888 k
= "View%d" % view_num
889 if k
not in view_keys
:
890 die("Expected view key %s missing" % k
)
891 view
.append(entry
[k
])
896 """Grab the client directory."""
898 output
= p4CmdList("client -o")
900 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
903 if "Root" not in entry
:
904 die('Client has no "Root"')
909 # P4 wildcards are not allowed in filenames. P4 complains
910 # if you simply add them, but you can force it with "-f", in
911 # which case it translates them into %xx encoding internally.
913 def wildcard_decode(path
):
914 # Search for and fix just these four characters. Do % last so
915 # that fixing it does not inadvertently create new %-escapes.
916 # Cannot have * in a filename in windows; untested as to
917 # what p4 would do in such a case.
918 if not platform
.system() == "Windows":
919 path
= path
.replace("%2A", "*")
920 path
= path
.replace("%23", "#") \
921 .replace("%40", "@") \
925 def wildcard_encode(path
):
926 # do % first to avoid double-encoding the %s introduced here
927 path
= path
.replace("%", "%25") \
928 .replace("*", "%2A") \
929 .replace("#", "%23") \
933 def wildcard_present(path
):
934 m
= re
.search("[*#@%]", path
)
937 class LargeFileSystem(object):
938 """Base class for large file system support."""
940 def __init__(self
, writeToGitStream
):
941 self
.largeFiles
= set()
942 self
.writeToGitStream
= writeToGitStream
944 def generatePointer(self
, cloneDestination
, contentFile
):
945 """Return the content of a pointer file that is stored in Git instead of
946 the actual content."""
947 assert False, "Method 'generatePointer' required in " + self
.__class
__.__name
__
949 def pushFile(self
, localLargeFile
):
950 """Push the actual content which is not stored in the Git repository to
952 assert False, "Method 'pushFile' required in " + self
.__class
__.__name
__
954 def hasLargeFileExtension(self
, relPath
):
957 [relPath
.endswith('.' + e
) for e
in gitConfigList('git-p4.largeFileExtensions')],
961 def generateTempFile(self
, contents
):
962 contentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
966 return contentFile
.name
968 def exceedsLargeFileThreshold(self
, relPath
, contents
):
969 if gitConfigInt('git-p4.largeFileThreshold'):
970 contentsSize
= sum(len(d
) for d
in contents
)
971 if contentsSize
> gitConfigInt('git-p4.largeFileThreshold'):
973 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
974 contentsSize
= sum(len(d
) for d
in contents
)
975 if contentsSize
<= gitConfigInt('git-p4.largeFileCompressedThreshold'):
977 contentTempFile
= self
.generateTempFile(contents
)
978 compressedContentFile
= tempfile
.NamedTemporaryFile(prefix
='git-p4-large-file', delete
=False)
979 zf
= zipfile
.ZipFile(compressedContentFile
.name
, mode
='w')
980 zf
.write(contentTempFile
, compress_type
=zipfile
.ZIP_DEFLATED
)
982 compressedContentsSize
= zf
.infolist()[0].compress_size
983 os
.remove(contentTempFile
)
984 os
.remove(compressedContentFile
.name
)
985 if compressedContentsSize
> gitConfigInt('git-p4.largeFileCompressedThreshold'):
989 def addLargeFile(self
, relPath
):
990 self
.largeFiles
.add(relPath
)
992 def removeLargeFile(self
, relPath
):
993 self
.largeFiles
.remove(relPath
)
995 def isLargeFile(self
, relPath
):
996 return relPath
in self
.largeFiles
998 def processContent(self
, git_mode
, relPath
, contents
):
999 """Processes the content of git fast import. This method decides if a
1000 file is stored in the large file system and handles all necessary
1002 if self
.exceedsLargeFileThreshold(relPath
, contents
) or self
.hasLargeFileExtension(relPath
):
1003 contentTempFile
= self
.generateTempFile(contents
)
1004 (git_mode
, contents
, localLargeFile
) = self
.generatePointer(contentTempFile
)
1006 # Move temp file to final location in large file system
1007 largeFileDir
= os
.path
.dirname(localLargeFile
)
1008 if not os
.path
.isdir(largeFileDir
):
1009 os
.makedirs(largeFileDir
)
1010 shutil
.move(contentTempFile
, localLargeFile
)
1011 self
.addLargeFile(relPath
)
1012 if gitConfigBool('git-p4.largeFilePush'):
1013 self
.pushFile(localLargeFile
)
1015 sys
.stderr
.write("%s moved to large file system (%s)\n" % (relPath
, localLargeFile
))
1016 return (git_mode
, contents
)
1018 class MockLFS(LargeFileSystem
):
1019 """Mock large file system for testing."""
1021 def generatePointer(self
, contentFile
):
1022 """The pointer content is the original content prefixed with "pointer-".
1023 The local filename of the large file storage is derived from the file content.
1025 with
open(contentFile
, 'r') as f
:
1028 pointerContents
= 'pointer-' + content
1029 localLargeFile
= os
.path
.join(os
.getcwd(), '.git', 'mock-storage', 'local', content
[:-1])
1030 return (gitMode
, pointerContents
, localLargeFile
)
1032 def pushFile(self
, localLargeFile
):
1033 """The remote filename of the large file storage is the same as the local
1034 one but in a different directory.
1036 remotePath
= os
.path
.join(os
.path
.dirname(localLargeFile
), '..', 'remote')
1037 if not os
.path
.exists(remotePath
):
1038 os
.makedirs(remotePath
)
1039 shutil
.copyfile(localLargeFile
, os
.path
.join(remotePath
, os
.path
.basename(localLargeFile
)))
1041 class GitLFS(LargeFileSystem
):
1042 """Git LFS as backend for the git-p4 large file system.
1043 See https://git-lfs.github.com/ for details."""
1045 def __init__(self
, *args
):
1046 LargeFileSystem
.__init
__(self
, *args
)
1047 self
.baseGitAttributes
= []
1049 def generatePointer(self
, contentFile
):
1050 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1051 mode and content which is stored in the Git repository instead of
1052 the actual content. Return also the new location of the actual
1055 pointerProcess
= subprocess
.Popen(
1056 ['git', 'lfs', 'pointer', '--file=' + contentFile
],
1057 stdout
=subprocess
.PIPE
1059 pointerFile
= pointerProcess
.stdout
.read()
1060 if pointerProcess
.wait():
1061 os
.remove(contentFile
)
1062 die('git-lfs pointer command failed. Did you install the extension?')
1063 pointerContents
= [i
+'\n' for i
in pointerFile
.split('\n')[2:][:-1]]
1064 oid
= pointerContents
[1].split(' ')[1].split(':')[1][:-1]
1065 localLargeFile
= os
.path
.join(
1067 '.git', 'lfs', 'objects', oid
[:2], oid
[2:4],
1070 # LFS Spec states that pointer files should not have the executable bit set.
1072 return (gitMode
, pointerContents
, localLargeFile
)
1074 def pushFile(self
, localLargeFile
):
1075 uploadProcess
= subprocess
.Popen(
1076 ['git', 'lfs', 'push', '--object-id', 'origin', os
.path
.basename(localLargeFile
)]
1078 if uploadProcess
.wait():
1079 die('git-lfs push command failed. Did you define a remote?')
1081 def generateGitAttributes(self
):
1083 self
.baseGitAttributes
+
1087 '# Git LFS (see https://git-lfs.github.com/)\n',
1090 ['*.' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1091 for f
in sorted(gitConfigList('git-p4.largeFileExtensions'))
1093 ['/' + f
.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1094 for f
in sorted(self
.largeFiles
) if not self
.hasLargeFileExtension(f
)
1098 def addLargeFile(self
, relPath
):
1099 LargeFileSystem
.addLargeFile(self
, relPath
)
1100 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1102 def removeLargeFile(self
, relPath
):
1103 LargeFileSystem
.removeLargeFile(self
, relPath
)
1104 self
.writeToGitStream('100644', '.gitattributes', self
.generateGitAttributes())
1106 def processContent(self
, git_mode
, relPath
, contents
):
1107 if relPath
== '.gitattributes':
1108 self
.baseGitAttributes
= contents
1109 return (git_mode
, self
.generateGitAttributes())
1111 return LargeFileSystem
.processContent(self
, git_mode
, relPath
, contents
)
1115 self
.usage
= "usage: %prog [options]"
1116 self
.needsGit
= True
1117 self
.verbose
= False
1121 self
.userMapFromPerforceServer
= False
1122 self
.myP4UserId
= None
1126 return self
.myP4UserId
1128 results
= p4CmdList("user -o")
1130 if r
.has_key('User'):
1131 self
.myP4UserId
= r
['User']
1133 die("Could not find your p4 user id")
1135 def p4UserIsMe(self
, p4User
):
1136 # return True if the given p4 user is actually me
1137 me
= self
.p4UserId()
1138 if not p4User
or p4User
!= me
:
1143 def getUserCacheFilename(self
):
1144 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
1145 return home
+ "/.gitp4-usercache.txt"
1147 def getUserMapFromPerforceServer(self
):
1148 if self
.userMapFromPerforceServer
:
1153 for output
in p4CmdList("users"):
1154 if not output
.has_key("User"):
1156 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
1157 self
.emails
[output
["Email"]] = output
["User"]
1161 for (key
, val
) in self
.users
.items():
1162 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
1164 open(self
.getUserCacheFilename(), "wb").write(s
)
1165 self
.userMapFromPerforceServer
= True
1167 def loadUserMapFromCache(self
):
1169 self
.userMapFromPerforceServer
= False
1171 cache
= open(self
.getUserCacheFilename(), "rb")
1172 lines
= cache
.readlines()
1175 entry
= line
.strip().split("\t")
1176 self
.users
[entry
[0]] = entry
[1]
1178 self
.getUserMapFromPerforceServer()
1180 class P4Debug(Command
):
1182 Command
.__init
__(self
)
1184 self
.description
= "A tool to debug the output of p4 -G."
1185 self
.needsGit
= False
1187 def run(self
, args
):
1189 for output
in p4CmdList(args
):
1190 print 'Element: %d' % j
1195 class P4RollBack(Command
):
1197 Command
.__init
__(self
)
1199 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
1201 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
1202 self
.rollbackLocalBranches
= False
1204 def run(self
, args
):
1207 maxChange
= int(args
[0])
1209 if "p4ExitCode" in p4Cmd("changes -m 1"):
1210 die("Problems executing p4");
1212 if self
.rollbackLocalBranches
:
1213 refPrefix
= "refs/heads/"
1214 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
1216 refPrefix
= "refs/remotes/"
1217 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
1220 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
1222 ref
= refPrefix
+ line
1223 log
= extractLogMessageFromGitCommit(ref
)
1224 settings
= extractSettingsGitLog(log
)
1226 depotPaths
= settings
['depot-paths']
1227 change
= settings
['change']
1231 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
1232 for p
in depotPaths
]))) == 0:
1233 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
1234 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
1237 while change
and int(change
) > maxChange
:
1240 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
1241 system("git update-ref %s \"%s^\"" % (ref
, ref
))
1242 log
= extractLogMessageFromGitCommit(ref
)
1243 settings
= extractSettingsGitLog(log
)
1246 depotPaths
= settings
['depot-paths']
1247 change
= settings
['change']
1250 print "%s rewound to %s" % (ref
, change
)
1254 class P4Submit(Command
, P4UserMap
):
1256 conflict_behavior_choices
= ("ask", "skip", "quit")
1259 Command
.__init
__(self
)
1260 P4UserMap
.__init
__(self
)
1262 optparse
.make_option("--origin", dest
="origin"),
1263 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
1264 # preserve the user, requires relevant p4 permissions
1265 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
1266 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
1267 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
1268 optparse
.make_option("--prepare-p4-only", dest
="prepare_p4_only", action
="store_true"),
1269 optparse
.make_option("--conflict", dest
="conflict_behavior",
1270 choices
=self
.conflict_behavior_choices
),
1271 optparse
.make_option("--branch", dest
="branch"),
1273 self
.description
= "Submit changes from git to the perforce depot."
1274 self
.usage
+= " [name of git branch to submit into perforce depot]"
1276 self
.detectRenames
= False
1277 self
.preserveUser
= gitConfigBool("git-p4.preserveUser")
1278 self
.dry_run
= False
1279 self
.prepare_p4_only
= False
1280 self
.conflict_behavior
= None
1281 self
.isWindows
= (platform
.system() == "Windows")
1282 self
.exportLabels
= False
1283 self
.p4HasMoveCommand
= p4_has_move_command()
1286 if gitConfig('git-p4.largeFileSystem'):
1287 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1290 if len(p4CmdList("opened ...")) > 0:
1291 die("You have files opened with perforce! Close them before starting the sync.")
1293 def separate_jobs_from_description(self
, message
):
1294 """Extract and return a possible Jobs field in the commit
1295 message. It goes into a separate section in the p4 change
1298 A jobs line starts with "Jobs:" and looks like a new field
1299 in a form. Values are white-space separated on the same
1300 line or on following lines that start with a tab.
1302 This does not parse and extract the full git commit message
1303 like a p4 form. It just sees the Jobs: line as a marker
1304 to pass everything from then on directly into the p4 form,
1305 but outside the description section.
1307 Return a tuple (stripped log message, jobs string)."""
1309 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
1311 return (message
, None)
1313 jobtext
= message
[m
.start():]
1314 stripped_message
= message
[:m
.start()].rstrip()
1315 return (stripped_message
, jobtext
)
1317 def prepareLogMessage(self
, template
, message
, jobs
):
1318 """Edits the template returned from "p4 change -o" to insert
1319 the message in the Description field, and the jobs text in
1323 inDescriptionSection
= False
1325 for line
in template
.split("\n"):
1326 if line
.startswith("#"):
1327 result
+= line
+ "\n"
1330 if inDescriptionSection
:
1331 if line
.startswith("Files:") or line
.startswith("Jobs:"):
1332 inDescriptionSection
= False
1333 # insert Jobs section
1335 result
+= jobs
+ "\n"
1339 if line
.startswith("Description:"):
1340 inDescriptionSection
= True
1342 for messageLine
in message
.split("\n"):
1343 line
+= "\t" + messageLine
+ "\n"
1345 result
+= line
+ "\n"
1349 def patchRCSKeywords(self
, file, pattern
):
1350 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1351 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
1353 outFile
= os
.fdopen(handle
, "w+")
1354 inFile
= open(file, "r")
1355 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1356 for line
in inFile
.readlines():
1357 line
= regexp
.sub(r
'$\1$', line
)
1361 # Forcibly overwrite the original file
1363 shutil
.move(outFileName
, file)
1365 # cleanup our temporary file
1366 os
.unlink(outFileName
)
1367 print "Failed to strip RCS keywords in %s" % file
1370 print "Patched up RCS keywords in %s" % file
1372 def p4UserForCommit(self
,id):
1373 # Return the tuple (perforce user,git email) for a given git commit id
1374 self
.getUserMapFromPerforceServer()
1375 gitEmail
= read_pipe(["git", "log", "--max-count=1",
1376 "--format=%ae", id])
1377 gitEmail
= gitEmail
.strip()
1378 if not self
.emails
.has_key(gitEmail
):
1379 return (None,gitEmail
)
1381 return (self
.emails
[gitEmail
],gitEmail
)
1383 def checkValidP4Users(self
,commits
):
1384 # check if any git authors cannot be mapped to p4 users
1386 (user
,email
) = self
.p4UserForCommit(id)
1388 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
1389 if gitConfigBool("git-p4.allowMissingP4Users"):
1392 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
1394 def lastP4Changelist(self
):
1395 # Get back the last changelist number submitted in this client spec. This
1396 # then gets used to patch up the username in the change. If the same
1397 # client spec is being used by multiple processes then this might go
1399 results
= p4CmdList("client -o") # find the current client
1402 if r
.has_key('Client'):
1403 client
= r
['Client']
1406 die("could not get client spec")
1407 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
1409 if r
.has_key('change'):
1411 die("Could not get changelist number for last submit - cannot patch up user details")
1413 def modifyChangelistUser(self
, changelist
, newUser
):
1414 # fixup the user field of a changelist after it has been submitted.
1415 changes
= p4CmdList("change -o %s" % changelist
)
1416 if len(changes
) != 1:
1417 die("Bad output from p4 change modifying %s to user %s" %
1418 (changelist
, newUser
))
1421 if c
['User'] == newUser
: return # nothing to do
1423 input = marshal
.dumps(c
)
1425 result
= p4CmdList("change -f -i", stdin
=input)
1427 if r
.has_key('code'):
1428 if r
['code'] == 'error':
1429 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1430 if r
.has_key('data'):
1431 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1433 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1435 def canChangeChangelists(self
):
1436 # check to see if we have p4 admin or super-user permissions, either of
1437 # which are required to modify changelists.
1438 results
= p4CmdList(["protects", self
.depotPath
])
1440 if r
.has_key('perm'):
1441 if r
['perm'] == 'admin':
1443 if r
['perm'] == 'super':
1447 def prepareSubmitTemplate(self
):
1448 """Run "p4 change -o" to grab a change specification template.
1449 This does not use "p4 -G", as it is nice to keep the submission
1450 template in original order, since a human might edit it.
1452 Remove lines in the Files section that show changes to files
1453 outside the depot path we're committing into."""
1456 inFilesSection
= False
1457 for line
in p4_read_pipe_lines(['change', '-o']):
1458 if line
.endswith("\r\n"):
1459 line
= line
[:-2] + "\n"
1461 if line
.startswith("\t"):
1462 # path starts and ends with a tab
1464 lastTab
= path
.rfind("\t")
1466 path
= path
[:lastTab
]
1467 if not p4PathStartsWith(path
, self
.depotPath
):
1470 inFilesSection
= False
1472 if line
.startswith("Files:"):
1473 inFilesSection
= True
1479 def edit_template(self
, template_file
):
1480 """Invoke the editor to let the user change the submission
1481 message. Return true if okay to continue with the submit."""
1483 # if configured to skip the editing part, just submit
1484 if gitConfigBool("git-p4.skipSubmitEdit"):
1487 # look at the modification time, to check later if the user saved
1489 mtime
= os
.stat(template_file
).st_mtime
1492 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1493 editor
= os
.environ
.get("P4EDITOR")
1495 editor
= read_pipe("git var GIT_EDITOR").strip()
1496 system(["sh", "-c", ('%s "$@"' % editor
), editor
, template_file
])
1498 # If the file was not saved, prompt to see if this patch should
1499 # be skipped. But skip this verification step if configured so.
1500 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1503 # modification time updated means user saved the file
1504 if os
.stat(template_file
).st_mtime
> mtime
:
1508 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1514 def get_diff_description(self
, editedFiles
, filesToAdd
):
1516 if os
.environ
.has_key("P4DIFF"):
1517 del(os
.environ
["P4DIFF"])
1519 for editedFile
in editedFiles
:
1520 diff
+= p4_read_pipe(['diff', '-du',
1521 wildcard_encode(editedFile
)])
1525 for newFile
in filesToAdd
:
1526 newdiff
+= "==== new file ====\n"
1527 newdiff
+= "--- /dev/null\n"
1528 newdiff
+= "+++ %s\n" % newFile
1529 f
= open(newFile
, "r")
1530 for line
in f
.readlines():
1531 newdiff
+= "+" + line
1534 return (diff
+ newdiff
).replace('\r\n', '\n')
1536 def applyCommit(self
, id):
1537 """Apply one commit, return True if it succeeded."""
1539 print "Applying", read_pipe(["git", "show", "-s",
1540 "--format=format:%h %s", id])
1542 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1544 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1546 filesToDelete
= set()
1548 pureRenameCopy
= set()
1549 filesToChangeExecBit
= {}
1552 diff
= parseDiffTreeEntry(line
)
1553 modifier
= diff
['status']
1557 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1558 filesToChangeExecBit
[path
] = diff
['dst_mode']
1559 editedFiles
.add(path
)
1560 elif modifier
== "A":
1561 filesToAdd
.add(path
)
1562 filesToChangeExecBit
[path
] = diff
['dst_mode']
1563 if path
in filesToDelete
:
1564 filesToDelete
.remove(path
)
1565 elif modifier
== "D":
1566 filesToDelete
.add(path
)
1567 if path
in filesToAdd
:
1568 filesToAdd
.remove(path
)
1569 elif modifier
== "C":
1570 src
, dest
= diff
['src'], diff
['dst']
1571 p4_integrate(src
, dest
)
1572 pureRenameCopy
.add(dest
)
1573 if diff
['src_sha1'] != diff
['dst_sha1']:
1575 pureRenameCopy
.discard(dest
)
1576 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1578 pureRenameCopy
.discard(dest
)
1579 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1581 # turn off read-only attribute
1582 os
.chmod(dest
, stat
.S_IWRITE
)
1584 editedFiles
.add(dest
)
1585 elif modifier
== "R":
1586 src
, dest
= diff
['src'], diff
['dst']
1587 if self
.p4HasMoveCommand
:
1588 p4_edit(src
) # src must be open before move
1589 p4_move(src
, dest
) # opens for (move/delete, move/add)
1591 p4_integrate(src
, dest
)
1592 if diff
['src_sha1'] != diff
['dst_sha1']:
1595 pureRenameCopy
.add(dest
)
1596 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1597 if not self
.p4HasMoveCommand
:
1598 p4_edit(dest
) # with move: already open, writable
1599 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1600 if not self
.p4HasMoveCommand
:
1602 os
.chmod(dest
, stat
.S_IWRITE
)
1604 filesToDelete
.add(src
)
1605 editedFiles
.add(dest
)
1607 die("unknown modifier %s for %s" % (modifier
, path
))
1609 diffcmd
= "git diff-tree --full-index -p \"%s\"" % (id)
1610 patchcmd
= diffcmd
+ " | git apply "
1611 tryPatchCmd
= patchcmd
+ "--check -"
1612 applyPatchCmd
= patchcmd
+ "--check --apply -"
1613 patch_succeeded
= True
1615 if os
.system(tryPatchCmd
) != 0:
1616 fixed_rcs_keywords
= False
1617 patch_succeeded
= False
1618 print "Unfortunately applying the change failed!"
1620 # Patch failed, maybe it's just RCS keyword woes. Look through
1621 # the patch to see if that's possible.
1622 if gitConfigBool("git-p4.attemptRCSCleanup"):
1626 for file in editedFiles | filesToDelete
:
1627 # did this file's delta contain RCS keywords?
1628 pattern
= p4_keywords_regexp_for_file(file)
1631 # this file is a possibility...look for RCS keywords.
1632 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1633 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1634 if regexp
.search(line
):
1636 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1637 kwfiles
[file] = pattern
1640 for file in kwfiles
:
1642 print "zapping %s with %s" % (line
,pattern
)
1643 # File is being deleted, so not open in p4. Must
1644 # disable the read-only bit on windows.
1645 if self
.isWindows
and file not in editedFiles
:
1646 os
.chmod(file, stat
.S_IWRITE
)
1647 self
.patchRCSKeywords(file, kwfiles
[file])
1648 fixed_rcs_keywords
= True
1650 if fixed_rcs_keywords
:
1651 print "Retrying the patch with RCS keywords cleaned up"
1652 if os
.system(tryPatchCmd
) == 0:
1653 patch_succeeded
= True
1655 if not patch_succeeded
:
1656 for f
in editedFiles
:
1661 # Apply the patch for real, and do add/delete/+x handling.
1663 system(applyPatchCmd
)
1665 for f
in filesToAdd
:
1667 for f
in filesToDelete
:
1671 # Set/clear executable bits
1672 for f
in filesToChangeExecBit
.keys():
1673 mode
= filesToChangeExecBit
[f
]
1674 setP4ExecBit(f
, mode
)
1677 # Build p4 change description, starting with the contents
1678 # of the git commit message.
1680 logMessage
= extractLogMessageFromGitCommit(id)
1681 logMessage
= logMessage
.strip()
1682 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1684 template
= self
.prepareSubmitTemplate()
1685 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1687 if self
.preserveUser
:
1688 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1690 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1691 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1692 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1693 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1695 separatorLine
= "######## everything below this line is just the diff #######\n"
1696 if not self
.prepare_p4_only
:
1697 submitTemplate
+= separatorLine
1698 submitTemplate
+= self
.get_diff_description(editedFiles
, filesToAdd
)
1700 (handle
, fileName
) = tempfile
.mkstemp()
1701 tmpFile
= os
.fdopen(handle
, "w+b")
1703 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1704 tmpFile
.write(submitTemplate
)
1707 if self
.prepare_p4_only
:
1709 # Leave the p4 tree prepared, and the submit template around
1710 # and let the user decide what to do next
1713 print "P4 workspace prepared for submission."
1714 print "To submit or revert, go to client workspace"
1715 print " " + self
.clientPath
1717 print "To submit, use \"p4 submit\" to write a new description,"
1718 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1719 " \"git p4\"." % fileName
1720 print "You can delete the file \"%s\" when finished." % fileName
1722 if self
.preserveUser
and p4User
and not self
.p4UserIsMe(p4User
):
1723 print "To preserve change ownership by user %s, you must\n" \
1724 "do \"p4 change -f <change>\" after submitting and\n" \
1725 "edit the User field."
1727 print "After submitting, renamed files must be re-synced."
1728 print "Invoke \"p4 sync -f\" on each of these files:"
1729 for f
in pureRenameCopy
:
1733 print "To revert the changes, use \"p4 revert ...\", and delete"
1734 print "the submit template file \"%s\"" % fileName
1736 print "Since the commit adds new files, they must be deleted:"
1737 for f
in filesToAdd
:
1743 # Let the user edit the change description, then submit it.
1745 if self
.edit_template(fileName
):
1746 # read the edited message and submit
1748 tmpFile
= open(fileName
, "rb")
1749 message
= tmpFile
.read()
1752 message
= message
.replace("\r\n", "\n")
1753 submitTemplate
= message
[:message
.index(separatorLine
)]
1754 p4_write_pipe(['submit', '-i'], submitTemplate
)
1756 if self
.preserveUser
:
1758 # Get last changelist number. Cannot easily get it from
1759 # the submit command output as the output is
1761 changelist
= self
.lastP4Changelist()
1762 self
.modifyChangelistUser(changelist
, p4User
)
1764 # The rename/copy happened by applying a patch that created a
1765 # new file. This leaves it writable, which confuses p4.
1766 for f
in pureRenameCopy
:
1772 print "Submission cancelled, undoing p4 changes."
1773 for f
in editedFiles
:
1775 for f
in filesToAdd
:
1778 for f
in filesToDelete
:
1784 # Export git tags as p4 labels. Create a p4 label and then tag
1786 def exportGitTags(self
, gitTags
):
1787 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1788 if len(validLabelRegexp
) == 0:
1789 validLabelRegexp
= defaultLabelRegexp
1790 m
= re
.compile(validLabelRegexp
)
1792 for name
in gitTags
:
1794 if not m
.match(name
):
1796 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1799 # Get the p4 commit this corresponds to
1800 logMessage
= extractLogMessageFromGitCommit(name
)
1801 values
= extractSettingsGitLog(logMessage
)
1803 if not values
.has_key('change'):
1804 # a tag pointing to something not sent to p4; ignore
1806 print "git tag %s does not give a p4 commit" % name
1809 changelist
= values
['change']
1811 # Get the tag details.
1815 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1818 if re
.match(r
'tag\s+', l
):
1820 elif re
.match(r
'\s*$', l
):
1827 body
= ["lightweight tag imported by git p4\n"]
1829 # Create the label - use the same view as the client spec we are using
1830 clientSpec
= getClientSpec()
1832 labelTemplate
= "Label: %s\n" % name
1833 labelTemplate
+= "Description:\n"
1835 labelTemplate
+= "\t" + b
+ "\n"
1836 labelTemplate
+= "View:\n"
1837 for depot_side
in clientSpec
.mappings
:
1838 labelTemplate
+= "\t%s\n" % depot_side
1841 print "Would create p4 label %s for tag" % name
1842 elif self
.prepare_p4_only
:
1843 print "Not creating p4 label %s for tag due to option" \
1844 " --prepare-p4-only" % name
1846 p4_write_pipe(["label", "-i"], labelTemplate
)
1849 p4_system(["tag", "-l", name
] +
1850 ["%s@%s" % (depot_side
, changelist
) for depot_side
in clientSpec
.mappings
])
1853 print "created p4 label for tag %s" % name
1855 def run(self
, args
):
1857 self
.master
= currentGitBranch()
1858 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1859 die("Detecting current git branch failed!")
1860 elif len(args
) == 1:
1861 self
.master
= args
[0]
1862 if not branchExists(self
.master
):
1863 die("Branch %s does not exist" % self
.master
)
1867 allowSubmit
= gitConfig("git-p4.allowSubmit")
1868 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1869 die("%s is not in git-p4.allowSubmit" % self
.master
)
1871 [upstream
, settings
] = findUpstreamBranchPoint()
1872 self
.depotPath
= settings
['depot-paths'][0]
1873 if len(self
.origin
) == 0:
1874 self
.origin
= upstream
1876 if self
.preserveUser
:
1877 if not self
.canChangeChangelists():
1878 die("Cannot preserve user names without p4 super-user or admin permissions")
1880 # if not set from the command line, try the config file
1881 if self
.conflict_behavior
is None:
1882 val
= gitConfig("git-p4.conflict")
1884 if val
not in self
.conflict_behavior_choices
:
1885 die("Invalid value '%s' for config git-p4.conflict" % val
)
1888 self
.conflict_behavior
= val
1891 print "Origin branch is " + self
.origin
1893 if len(self
.depotPath
) == 0:
1894 print "Internal error: cannot locate perforce depot path from existing branches"
1897 self
.useClientSpec
= False
1898 if gitConfigBool("git-p4.useclientspec"):
1899 self
.useClientSpec
= True
1900 if self
.useClientSpec
:
1901 self
.clientSpecDirs
= getClientSpec()
1903 # Check for the existance of P4 branches
1904 branchesDetected
= (len(p4BranchesInGit().keys()) > 1)
1906 if self
.useClientSpec
and not branchesDetected
:
1907 # all files are relative to the client spec
1908 self
.clientPath
= getClientRoot()
1910 self
.clientPath
= p4Where(self
.depotPath
)
1912 if self
.clientPath
== "":
1913 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1915 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1916 self
.oldWorkingDirectory
= os
.getcwd()
1918 # ensure the clientPath exists
1919 new_client_dir
= False
1920 if not os
.path
.exists(self
.clientPath
):
1921 new_client_dir
= True
1922 os
.makedirs(self
.clientPath
)
1924 chdir(self
.clientPath
, is_client_path
=True)
1926 print "Would synchronize p4 checkout in %s" % self
.clientPath
1928 print "Synchronizing p4 checkout..."
1930 # old one was destroyed, and maybe nobody told p4
1931 p4_sync("...", "-f")
1937 for line
in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self
.origin
, self
.master
)]):
1938 commits
.append(line
.strip())
1941 if self
.preserveUser
or gitConfigBool("git-p4.skipUserNameCheck"):
1942 self
.checkAuthorship
= False
1944 self
.checkAuthorship
= True
1946 if self
.preserveUser
:
1947 self
.checkValidP4Users(commits
)
1950 # Build up a set of options to be passed to diff when
1951 # submitting each commit to p4.
1953 if self
.detectRenames
:
1954 # command-line -M arg
1955 self
.diffOpts
= "-M"
1957 # If not explicitly set check the config variable
1958 detectRenames
= gitConfig("git-p4.detectRenames")
1960 if detectRenames
.lower() == "false" or detectRenames
== "":
1962 elif detectRenames
.lower() == "true":
1963 self
.diffOpts
= "-M"
1965 self
.diffOpts
= "-M%s" % detectRenames
1967 # no command-line arg for -C or --find-copies-harder, just
1969 detectCopies
= gitConfig("git-p4.detectCopies")
1970 if detectCopies
.lower() == "false" or detectCopies
== "":
1972 elif detectCopies
.lower() == "true":
1973 self
.diffOpts
+= " -C"
1975 self
.diffOpts
+= " -C%s" % detectCopies
1977 if gitConfigBool("git-p4.detectCopiesHarder"):
1978 self
.diffOpts
+= " --find-copies-harder"
1981 # Apply the commits, one at a time. On failure, ask if should
1982 # continue to try the rest of the patches, or quit.
1987 last
= len(commits
) - 1
1988 for i
, commit
in enumerate(commits
):
1990 print " ", read_pipe(["git", "show", "-s",
1991 "--format=format:%h %s", commit
])
1994 ok
= self
.applyCommit(commit
)
1996 applied
.append(commit
)
1998 if self
.prepare_p4_only
and i
< last
:
1999 print "Processing only the first commit due to option" \
2000 " --prepare-p4-only"
2005 # prompt for what to do, or use the option/variable
2006 if self
.conflict_behavior
== "ask":
2007 print "What do you want to do?"
2008 response
= raw_input("[s]kip this commit but apply"
2009 " the rest, or [q]uit? ")
2012 elif self
.conflict_behavior
== "skip":
2014 elif self
.conflict_behavior
== "quit":
2017 die("Unknown conflict_behavior '%s'" %
2018 self
.conflict_behavior
)
2020 if response
[0] == "s":
2021 print "Skipping this commit, but applying the rest"
2023 if response
[0] == "q":
2030 chdir(self
.oldWorkingDirectory
)
2034 elif self
.prepare_p4_only
:
2036 elif len(commits
) == len(applied
):
2037 print "All commits applied!"
2041 sync
.branch
= self
.branch
2048 if len(applied
) == 0:
2049 print "No commits applied."
2051 print "Applied only the commits marked with '*':"
2057 print star
, read_pipe(["git", "show", "-s",
2058 "--format=format:%h %s", c
])
2059 print "You will have to do 'git p4 sync' and rebase."
2061 if gitConfigBool("git-p4.exportLabels"):
2062 self
.exportLabels
= True
2064 if self
.exportLabels
:
2065 p4Labels
= getP4Labels(self
.depotPath
)
2066 gitTags
= getGitTags()
2068 missingGitTags
= gitTags
- p4Labels
2069 self
.exportGitTags(missingGitTags
)
2071 # exit with error unless everything applied perfectly
2072 if len(commits
) != len(applied
):
2078 """Represent a p4 view ("p4 help views"), and map files in a
2079 repo according to the view."""
2081 def __init__(self
, client_name
):
2083 self
.client_prefix
= "//%s/" % client_name
2084 # cache results of "p4 where" to lookup client file locations
2085 self
.client_spec_path_cache
= {}
2087 def append(self
, view_line
):
2088 """Parse a view line, splitting it into depot and client
2089 sides. Append to self.mappings, preserving order. This
2090 is only needed for tag creation."""
2092 # Split the view line into exactly two words. P4 enforces
2093 # structure on these lines that simplifies this quite a bit.
2095 # Either or both words may be double-quoted.
2096 # Single quotes do not matter.
2097 # Double-quote marks cannot occur inside the words.
2098 # A + or - prefix is also inside the quotes.
2099 # There are no quotes unless they contain a space.
2100 # The line is already white-space stripped.
2101 # The two words are separated by a single space.
2103 if view_line
[0] == '"':
2104 # First word is double quoted. Find its end.
2105 close_quote_index
= view_line
.find('"', 1)
2106 if close_quote_index
<= 0:
2107 die("No first-word closing quote found: %s" % view_line
)
2108 depot_side
= view_line
[1:close_quote_index
]
2109 # skip closing quote and space
2110 rhs_index
= close_quote_index
+ 1 + 1
2112 space_index
= view_line
.find(" ")
2113 if space_index
<= 0:
2114 die("No word-splitting space found: %s" % view_line
)
2115 depot_side
= view_line
[0:space_index
]
2116 rhs_index
= space_index
+ 1
2118 # prefix + means overlay on previous mapping
2119 if depot_side
.startswith("+"):
2120 depot_side
= depot_side
[1:]
2122 # prefix - means exclude this path, leave out of mappings
2124 if depot_side
.startswith("-"):
2126 depot_side
= depot_side
[1:]
2129 self
.mappings
.append(depot_side
)
2131 def convert_client_path(self
, clientFile
):
2132 # chop off //client/ part to make it relative
2133 if not clientFile
.startswith(self
.client_prefix
):
2134 die("No prefix '%s' on clientFile '%s'" %
2135 (self
.client_prefix
, clientFile
))
2136 return clientFile
[len(self
.client_prefix
):]
2138 def update_client_spec_path_cache(self
, files
):
2139 """ Caching file paths by "p4 where" batch query """
2141 # List depot file paths exclude that already cached
2142 fileArgs
= [f
['path'] for f
in files
if f
['path'] not in self
.client_spec_path_cache
]
2144 if len(fileArgs
) == 0:
2145 return # All files in cache
2147 where_result
= p4CmdList(["-x", "-", "where"], stdin
=fileArgs
)
2148 for res
in where_result
:
2149 if "code" in res
and res
["code"] == "error":
2150 # assume error is "... file(s) not in client view"
2152 if "clientFile" not in res
:
2153 die("No clientFile in 'p4 where' output")
2155 # it will list all of them, but only one not unmap-ped
2157 if gitConfigBool("core.ignorecase"):
2158 res
['depotFile'] = res
['depotFile'].lower()
2159 self
.client_spec_path_cache
[res
['depotFile']] = self
.convert_client_path(res
["clientFile"])
2161 # not found files or unmap files set to ""
2162 for depotFile
in fileArgs
:
2163 if gitConfigBool("core.ignorecase"):
2164 depotFile
= depotFile
.lower()
2165 if depotFile
not in self
.client_spec_path_cache
:
2166 self
.client_spec_path_cache
[depotFile
] = ""
2168 def map_in_client(self
, depot_path
):
2169 """Return the relative location in the client where this
2170 depot file should live. Returns "" if the file should
2171 not be mapped in the client."""
2173 if gitConfigBool("core.ignorecase"):
2174 depot_path
= depot_path
.lower()
2176 if depot_path
in self
.client_spec_path_cache
:
2177 return self
.client_spec_path_cache
[depot_path
]
2179 die( "Error: %s is not found in client spec path" % depot_path
)
2182 class P4Sync(Command
, P4UserMap
):
2183 delete_actions
= ( "delete", "move/delete", "purge" )
2186 Command
.__init
__(self
)
2187 P4UserMap
.__init
__(self
)
2189 optparse
.make_option("--branch", dest
="branch"),
2190 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
2191 optparse
.make_option("--changesfile", dest
="changesFile"),
2192 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
2193 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
2194 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2195 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
2196 help="Import into refs/heads/ , not refs/remotes"),
2197 optparse
.make_option("--max-changes", dest
="maxChanges",
2198 help="Maximum number of changes to import"),
2199 optparse
.make_option("--changes-block-size", dest
="changes_block_size", type="int",
2200 help="Internal block size to use when iteratively calling p4 changes"),
2201 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
2202 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2203 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
2204 help="Only sync files that are included in the Perforce Client Spec"),
2205 optparse
.make_option("-/", dest
="cloneExclude",
2206 action
="append", type="string",
2207 help="exclude depot path"),
2209 self
.description
= """Imports from Perforce into a git repository.\n
2211 //depot/my/project/ -- to import the current head
2212 //depot/my/project/@all -- to import everything
2213 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2215 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2217 self
.usage
+= " //depot/path[@revRange]"
2219 self
.createdBranches
= set()
2220 self
.committedChanges
= set()
2222 self
.detectBranches
= False
2223 self
.detectLabels
= False
2224 self
.importLabels
= False
2225 self
.changesFile
= ""
2226 self
.syncWithOrigin
= True
2227 self
.importIntoRemotes
= True
2228 self
.maxChanges
= ""
2229 self
.changes_block_size
= None
2230 self
.keepRepoPath
= False
2231 self
.depotPaths
= None
2232 self
.p4BranchesInGit
= []
2233 self
.cloneExclude
= []
2234 self
.useClientSpec
= False
2235 self
.useClientSpec_from_options
= False
2236 self
.clientSpecDirs
= None
2237 self
.tempBranches
= []
2238 self
.tempBranchLocation
= "git-p4-tmp"
2239 self
.largeFileSystem
= None
2241 if gitConfig('git-p4.largeFileSystem'):
2242 largeFileSystemConstructor
= globals()[gitConfig('git-p4.largeFileSystem')]
2243 self
.largeFileSystem
= largeFileSystemConstructor(
2244 lambda git_mode
, relPath
, contents
: self
.writeToGitStream(git_mode
, relPath
, contents
)
2247 if gitConfig("git-p4.syncFromOrigin") == "false":
2248 self
.syncWithOrigin
= False
2250 # This is required for the "append" cloneExclude action
2251 def ensure_value(self
, attr
, value
):
2252 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2253 setattr(self
, attr
, value
)
2254 return getattr(self
, attr
)
2256 # Force a checkpoint in fast-import and wait for it to finish
2257 def checkpoint(self
):
2258 self
.gitStream
.write("checkpoint\n\n")
2259 self
.gitStream
.write("progress checkpoint\n\n")
2260 out
= self
.gitOutput
.readline()
2262 print "checkpoint finished: " + out
2264 def extractFilesFromCommit(self
, commit
):
2265 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
2266 for path
in self
.cloneExclude
]
2269 while commit
.has_key("depotFile%s" % fnum
):
2270 path
= commit
["depotFile%s" % fnum
]
2272 if [p
for p
in self
.cloneExclude
2273 if p4PathStartsWith(path
, p
)]:
2276 found
= [p
for p
in self
.depotPaths
2277 if p4PathStartsWith(path
, p
)]
2284 file["rev"] = commit
["rev%s" % fnum
]
2285 file["action"] = commit
["action%s" % fnum
]
2286 file["type"] = commit
["type%s" % fnum
]
2291 def stripRepoPath(self
, path
, prefixes
):
2292 """When streaming files, this is called to map a p4 depot path
2293 to where it should go in git. The prefixes are either
2294 self.depotPaths, or self.branchPrefixes in the case of
2295 branch detection."""
2297 if self
.useClientSpec
:
2298 # branch detection moves files up a level (the branch name)
2299 # from what client spec interpretation gives
2300 path
= self
.clientSpecDirs
.map_in_client(path
)
2301 if self
.detectBranches
:
2302 for b
in self
.knownBranches
:
2303 if path
.startswith(b
+ "/"):
2304 path
= path
[len(b
)+1:]
2306 elif self
.keepRepoPath
:
2307 # Preserve everything in relative path name except leading
2308 # //depot/; just look at first prefix as they all should
2309 # be in the same depot.
2310 depot
= re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])
2311 if p4PathStartsWith(path
, depot
):
2312 path
= path
[len(depot
):]
2316 if p4PathStartsWith(path
, p
):
2317 path
= path
[len(p
):]
2320 path
= wildcard_decode(path
)
2323 def splitFilesIntoBranches(self
, commit
):
2324 """Look at each depotFile in the commit to figure out to what
2325 branch it belongs."""
2327 if self
.clientSpecDirs
:
2328 files
= self
.extractFilesFromCommit(commit
)
2329 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2333 while commit
.has_key("depotFile%s" % fnum
):
2334 path
= commit
["depotFile%s" % fnum
]
2335 found
= [p
for p
in self
.depotPaths
2336 if p4PathStartsWith(path
, p
)]
2343 file["rev"] = commit
["rev%s" % fnum
]
2344 file["action"] = commit
["action%s" % fnum
]
2345 file["type"] = commit
["type%s" % fnum
]
2348 # start with the full relative path where this file would
2350 if self
.useClientSpec
:
2351 relPath
= self
.clientSpecDirs
.map_in_client(path
)
2353 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
2355 for branch
in self
.knownBranches
.keys():
2356 # add a trailing slash so that a commit into qt/4.2foo
2357 # doesn't end up in qt/4.2, e.g.
2358 if relPath
.startswith(branch
+ "/"):
2359 if branch
not in branches
:
2360 branches
[branch
] = []
2361 branches
[branch
].append(file)
2366 def writeToGitStream(self
, gitMode
, relPath
, contents
):
2367 self
.gitStream
.write('M %s inline %s\n' % (gitMode
, relPath
))
2368 self
.gitStream
.write('data %d\n' % sum(len(d
) for d
in contents
))
2370 self
.gitStream
.write(d
)
2371 self
.gitStream
.write('\n')
2373 # output one file from the P4 stream
2374 # - helper for streamP4Files
2376 def streamOneP4File(self
, file, contents
):
2377 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
2379 size
= int(self
.stream_file
['fileSize'])
2380 sys
.stdout
.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath
, size
/1024/1024))
2383 (type_base
, type_mods
) = split_p4_type(file["type"])
2386 if "x" in type_mods
:
2388 if type_base
== "symlink":
2390 # p4 print on a symlink sometimes contains "target\n";
2391 # if it does, remove the newline
2392 data
= ''.join(contents
)
2394 # Some version of p4 allowed creating a symlink that pointed
2395 # to nothing. This causes p4 errors when checking out such
2396 # a change, and errors here too. Work around it by ignoring
2397 # the bad symlink; hopefully a future change fixes it.
2398 print "\nIgnoring empty symlink in %s" % file['depotFile']
2400 elif data
[-1] == '\n':
2401 contents
= [data
[:-1]]
2405 if type_base
== "utf16":
2406 # p4 delivers different text in the python output to -G
2407 # than it does when using "print -o", or normal p4 client
2408 # operations. utf16 is converted to ascii or utf8, perhaps.
2409 # But ascii text saved as -t utf16 is completely mangled.
2410 # Invoke print -o to get the real contents.
2412 # On windows, the newlines will always be mangled by print, so put
2413 # them back too. This is not needed to the cygwin windows version,
2414 # just the native "NT" type.
2416 text
= p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
2417 if p4_version_string().find("/NT") >= 0:
2418 text
= text
.replace("\r\n", "\n")
2421 if type_base
== "apple":
2422 # Apple filetype files will be streamed as a concatenation of
2423 # its appledouble header and the contents. This is useless
2424 # on both macs and non-macs. If using "print -q -o xx", it
2425 # will create "xx" with the data, and "%xx" with the header.
2426 # This is also not very useful.
2428 # Ideally, someday, this script can learn how to generate
2429 # appledouble files directly and import those to git, but
2430 # non-mac machines can never find a use for apple filetype.
2431 print "\nIgnoring apple filetype file %s" % file['depotFile']
2434 # Note that we do not try to de-mangle keywords on utf16 files,
2435 # even though in theory somebody may want that.
2436 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
2438 regexp
= re
.compile(pattern
, re
.VERBOSE
)
2439 text
= ''.join(contents
)
2440 text
= regexp
.sub(r
'$\1$', text
)
2443 if self
.largeFileSystem
:
2444 (git_mode
, contents
) = self
.largeFileSystem
.processContent(git_mode
, relPath
, contents
)
2446 self
.writeToGitStream(git_mode
, relPath
, contents
)
2448 def streamOneP4Deletion(self
, file):
2449 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
2451 sys
.stdout
.write("delete %s\n" % relPath
)
2453 self
.gitStream
.write("D %s\n" % relPath
)
2455 if self
.largeFileSystem
and self
.largeFileSystem
.isLargeFile(relPath
):
2456 self
.largeFileSystem
.removeLargeFile(relPath
)
2458 # handle another chunk of streaming data
2459 def streamP4FilesCb(self
, marshalled
):
2461 # catch p4 errors and complain
2463 if "code" in marshalled
:
2464 if marshalled
["code"] == "error":
2465 if "data" in marshalled
:
2466 err
= marshalled
["data"].rstrip()
2468 if not err
and 'fileSize' in self
.stream_file
:
2469 required_bytes
= int((4 * int(self
.stream_file
["fileSize"])) - calcDiskFree())
2470 if required_bytes
> 0:
2471 err
= 'Not enough space left on %s! Free at least %i MB.' % (
2472 os
.getcwd(), required_bytes
/1024/1024
2477 if self
.stream_have_file_info
:
2478 if "depotFile" in self
.stream_file
:
2479 f
= self
.stream_file
["depotFile"]
2480 # force a failure in fast-import, else an empty
2481 # commit will be made
2482 self
.gitStream
.write("\n")
2483 self
.gitStream
.write("die-now\n")
2484 self
.gitStream
.close()
2485 # ignore errors, but make sure it exits first
2486 self
.importProcess
.wait()
2488 die("Error from p4 print for %s: %s" % (f
, err
))
2490 die("Error from p4 print: %s" % err
)
2492 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2493 # start of a new file - output the old one first
2494 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2495 self
.stream_file
= {}
2496 self
.stream_contents
= []
2497 self
.stream_have_file_info
= False
2499 # pick up the new file information... for the
2500 # 'data' field we need to append to our array
2501 for k
in marshalled
.keys():
2503 if 'streamContentSize' not in self
.stream_file
:
2504 self
.stream_file
['streamContentSize'] = 0
2505 self
.stream_file
['streamContentSize'] += len(marshalled
['data'])
2506 self
.stream_contents
.append(marshalled
['data'])
2508 self
.stream_file
[k
] = marshalled
[k
]
2511 'streamContentSize' in self
.stream_file
and
2512 'fileSize' in self
.stream_file
and
2513 'depotFile' in self
.stream_file
):
2514 size
= int(self
.stream_file
["fileSize"])
2516 progress
= 100*self
.stream_file
['streamContentSize']/size
2517 sys
.stdout
.write('\r%s %d%% (%i MB)' % (self
.stream_file
['depotFile'], progress
, int(size
/1024/1024)))
2520 self
.stream_have_file_info
= True
2522 # Stream directly from "p4 files" into "git fast-import"
2523 def streamP4Files(self
, files
):
2529 # if using a client spec, only add the files that have
2530 # a path in the client
2531 if self
.clientSpecDirs
:
2532 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
2535 filesForCommit
.append(f
)
2536 if f
['action'] in self
.delete_actions
:
2537 filesToDelete
.append(f
)
2539 filesToRead
.append(f
)
2542 for f
in filesToDelete
:
2543 self
.streamOneP4Deletion(f
)
2545 if len(filesToRead
) > 0:
2546 self
.stream_file
= {}
2547 self
.stream_contents
= []
2548 self
.stream_have_file_info
= False
2550 # curry self argument
2551 def streamP4FilesCbSelf(entry
):
2552 self
.streamP4FilesCb(entry
)
2554 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2556 p4CmdList(["-x", "-", "print"],
2558 cb
=streamP4FilesCbSelf
)
2561 if self
.stream_file
.has_key('depotFile'):
2562 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2564 def make_email(self
, userid
):
2565 if userid
in self
.users
:
2566 return self
.users
[userid
]
2568 return "%s <a@b>" % userid
2571 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2573 print "writing tag %s for commit %s" % (labelName
, commit
)
2574 gitStream
.write("tag %s\n" % labelName
)
2575 gitStream
.write("from %s\n" % commit
)
2577 if labelDetails
.has_key('Owner'):
2578 owner
= labelDetails
["Owner"]
2582 # Try to use the owner of the p4 label, or failing that,
2583 # the current p4 user id.
2585 email
= self
.make_email(owner
)
2587 email
= self
.make_email(self
.p4UserId())
2588 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2590 gitStream
.write("tagger %s\n" % tagger
)
2592 print "labelDetails=",labelDetails
2593 if labelDetails
.has_key('Description'):
2594 description
= labelDetails
['Description']
2596 description
= 'Label from git p4'
2598 gitStream
.write("data %d\n" % len(description
))
2599 gitStream
.write(description
)
2600 gitStream
.write("\n")
2602 def commit(self
, details
, files
, branch
, parent
= ""):
2603 epoch
= details
["time"]
2604 author
= details
["user"]
2607 print "commit into %s" % branch
2609 # start with reading files; if that fails, we should not
2613 if [p
for p
in self
.branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2614 new_files
.append (f
)
2616 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2618 if self
.clientSpecDirs
:
2619 self
.clientSpecDirs
.update_client_spec_path_cache(files
)
2621 self
.gitStream
.write("commit %s\n" % branch
)
2622 # gitStream.write("mark :%s\n" % details["change"])
2623 self
.committedChanges
.add(int(details
["change"]))
2625 if author
not in self
.users
:
2626 self
.getUserMapFromPerforceServer()
2627 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2629 self
.gitStream
.write("committer %s\n" % committer
)
2631 self
.gitStream
.write("data <<EOT\n")
2632 self
.gitStream
.write(details
["desc"])
2633 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2634 (','.join(self
.branchPrefixes
), details
["change"]))
2635 if len(details
['options']) > 0:
2636 self
.gitStream
.write(": options = %s" % details
['options'])
2637 self
.gitStream
.write("]\nEOT\n\n")
2641 print "parent %s" % parent
2642 self
.gitStream
.write("from %s\n" % parent
)
2644 self
.streamP4Files(new_files
)
2645 self
.gitStream
.write("\n")
2647 change
= int(details
["change"])
2649 if self
.labels
.has_key(change
):
2650 label
= self
.labels
[change
]
2651 labelDetails
= label
[0]
2652 labelRevisions
= label
[1]
2654 print "Change %s is labelled %s" % (change
, labelDetails
)
2656 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2657 for p
in self
.branchPrefixes
])
2659 if len(files
) == len(labelRevisions
):
2663 if info
["action"] in self
.delete_actions
:
2665 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2667 if cleanedFiles
== labelRevisions
:
2668 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2672 print ("Tag %s does not match with change %s: files do not match."
2673 % (labelDetails
["label"], change
))
2677 print ("Tag %s does not match with change %s: file count is different."
2678 % (labelDetails
["label"], change
))
2680 # Build a dictionary of changelists and labels, for "detect-labels" option.
2681 def getLabels(self
):
2684 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2685 if len(l
) > 0 and not self
.silent
:
2686 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2689 label
= output
["label"]
2693 print "Querying files for label %s" % label
2694 for file in p4CmdList(["files"] +
2695 ["%s...@%s" % (p
, label
)
2696 for p
in self
.depotPaths
]):
2697 revisions
[file["depotFile"]] = file["rev"]
2698 change
= int(file["change"])
2699 if change
> newestChange
:
2700 newestChange
= change
2702 self
.labels
[newestChange
] = [output
, revisions
]
2705 print "Label changes: %s" % self
.labels
.keys()
2707 # Import p4 labels as git tags. A direct mapping does not
2708 # exist, so assume that if all the files are at the same revision
2709 # then we can use that, or it's something more complicated we should
2711 def importP4Labels(self
, stream
, p4Labels
):
2713 print "import p4 labels: " + ' '.join(p4Labels
)
2715 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2716 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2717 if len(validLabelRegexp
) == 0:
2718 validLabelRegexp
= defaultLabelRegexp
2719 m
= re
.compile(validLabelRegexp
)
2721 for name
in p4Labels
:
2724 if not m
.match(name
):
2726 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2729 if name
in ignoredP4Labels
:
2732 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2734 # get the most recent changelist for each file in this label
2735 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2736 for p
in self
.depotPaths
])
2738 if change
.has_key('change'):
2739 # find the corresponding git commit; take the oldest commit
2740 changelist
= int(change
['change'])
2741 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2742 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2743 if len(gitCommit
) == 0:
2744 print "could not find git commit for changelist %d" % changelist
2746 gitCommit
= gitCommit
.strip()
2748 # Convert from p4 time format
2750 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2752 print "Could not convert label time %s" % labelDetails
['Update']
2755 when
= int(time
.mktime(tmwhen
))
2756 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2758 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2761 print "Label %s has no changelists - possibly deleted?" % name
2764 # We can't import this label; don't try again as it will get very
2765 # expensive repeatedly fetching all the files for labels that will
2766 # never be imported. If the label is moved in the future, the
2767 # ignore will need to be removed manually.
2768 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2770 def guessProjectName(self
):
2771 for p
in self
.depotPaths
:
2774 p
= p
[p
.strip().rfind("/") + 1:]
2775 if not p
.endswith("/"):
2779 def getBranchMapping(self
):
2780 lostAndFoundBranches
= set()
2782 user
= gitConfig("git-p4.branchUser")
2784 command
= "branches -u %s" % user
2786 command
= "branches"
2788 for info
in p4CmdList(command
):
2789 details
= p4Cmd(["branch", "-o", info
["branch"]])
2791 while details
.has_key("View%s" % viewIdx
):
2792 paths
= details
["View%s" % viewIdx
].split(" ")
2793 viewIdx
= viewIdx
+ 1
2794 # require standard //depot/foo/... //depot/bar/... mapping
2795 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2798 destination
= paths
[1]
2800 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2801 source
= source
[len(self
.depotPaths
[0]):-4]
2802 destination
= destination
[len(self
.depotPaths
[0]):-4]
2804 if destination
in self
.knownBranches
:
2806 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2807 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2810 self
.knownBranches
[destination
] = source
2812 lostAndFoundBranches
.discard(destination
)
2814 if source
not in self
.knownBranches
:
2815 lostAndFoundBranches
.add(source
)
2817 # Perforce does not strictly require branches to be defined, so we also
2818 # check git config for a branch list.
2820 # Example of branch definition in git config file:
2822 # branchList=main:branchA
2823 # branchList=main:branchB
2824 # branchList=branchA:branchC
2825 configBranches
= gitConfigList("git-p4.branchList")
2826 for branch
in configBranches
:
2828 (source
, destination
) = branch
.split(":")
2829 self
.knownBranches
[destination
] = source
2831 lostAndFoundBranches
.discard(destination
)
2833 if source
not in self
.knownBranches
:
2834 lostAndFoundBranches
.add(source
)
2837 for branch
in lostAndFoundBranches
:
2838 self
.knownBranches
[branch
] = branch
2840 def getBranchMappingFromGitBranches(self
):
2841 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2842 for branch
in branches
.keys():
2843 if branch
== "master":
2846 branch
= branch
[len(self
.projectName
):]
2847 self
.knownBranches
[branch
] = branch
2849 def updateOptionDict(self
, d
):
2851 if self
.keepRepoPath
:
2852 option_keys
['keepRepoPath'] = 1
2854 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2856 def readOptions(self
, d
):
2857 self
.keepRepoPath
= (d
.has_key('options')
2858 and ('keepRepoPath' in d
['options']))
2860 def gitRefForBranch(self
, branch
):
2861 if branch
== "main":
2862 return self
.refPrefix
+ "master"
2864 if len(branch
) <= 0:
2867 return self
.refPrefix
+ self
.projectName
+ branch
2869 def gitCommitByP4Change(self
, ref
, change
):
2871 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2874 latestCommit
= parseRevision(ref
)
2878 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2879 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2884 log
= extractLogMessageFromGitCommit(next
)
2885 settings
= extractSettingsGitLog(log
)
2886 currentChange
= int(settings
['change'])
2888 print "current change %s" % currentChange
2890 if currentChange
== change
:
2892 print "found %s" % next
2895 if currentChange
< change
:
2896 earliestCommit
= "^%s" % next
2898 latestCommit
= "%s" % next
2902 def importNewBranch(self
, branch
, maxChange
):
2903 # make fast-import flush all changes to disk and update the refs using the checkpoint
2904 # command so that we can try to find the branch parent in the git history
2905 self
.gitStream
.write("checkpoint\n\n");
2906 self
.gitStream
.flush();
2907 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2908 range = "@1,%s" % maxChange
2909 #print "prefix" + branchPrefix
2910 changes
= p4ChangesForPaths([branchPrefix
], range, self
.changes_block_size
)
2911 if len(changes
) <= 0:
2913 firstChange
= changes
[0]
2914 #print "first change in branch: %s" % firstChange
2915 sourceBranch
= self
.knownBranches
[branch
]
2916 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2917 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2918 #print "source " + sourceBranch
2920 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2921 #print "branch parent: %s" % branchParentChange
2922 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2923 if len(gitParent
) > 0:
2924 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2925 #print "parent git commit: %s" % gitParent
2927 self
.importChanges(changes
)
2930 def searchParent(self
, parent
, branch
, target
):
2932 for blob
in read_pipe_lines(["git", "rev-list", "--reverse",
2933 "--no-merges", parent
]):
2935 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2938 print "Found parent of %s in commit %s" % (branch
, blob
)
2945 def importChanges(self
, changes
):
2947 for change
in changes
:
2948 description
= p4_describe(change
)
2949 self
.updateOptionDict(description
)
2952 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2957 if self
.detectBranches
:
2958 branches
= self
.splitFilesIntoBranches(description
)
2959 for branch
in branches
.keys():
2961 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2962 self
.branchPrefixes
= [ branchPrefix
]
2966 filesForCommit
= branches
[branch
]
2969 print "branch is %s" % branch
2971 self
.updatedBranches
.add(branch
)
2973 if branch
not in self
.createdBranches
:
2974 self
.createdBranches
.add(branch
)
2975 parent
= self
.knownBranches
[branch
]
2976 if parent
== branch
:
2979 fullBranch
= self
.projectName
+ branch
2980 if fullBranch
not in self
.p4BranchesInGit
:
2982 print("\n Importing new branch %s" % fullBranch
);
2983 if self
.importNewBranch(branch
, change
- 1):
2985 self
.p4BranchesInGit
.append(fullBranch
)
2987 print("\n Resuming with change %s" % change
);
2990 print "parent determined through known branches: %s" % parent
2992 branch
= self
.gitRefForBranch(branch
)
2993 parent
= self
.gitRefForBranch(parent
)
2996 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2998 if len(parent
) == 0 and branch
in self
.initialParents
:
2999 parent
= self
.initialParents
[branch
]
3000 del self
.initialParents
[branch
]
3004 tempBranch
= "%s/%d" % (self
.tempBranchLocation
, change
)
3006 print "Creating temporary branch: " + tempBranch
3007 self
.commit(description
, filesForCommit
, tempBranch
)
3008 self
.tempBranches
.append(tempBranch
)
3010 blob
= self
.searchParent(parent
, branch
, tempBranch
)
3012 self
.commit(description
, filesForCommit
, branch
, blob
)
3015 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
3016 self
.commit(description
, filesForCommit
, branch
, parent
)
3018 files
= self
.extractFilesFromCommit(description
)
3019 self
.commit(description
, files
, self
.branch
,
3021 # only needed once, to connect to the previous commit
3022 self
.initialParent
= ""
3024 print self
.gitError
.read()
3027 def importHeadRevision(self
, revision
):
3028 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
3031 details
["user"] = "git perforce import user"
3032 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
3033 % (' '.join(self
.depotPaths
), revision
))
3034 details
["change"] = revision
3038 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
3040 for info
in p4CmdList(["files"] + fileArgs
):
3042 if 'code' in info
and info
['code'] == 'error':
3043 sys
.stderr
.write("p4 returned an error: %s\n"
3045 if info
['data'].find("must refer to client") >= 0:
3046 sys
.stderr
.write("This particular p4 error is misleading.\n")
3047 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
3048 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
3050 if 'p4ExitCode' in info
:
3051 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
3055 change
= int(info
["change"])
3056 if change
> newestRevision
:
3057 newestRevision
= change
3059 if info
["action"] in self
.delete_actions
:
3060 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3061 #fileCnt = fileCnt + 1
3064 for prop
in ["depotFile", "rev", "action", "type" ]:
3065 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
3067 fileCnt
= fileCnt
+ 1
3069 details
["change"] = newestRevision
3071 # Use time from top-most change so that all git p4 clones of
3072 # the same p4 repo have the same commit SHA1s.
3073 res
= p4_describe(newestRevision
)
3074 details
["time"] = res
["time"]
3076 self
.updateOptionDict(details
)
3078 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
3080 print "IO error with git fast-import. Is your git version recent enough?"
3081 print self
.gitError
.read()
3084 def run(self
, args
):
3085 self
.depotPaths
= []
3086 self
.changeRange
= ""
3087 self
.previousDepotPaths
= []
3088 self
.hasOrigin
= False
3090 # map from branch depot path to parent branch
3091 self
.knownBranches
= {}
3092 self
.initialParents
= {}
3094 if self
.importIntoRemotes
:
3095 self
.refPrefix
= "refs/remotes/p4/"
3097 self
.refPrefix
= "refs/heads/p4/"
3099 if self
.syncWithOrigin
:
3100 self
.hasOrigin
= originP4BranchesExist()
3103 print 'Syncing with origin first, using "git fetch origin"'
3104 system("git fetch origin")
3106 branch_arg_given
= bool(self
.branch
)
3107 if len(self
.branch
) == 0:
3108 self
.branch
= self
.refPrefix
+ "master"
3109 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
3110 system("git update-ref %s refs/heads/p4" % self
.branch
)
3111 system("git branch -D p4")
3113 # accept either the command-line option, or the configuration variable
3114 if self
.useClientSpec
:
3115 # will use this after clone to set the variable
3116 self
.useClientSpec_from_options
= True
3118 if gitConfigBool("git-p4.useclientspec"):
3119 self
.useClientSpec
= True
3120 if self
.useClientSpec
:
3121 self
.clientSpecDirs
= getClientSpec()
3123 # TODO: should always look at previous commits,
3124 # merge with previous imports, if possible.
3127 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
3129 # branches holds mapping from branch name to sha1
3130 branches
= p4BranchesInGit(self
.importIntoRemotes
)
3132 # restrict to just this one, disabling detect-branches
3133 if branch_arg_given
:
3134 short
= self
.branch
.split("/")[-1]
3135 if short
in branches
:
3136 self
.p4BranchesInGit
= [ short
]
3138 self
.p4BranchesInGit
= branches
.keys()
3140 if len(self
.p4BranchesInGit
) > 1:
3142 print "Importing from/into multiple branches"
3143 self
.detectBranches
= True
3144 for branch
in branches
.keys():
3145 self
.initialParents
[self
.refPrefix
+ branch
] = \
3149 print "branches: %s" % self
.p4BranchesInGit
3152 for branch
in self
.p4BranchesInGit
:
3153 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
3155 settings
= extractSettingsGitLog(logMsg
)
3157 self
.readOptions(settings
)
3158 if (settings
.has_key('depot-paths')
3159 and settings
.has_key ('change')):
3160 change
= int(settings
['change']) + 1
3161 p4Change
= max(p4Change
, change
)
3163 depotPaths
= sorted(settings
['depot-paths'])
3164 if self
.previousDepotPaths
== []:
3165 self
.previousDepotPaths
= depotPaths
3168 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
3169 prev_list
= prev
.split("/")
3170 cur_list
= cur
.split("/")
3171 for i
in range(0, min(len(cur_list
), len(prev_list
))):
3172 if cur_list
[i
] <> prev_list
[i
]:
3176 paths
.append ("/".join(cur_list
[:i
+ 1]))
3178 self
.previousDepotPaths
= paths
3181 self
.depotPaths
= sorted(self
.previousDepotPaths
)
3182 self
.changeRange
= "@%s,#head" % p4Change
3183 if not self
.silent
and not self
.detectBranches
:
3184 print "Performing incremental import into %s git branch" % self
.branch
3186 # accept multiple ref name abbreviations:
3187 # refs/foo/bar/branch -> use it exactly
3188 # p4/branch -> prepend refs/remotes/ or refs/heads/
3189 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3190 if not self
.branch
.startswith("refs/"):
3191 if self
.importIntoRemotes
:
3192 prepend
= "refs/remotes/"
3194 prepend
= "refs/heads/"
3195 if not self
.branch
.startswith("p4/"):
3197 self
.branch
= prepend
+ self
.branch
3199 if len(args
) == 0 and self
.depotPaths
:
3201 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
3203 if self
.depotPaths
and self
.depotPaths
!= args
:
3204 print ("previous import used depot path %s and now %s was specified. "
3205 "This doesn't work!" % (' '.join (self
.depotPaths
),
3209 self
.depotPaths
= sorted(args
)
3214 # Make sure no revision specifiers are used when --changesfile
3216 bad_changesfile
= False
3217 if len(self
.changesFile
) > 0:
3218 for p
in self
.depotPaths
:
3219 if p
.find("@") >= 0 or p
.find("#") >= 0:
3220 bad_changesfile
= True
3223 die("Option --changesfile is incompatible with revision specifiers")
3226 for p
in self
.depotPaths
:
3227 if p
.find("@") != -1:
3228 atIdx
= p
.index("@")
3229 self
.changeRange
= p
[atIdx
:]
3230 if self
.changeRange
== "@all":
3231 self
.changeRange
= ""
3232 elif ',' not in self
.changeRange
:
3233 revision
= self
.changeRange
3234 self
.changeRange
= ""
3236 elif p
.find("#") != -1:
3237 hashIdx
= p
.index("#")
3238 revision
= p
[hashIdx
:]
3240 elif self
.previousDepotPaths
== []:
3241 # pay attention to changesfile, if given, else import
3242 # the entire p4 tree at the head revision
3243 if len(self
.changesFile
) == 0:
3246 p
= re
.sub ("\.\.\.$", "", p
)
3247 if not p
.endswith("/"):
3252 self
.depotPaths
= newPaths
3254 # --detect-branches may change this for each branch
3255 self
.branchPrefixes
= self
.depotPaths
3257 self
.loadUserMapFromCache()
3259 if self
.detectLabels
:
3262 if self
.detectBranches
:
3263 ## FIXME - what's a P4 projectName ?
3264 self
.projectName
= self
.guessProjectName()
3267 self
.getBranchMappingFromGitBranches()
3269 self
.getBranchMapping()
3271 print "p4-git branches: %s" % self
.p4BranchesInGit
3272 print "initial parents: %s" % self
.initialParents
3273 for b
in self
.p4BranchesInGit
:
3277 b
= b
[len(self
.projectName
):]
3278 self
.createdBranches
.add(b
)
3280 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
3282 self
.importProcess
= subprocess
.Popen(["git", "fast-import"],
3283 stdin
=subprocess
.PIPE
,
3284 stdout
=subprocess
.PIPE
,
3285 stderr
=subprocess
.PIPE
);
3286 self
.gitOutput
= self
.importProcess
.stdout
3287 self
.gitStream
= self
.importProcess
.stdin
3288 self
.gitError
= self
.importProcess
.stderr
3291 self
.importHeadRevision(revision
)
3295 if len(self
.changesFile
) > 0:
3296 output
= open(self
.changesFile
).readlines()
3299 changeSet
.add(int(line
))
3301 for change
in changeSet
:
3302 changes
.append(change
)
3306 # catch "git p4 sync" with no new branches, in a repo that
3307 # does not have any existing p4 branches
3309 if not self
.p4BranchesInGit
:
3310 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3312 # The default branch is master, unless --branch is used to
3313 # specify something else. Make sure it exists, or complain
3314 # nicely about how to use --branch.
3315 if not self
.detectBranches
:
3316 if not branch_exists(self
.branch
):
3317 if branch_arg_given
:
3318 die("Error: branch %s does not exist." % self
.branch
)
3320 die("Error: no branch %s; perhaps specify one with --branch." %
3324 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
3326 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
, self
.changes_block_size
)
3328 if len(self
.maxChanges
) > 0:
3329 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
3331 if len(changes
) == 0:
3333 print "No changes to import!"
3335 if not self
.silent
and not self
.detectBranches
:
3336 print "Import destination: %s" % self
.branch
3338 self
.updatedBranches
= set()
3340 if not self
.detectBranches
:
3342 # start a new branch
3343 self
.initialParent
= ""
3345 # build on a previous revision
3346 self
.initialParent
= parseRevision(self
.branch
)
3348 self
.importChanges(changes
)
3352 if len(self
.updatedBranches
) > 0:
3353 sys
.stdout
.write("Updated branches: ")
3354 for b
in self
.updatedBranches
:
3355 sys
.stdout
.write("%s " % b
)
3356 sys
.stdout
.write("\n")
3358 if gitConfigBool("git-p4.importLabels"):
3359 self
.importLabels
= True
3361 if self
.importLabels
:
3362 p4Labels
= getP4Labels(self
.depotPaths
)
3363 gitTags
= getGitTags()
3365 missingP4Labels
= p4Labels
- gitTags
3366 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
3368 self
.gitStream
.close()
3369 if self
.importProcess
.wait() != 0:
3370 die("fast-import failed: %s" % self
.gitError
.read())
3371 self
.gitOutput
.close()
3372 self
.gitError
.close()
3374 # Cleanup temporary branches created during import
3375 if self
.tempBranches
!= []:
3376 for branch
in self
.tempBranches
:
3377 read_pipe("git update-ref -d %s" % branch
)
3378 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
3380 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3381 # a convenient shortcut refname "p4".
3382 if self
.importIntoRemotes
:
3383 head_ref
= self
.refPrefix
+ "HEAD"
3384 if not gitBranchExists(head_ref
) and gitBranchExists(self
.branch
):
3385 system(["git", "symbolic-ref", head_ref
, self
.branch
])
3389 class P4Rebase(Command
):
3391 Command
.__init
__(self
)
3393 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
3395 self
.importLabels
= False
3396 self
.description
= ("Fetches the latest revision from perforce and "
3397 + "rebases the current work (branch) against it")
3399 def run(self
, args
):
3401 sync
.importLabels
= self
.importLabels
3404 return self
.rebase()
3407 if os
.system("git update-index --refresh") != 0:
3408 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3409 if len(read_pipe("git diff-index HEAD --")) > 0:
3410 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3412 [upstream
, settings
] = findUpstreamBranchPoint()
3413 if len(upstream
) == 0:
3414 die("Cannot find upstream branchpoint for rebase")
3416 # the branchpoint may be p4/foo~3, so strip off the parent
3417 upstream
= re
.sub("~[0-9]+$", "", upstream
)
3419 print "Rebasing the current branch onto %s" % upstream
3420 oldHead
= read_pipe("git rev-parse HEAD").strip()
3421 system("git rebase %s" % upstream
)
3422 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead
)
3425 class P4Clone(P4Sync
):
3427 P4Sync
.__init
__(self
)
3428 self
.description
= "Creates a new git repository and imports from Perforce into it"
3429 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
3431 optparse
.make_option("--destination", dest
="cloneDestination",
3432 action
='store', default
=None,
3433 help="where to leave result of the clone"),
3434 optparse
.make_option("--bare", dest
="cloneBare",
3435 action
="store_true", default
=False),
3437 self
.cloneDestination
= None
3438 self
.needsGit
= False
3439 self
.cloneBare
= False
3441 def defaultDestination(self
, args
):
3442 ## TODO: use common prefix of args?
3444 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
3445 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
3446 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
3447 depotDir
= re
.sub(r
"/$", "", depotDir
)
3448 return os
.path
.split(depotDir
)[1]
3450 def run(self
, args
):
3454 if self
.keepRepoPath
and not self
.cloneDestination
:
3455 sys
.stderr
.write("Must specify destination for --keep-path\n")
3460 if not self
.cloneDestination
and len(depotPaths
) > 1:
3461 self
.cloneDestination
= depotPaths
[-1]
3462 depotPaths
= depotPaths
[:-1]
3464 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
3465 for p
in depotPaths
:
3466 if not p
.startswith("//"):
3467 sys
.stderr
.write('Depot paths must start with "//": %s\n' % p
)
3470 if not self
.cloneDestination
:
3471 self
.cloneDestination
= self
.defaultDestination(args
)
3473 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
3475 if not os
.path
.exists(self
.cloneDestination
):
3476 os
.makedirs(self
.cloneDestination
)
3477 chdir(self
.cloneDestination
)
3479 init_cmd
= [ "git", "init" ]
3481 init_cmd
.append("--bare")
3482 retcode
= subprocess
.call(init_cmd
)
3484 raise CalledProcessError(retcode
, init_cmd
)
3486 if not P4Sync
.run(self
, depotPaths
):
3489 # create a master branch and check out a work tree
3490 if gitBranchExists(self
.branch
):
3491 system([ "git", "branch", "master", self
.branch
])
3492 if not self
.cloneBare
:
3493 system([ "git", "checkout", "-f" ])
3495 print 'Not checking out any branch, use ' \
3496 '"git checkout -q -b master <branch>"'
3498 # auto-set this variable if invoked with --use-client-spec
3499 if self
.useClientSpec_from_options
:
3500 system("git config --bool git-p4.useclientspec true")
3504 class P4Branches(Command
):
3506 Command
.__init
__(self
)
3508 self
.description
= ("Shows the git branches that hold imports and their "
3509 + "corresponding perforce depot paths")
3510 self
.verbose
= False
3512 def run(self
, args
):
3513 if originP4BranchesExist():
3514 createOrUpdateBranchesFromOrigin()
3516 cmdline
= "git rev-parse --symbolic "
3517 cmdline
+= " --remotes"
3519 for line
in read_pipe_lines(cmdline
):
3522 if not line
.startswith('p4/') or line
== "p4/HEAD":
3526 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
3527 settings
= extractSettingsGitLog(log
)
3529 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
3532 class HelpFormatter(optparse
.IndentedHelpFormatter
):
3534 optparse
.IndentedHelpFormatter
.__init
__(self
)
3536 def format_description(self
, description
):
3538 return description
+ "\n"
3542 def printUsage(commands
):
3543 print "usage: %s <command> [options]" % sys
.argv
[0]
3545 print "valid commands: %s" % ", ".join(commands
)
3547 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3552 "submit" : P4Submit
,
3553 "commit" : P4Submit
,
3555 "rebase" : P4Rebase
,
3557 "rollback" : P4RollBack
,
3558 "branches" : P4Branches
3563 if len(sys
.argv
[1:]) == 0:
3564 printUsage(commands
.keys())
3567 cmdName
= sys
.argv
[1]
3569 klass
= commands
[cmdName
]
3572 print "unknown command %s" % cmdName
3574 printUsage(commands
.keys())
3577 options
= cmd
.options
3578 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3582 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3584 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3586 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3588 description
= cmd
.description
,
3589 formatter
= HelpFormatter())
3591 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3593 verbose
= cmd
.verbose
3595 if cmd
.gitdir
== None:
3596 cmd
.gitdir
= os
.path
.abspath(".git")
3597 if not isValidGitDir(cmd
.gitdir
):
3598 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3599 if os
.path
.exists(cmd
.gitdir
):
3600 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3604 if not isValidGitDir(cmd
.gitdir
):
3605 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3606 cmd
.gitdir
+= "/.git"
3608 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3610 os
.environ
["GIT_DIR"] = cmd
.gitdir
3612 if not cmd
.run(args
):
3617 if __name__
== '__main__':