3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
124 expand
= isinstance(cmd
,basestring
)
126 sys
.stderr
.write("executing %s\n" % str(cmd
))
127 subprocess
.check_call(cmd
, shell
=expand
)
130 """Specifically invoke p4 as the system command. """
131 real_cmd
= p4_build_cmd(cmd
)
132 expand
= isinstance(real_cmd
, basestring
)
133 subprocess
.check_call(real_cmd
, shell
=expand
)
135 def p4_integrate(src
, dest
):
136 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
138 def p4_sync(f
, *options
):
139 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
142 # forcibly add file names with wildcards
143 if wildcard_present(f
):
144 p4_system(["add", "-f", f
])
146 p4_system(["add", f
])
149 p4_system(["delete", wildcard_encode(f
)])
152 p4_system(["edit", wildcard_encode(f
)])
155 p4_system(["revert", wildcard_encode(f
)])
157 def p4_reopen(type, f
):
158 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
161 # Canonicalize the p4 type and return a tuple of the
162 # base type, plus any modifiers. See "p4 help filetypes"
163 # for a list and explanation.
165 def split_p4_type(p4type
):
167 p4_filetypes_historical
= {
168 "ctempobj": "binary+Sw",
174 "tempobj": "binary+FSw",
175 "ubinary": "binary+F",
176 "uresource": "resource+F",
177 "uxbinary": "binary+Fx",
178 "xbinary": "binary+x",
180 "xtempobj": "binary+Swx",
182 "xunicode": "unicode+x",
185 if p4type
in p4_filetypes_historical
:
186 p4type
= p4_filetypes_historical
[p4type
]
188 s
= p4type
.split("+")
196 # return the raw p4 type of a file (text, text+ko, etc)
199 results
= p4CmdList(["fstat", "-T", "headType", file])
200 return results
[0]['headType']
203 # Given a type base and modifier, return a regexp matching
204 # the keywords that can be expanded in the file
206 def p4_keywords_regexp_for_type(base
, type_mods
):
207 if base
in ("text", "unicode", "binary"):
209 if "ko" in type_mods
:
211 elif "k" in type_mods
:
212 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
216 \$ # Starts with a dollar, followed by...
217 (%s) # one of the keywords, followed by...
218 (:[^$]+)? # possibly an old expansion, followed by...
226 # Given a file, return a regexp matching the possible
227 # RCS keywords that will be expanded, or None for files
228 # with kw expansion turned off.
230 def p4_keywords_regexp_for_file(file):
231 if not os
.path
.exists(file):
234 (type_base
, type_mods
) = split_p4_type(p4_type(file))
235 return p4_keywords_regexp_for_type(type_base
, type_mods
)
237 def setP4ExecBit(file, mode
):
238 # Reopens an already open file and changes the execute bit to match
239 # the execute bit setting in the passed in mode.
243 if not isModeExec(mode
):
244 p4Type
= getP4OpenedType(file)
245 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
246 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
247 if p4Type
[-1] == "+":
248 p4Type
= p4Type
[0:-1]
250 p4_reopen(p4Type
, file)
252 def getP4OpenedType(file):
253 # Returns the perforce file type for the given file.
255 result
= p4_read_pipe(["opened", wildcard_encode(file)])
256 match
= re
.match(".*\((.+)\)\r?$", result
)
258 return match
.group(1)
260 die("Could not determine file type for %s (result: '%s')" % (file, result
))
262 # Return the set of all p4 labels
263 def getP4Labels(depotPaths
):
265 if isinstance(depotPaths
,basestring
):
266 depotPaths
= [depotPaths
]
268 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
274 # Return the set of all git tags
277 for line
in read_pipe_lines(["git", "tag"]):
282 def diffTreePattern():
283 # This is a simple generator for the diff tree regex pattern. This could be
284 # a class variable if this and parseDiffTreeEntry were a part of a class.
285 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
289 def parseDiffTreeEntry(entry
):
290 """Parses a single diff tree entry into its component elements.
292 See git-diff-tree(1) manpage for details about the format of the diff
293 output. This method returns a dictionary with the following elements:
295 src_mode - The mode of the source file
296 dst_mode - The mode of the destination file
297 src_sha1 - The sha1 for the source file
298 dst_sha1 - The sha1 fr the destination file
299 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
300 status_score - The score for the status (applicable for 'C' and 'R'
301 statuses). This is None if there is no score.
302 src - The path for the source file.
303 dst - The path for the destination file. This is only present for
304 copy or renames. If it is not present, this is None.
306 If the pattern is not matched, None is returned."""
308 match
= diffTreePattern().next().match(entry
)
311 'src_mode': match
.group(1),
312 'dst_mode': match
.group(2),
313 'src_sha1': match
.group(3),
314 'dst_sha1': match
.group(4),
315 'status': match
.group(5),
316 'status_score': match
.group(6),
317 'src': match
.group(7),
318 'dst': match
.group(10)
322 def isModeExec(mode
):
323 # Returns True if the given git mode represents an executable file,
325 return mode
[-3:] == "755"
327 def isModeExecChanged(src_mode
, dst_mode
):
328 return isModeExec(src_mode
) != isModeExec(dst_mode
)
330 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
332 if isinstance(cmd
,basestring
):
339 cmd
= p4_build_cmd(cmd
)
341 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
343 # Use a temporary file to avoid deadlocks without
344 # subprocess.communicate(), which would put another copy
345 # of stdout into memory.
347 if stdin
is not None:
348 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
349 if isinstance(stdin
,basestring
):
350 stdin_file
.write(stdin
)
353 stdin_file
.write(i
+ '\n')
357 p4
= subprocess
.Popen(cmd
,
360 stdout
=subprocess
.PIPE
)
365 entry
= marshal
.load(p4
.stdout
)
375 entry
["p4ExitCode"] = exitCode
381 list = p4CmdList(cmd
)
387 def p4Where(depotPath
):
388 if not depotPath
.endswith("/"):
390 depotPath
= depotPath
+ "..."
391 outputList
= p4CmdList(["where", depotPath
])
393 for entry
in outputList
:
394 if "depotFile" in entry
:
395 if entry
["depotFile"] == depotPath
:
398 elif "data" in entry
:
399 data
= entry
.get("data")
400 space
= data
.find(" ")
401 if data
[:space
] == depotPath
:
406 if output
["code"] == "error":
410 clientPath
= output
.get("path")
411 elif "data" in output
:
412 data
= output
.get("data")
413 lastSpace
= data
.rfind(" ")
414 clientPath
= data
[lastSpace
+ 1:]
416 if clientPath
.endswith("..."):
417 clientPath
= clientPath
[:-3]
420 def currentGitBranch():
421 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
423 def isValidGitDir(path
):
424 if (os
.path
.exists(path
+ "/HEAD")
425 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
429 def parseRevision(ref
):
430 return read_pipe("git rev-parse %s" % ref
).strip()
432 def branchExists(ref
):
433 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
437 def extractLogMessageFromGitCommit(commit
):
440 ## fixme: title is first line of commit, not 1st paragraph.
442 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
451 def extractSettingsGitLog(log
):
453 for line
in log
.split("\n"):
455 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
459 assignments
= m
.group(1).split (':')
460 for a
in assignments
:
462 key
= vals
[0].strip()
463 val
= ('='.join (vals
[1:])).strip()
464 if val
.endswith ('\"') and val
.startswith('"'):
469 paths
= values
.get("depot-paths")
471 paths
= values
.get("depot-path")
473 values
['depot-paths'] = paths
.split(',')
476 def gitBranchExists(branch
):
477 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
478 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
479 return proc
.wait() == 0;
482 def gitConfig(key
, args
= None): # set args to "--bool", for instance
483 if not _gitConfig
.has_key(key
):
486 argsFilter
= "%s " % args
487 cmd
= "git config %s%s" % (argsFilter
, key
)
488 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
489 return _gitConfig
[key
]
491 def gitConfigList(key
):
492 if not _gitConfig
.has_key(key
):
493 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
494 return _gitConfig
[key
]
496 def p4BranchesInGit(branchesAreInRemotes
= True):
499 cmdline
= "git rev-parse --symbolic "
500 if branchesAreInRemotes
:
501 cmdline
+= " --remotes"
503 cmdline
+= " --branches"
505 for line
in read_pipe_lines(cmdline
):
508 ## only import to p4/
509 if not line
.startswith('p4/') or line
== "p4/HEAD":
514 branch
= re
.sub ("^p4/", "", line
)
516 branches
[branch
] = parseRevision(line
)
519 def findUpstreamBranchPoint(head
= "HEAD"):
520 branches
= p4BranchesInGit()
521 # map from depot-path to branch name
522 branchByDepotPath
= {}
523 for branch
in branches
.keys():
524 tip
= branches
[branch
]
525 log
= extractLogMessageFromGitCommit(tip
)
526 settings
= extractSettingsGitLog(log
)
527 if settings
.has_key("depot-paths"):
528 paths
= ",".join(settings
["depot-paths"])
529 branchByDepotPath
[paths
] = "remotes/p4/" + branch
533 while parent
< 65535:
534 commit
= head
+ "~%s" % parent
535 log
= extractLogMessageFromGitCommit(commit
)
536 settings
= extractSettingsGitLog(log
)
537 if settings
.has_key("depot-paths"):
538 paths
= ",".join(settings
["depot-paths"])
539 if branchByDepotPath
.has_key(paths
):
540 return [branchByDepotPath
[paths
], settings
]
544 return ["", settings
]
546 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
548 print ("Creating/updating branch(es) in %s based on origin branch(es)"
551 originPrefix
= "origin/p4/"
553 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
555 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
558 headName
= line
[len(originPrefix
):]
559 remoteHead
= localRefPrefix
+ headName
562 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
563 if (not original
.has_key('depot-paths')
564 or not original
.has_key('change')):
568 if not gitBranchExists(remoteHead
):
570 print "creating %s" % remoteHead
573 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
574 if settings
.has_key('change') > 0:
575 if settings
['depot-paths'] == original
['depot-paths']:
576 originP4Change
= int(original
['change'])
577 p4Change
= int(settings
['change'])
578 if originP4Change
> p4Change
:
579 print ("%s (%s) is newer than %s (%s). "
580 "Updating p4 branch from origin."
581 % (originHead
, originP4Change
,
582 remoteHead
, p4Change
))
585 print ("Ignoring: %s was imported from %s while "
586 "%s was imported from %s"
587 % (originHead
, ','.join(original
['depot-paths']),
588 remoteHead
, ','.join(settings
['depot-paths'])))
591 system("git update-ref %s %s" % (remoteHead
, originHead
))
593 def originP4BranchesExist():
594 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
596 def p4ChangesForPaths(depotPaths
, changeRange
):
600 cmd
+= ["%s...%s" % (p
, changeRange
)]
601 output
= p4_read_pipe_lines(cmd
)
605 changeNum
= int(line
.split(" ")[1])
606 changes
[changeNum
] = True
608 changelist
= changes
.keys()
612 def p4PathStartsWith(path
, prefix
):
613 # This method tries to remedy a potential mixed-case issue:
615 # If UserA adds //depot/DirA/file1
616 # and UserB adds //depot/dira/file2
618 # we may or may not have a problem. If you have core.ignorecase=true,
619 # we treat DirA and dira as the same directory
620 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
622 return path
.lower().startswith(prefix
.lower())
623 return path
.startswith(prefix
)
626 """Look at the p4 client spec, create a View() object that contains
627 all the mappings, and return it."""
629 specList
= p4CmdList("client -o")
630 if len(specList
) != 1:
631 die('Output from "client -o" is %d lines, expecting 1' %
634 # dictionary of all client parameters
637 # just the keys that start with "View"
638 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
643 # append the lines, in order, to the view
644 for view_num
in range(len(view_keys
)):
645 k
= "View%d" % view_num
646 if k
not in view_keys
:
647 die("Expected view key %s missing" % k
)
648 view
.append(entry
[k
])
653 """Grab the client directory."""
655 output
= p4CmdList("client -o")
657 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
660 if "Root" not in entry
:
661 die('Client has no "Root"')
666 # P4 wildcards are not allowed in filenames. P4 complains
667 # if you simply add them, but you can force it with "-f", in
668 # which case it translates them into %xx encoding internally.
670 def wildcard_decode(path
):
671 # Search for and fix just these four characters. Do % last so
672 # that fixing it does not inadvertently create new %-escapes.
673 # Cannot have * in a filename in windows; untested as to
674 # what p4 would do in such a case.
675 if not platform
.system() == "Windows":
676 path
= path
.replace("%2A", "*")
677 path
= path
.replace("%23", "#") \
678 .replace("%40", "@") \
682 def wildcard_encode(path
):
683 # do % first to avoid double-encoding the %s introduced here
684 path
= path
.replace("%", "%25") \
685 .replace("*", "%2A") \
686 .replace("#", "%23") \
690 def wildcard_present(path
):
691 return path
.translate(None, "*#@%") != path
695 self
.usage
= "usage: %prog [options]"
701 self
.userMapFromPerforceServer
= False
702 self
.myP4UserId
= None
706 return self
.myP4UserId
708 results
= p4CmdList("user -o")
710 if r
.has_key('User'):
711 self
.myP4UserId
= r
['User']
713 die("Could not find your p4 user id")
715 def p4UserIsMe(self
, p4User
):
716 # return True if the given p4 user is actually me
718 if not p4User
or p4User
!= me
:
723 def getUserCacheFilename(self
):
724 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
725 return home
+ "/.gitp4-usercache.txt"
727 def getUserMapFromPerforceServer(self
):
728 if self
.userMapFromPerforceServer
:
733 for output
in p4CmdList("users"):
734 if not output
.has_key("User"):
736 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
737 self
.emails
[output
["Email"]] = output
["User"]
741 for (key
, val
) in self
.users
.items():
742 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
744 open(self
.getUserCacheFilename(), "wb").write(s
)
745 self
.userMapFromPerforceServer
= True
747 def loadUserMapFromCache(self
):
749 self
.userMapFromPerforceServer
= False
751 cache
= open(self
.getUserCacheFilename(), "rb")
752 lines
= cache
.readlines()
755 entry
= line
.strip().split("\t")
756 self
.users
[entry
[0]] = entry
[1]
758 self
.getUserMapFromPerforceServer()
760 class P4Debug(Command
):
762 Command
.__init
__(self
)
764 self
.description
= "A tool to debug the output of p4 -G."
765 self
.needsGit
= False
769 for output
in p4CmdList(args
):
770 print 'Element: %d' % j
775 class P4RollBack(Command
):
777 Command
.__init
__(self
)
779 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
781 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
782 self
.rollbackLocalBranches
= False
787 maxChange
= int(args
[0])
789 if "p4ExitCode" in p4Cmd("changes -m 1"):
790 die("Problems executing p4");
792 if self
.rollbackLocalBranches
:
793 refPrefix
= "refs/heads/"
794 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
796 refPrefix
= "refs/remotes/"
797 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
800 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
802 ref
= refPrefix
+ line
803 log
= extractLogMessageFromGitCommit(ref
)
804 settings
= extractSettingsGitLog(log
)
806 depotPaths
= settings
['depot-paths']
807 change
= settings
['change']
811 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
812 for p
in depotPaths
]))) == 0:
813 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
814 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
817 while change
and int(change
) > maxChange
:
820 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
821 system("git update-ref %s \"%s^\"" % (ref
, ref
))
822 log
= extractLogMessageFromGitCommit(ref
)
823 settings
= extractSettingsGitLog(log
)
826 depotPaths
= settings
['depot-paths']
827 change
= settings
['change']
830 print "%s rewound to %s" % (ref
, change
)
834 class P4Submit(Command
, P4UserMap
):
836 Command
.__init
__(self
)
837 P4UserMap
.__init
__(self
)
839 optparse
.make_option("--origin", dest
="origin"),
840 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
841 # preserve the user, requires relevant p4 permissions
842 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
843 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
845 self
.description
= "Submit changes from git to the perforce depot."
846 self
.usage
+= " [name of git branch to submit into perforce depot]"
847 self
.interactive
= True
849 self
.detectRenames
= False
850 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
851 self
.isWindows
= (platform
.system() == "Windows")
852 self
.exportLabels
= False
855 if len(p4CmdList("opened ...")) > 0:
856 die("You have files opened with perforce! Close them before starting the sync.")
858 # replaces everything between 'Description:' and the next P4 submit template field with the
860 def prepareLogMessage(self
, template
, message
):
863 inDescriptionSection
= False
865 for line
in template
.split("\n"):
866 if line
.startswith("#"):
867 result
+= line
+ "\n"
870 if inDescriptionSection
:
871 if line
.startswith("Files:") or line
.startswith("Jobs:"):
872 inDescriptionSection
= False
876 if line
.startswith("Description:"):
877 inDescriptionSection
= True
879 for messageLine
in message
.split("\n"):
880 line
+= "\t" + messageLine
+ "\n"
882 result
+= line
+ "\n"
886 def patchRCSKeywords(self
, file, pattern
):
887 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
888 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
890 outFile
= os
.fdopen(handle
, "w+")
891 inFile
= open(file, "r")
892 regexp
= re
.compile(pattern
, re
.VERBOSE
)
893 for line
in inFile
.readlines():
894 line
= regexp
.sub(r
'$\1$', line
)
898 # Forcibly overwrite the original file
900 shutil
.move(outFileName
, file)
902 # cleanup our temporary file
903 os
.unlink(outFileName
)
904 print "Failed to strip RCS keywords in %s" % file
907 print "Patched up RCS keywords in %s" % file
909 def p4UserForCommit(self
,id):
910 # Return the tuple (perforce user,git email) for a given git commit id
911 self
.getUserMapFromPerforceServer()
912 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
913 gitEmail
= gitEmail
.strip()
914 if not self
.emails
.has_key(gitEmail
):
915 return (None,gitEmail
)
917 return (self
.emails
[gitEmail
],gitEmail
)
919 def checkValidP4Users(self
,commits
):
920 # check if any git authors cannot be mapped to p4 users
922 (user
,email
) = self
.p4UserForCommit(id)
924 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
925 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
928 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
930 def lastP4Changelist(self
):
931 # Get back the last changelist number submitted in this client spec. This
932 # then gets used to patch up the username in the change. If the same
933 # client spec is being used by multiple processes then this might go
935 results
= p4CmdList("client -o") # find the current client
938 if r
.has_key('Client'):
942 die("could not get client spec")
943 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
945 if r
.has_key('change'):
947 die("Could not get changelist number for last submit - cannot patch up user details")
949 def modifyChangelistUser(self
, changelist
, newUser
):
950 # fixup the user field of a changelist after it has been submitted.
951 changes
= p4CmdList("change -o %s" % changelist
)
952 if len(changes
) != 1:
953 die("Bad output from p4 change modifying %s to user %s" %
954 (changelist
, newUser
))
957 if c
['User'] == newUser
: return # nothing to do
959 input = marshal
.dumps(c
)
961 result
= p4CmdList("change -f -i", stdin
=input)
963 if r
.has_key('code'):
964 if r
['code'] == 'error':
965 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
966 if r
.has_key('data'):
967 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
969 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
971 def canChangeChangelists(self
):
972 # check to see if we have p4 admin or super-user permissions, either of
973 # which are required to modify changelists.
974 results
= p4CmdList(["protects", self
.depotPath
])
976 if r
.has_key('perm'):
977 if r
['perm'] == 'admin':
979 if r
['perm'] == 'super':
983 def prepareSubmitTemplate(self
):
984 # remove lines in the Files section that show changes to files outside the depot path we're committing into
986 inFilesSection
= False
987 for line
in p4_read_pipe_lines(['change', '-o']):
988 if line
.endswith("\r\n"):
989 line
= line
[:-2] + "\n"
991 if line
.startswith("\t"):
992 # path starts and ends with a tab
994 lastTab
= path
.rfind("\t")
996 path
= path
[:lastTab
]
997 if not p4PathStartsWith(path
, self
.depotPath
):
1000 inFilesSection
= False
1002 if line
.startswith("Files:"):
1003 inFilesSection
= True
1009 def edit_template(self
, template_file
):
1010 """Invoke the editor to let the user change the submission
1011 message. Return true if okay to continue with the submit."""
1013 # if configured to skip the editing part, just submit
1014 if gitConfig("git-p4.skipSubmitEdit") == "true":
1017 # look at the modification time, to check later if the user saved
1019 mtime
= os
.stat(template_file
).st_mtime
1022 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1023 editor
= os
.environ
.get("P4EDITOR")
1025 editor
= read_pipe("git var GIT_EDITOR").strip()
1026 system(editor
+ " " + template_file
)
1028 # If the file was not saved, prompt to see if this patch should
1029 # be skipped. But skip this verification step if configured so.
1030 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1033 # modification time updated means user saved the file
1034 if os
.stat(template_file
).st_mtime
> mtime
:
1038 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1044 def applyCommit(self
, id):
1045 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1047 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1050 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1052 filesToDelete
= set()
1054 pureRenameCopy
= set()
1055 filesToChangeExecBit
= {}
1058 diff
= parseDiffTreeEntry(line
)
1059 modifier
= diff
['status']
1063 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1064 filesToChangeExecBit
[path
] = diff
['dst_mode']
1065 editedFiles
.add(path
)
1066 elif modifier
== "A":
1067 filesToAdd
.add(path
)
1068 filesToChangeExecBit
[path
] = diff
['dst_mode']
1069 if path
in filesToDelete
:
1070 filesToDelete
.remove(path
)
1071 elif modifier
== "D":
1072 filesToDelete
.add(path
)
1073 if path
in filesToAdd
:
1074 filesToAdd
.remove(path
)
1075 elif modifier
== "C":
1076 src
, dest
= diff
['src'], diff
['dst']
1077 p4_integrate(src
, dest
)
1078 pureRenameCopy
.add(dest
)
1079 if diff
['src_sha1'] != diff
['dst_sha1']:
1081 pureRenameCopy
.discard(dest
)
1082 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1084 pureRenameCopy
.discard(dest
)
1085 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1087 editedFiles
.add(dest
)
1088 elif modifier
== "R":
1089 src
, dest
= diff
['src'], diff
['dst']
1090 p4_integrate(src
, dest
)
1091 if diff
['src_sha1'] != diff
['dst_sha1']:
1094 pureRenameCopy
.add(dest
)
1095 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1097 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1099 editedFiles
.add(dest
)
1100 filesToDelete
.add(src
)
1102 die("unknown modifier %s for %s" % (modifier
, path
))
1104 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1105 patchcmd
= diffcmd
+ " | git apply "
1106 tryPatchCmd
= patchcmd
+ "--check -"
1107 applyPatchCmd
= patchcmd
+ "--check --apply -"
1108 patch_succeeded
= True
1110 if os
.system(tryPatchCmd
) != 0:
1111 fixed_rcs_keywords
= False
1112 patch_succeeded
= False
1113 print "Unfortunately applying the change failed!"
1115 # Patch failed, maybe it's just RCS keyword woes. Look through
1116 # the patch to see if that's possible.
1117 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1121 for file in editedFiles | filesToDelete
:
1122 # did this file's delta contain RCS keywords?
1123 pattern
= p4_keywords_regexp_for_file(file)
1126 # this file is a possibility...look for RCS keywords.
1127 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1128 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1129 if regexp
.search(line
):
1131 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1132 kwfiles
[file] = pattern
1135 for file in kwfiles
:
1137 print "zapping %s with %s" % (line
,pattern
)
1138 self
.patchRCSKeywords(file, kwfiles
[file])
1139 fixed_rcs_keywords
= True
1141 if fixed_rcs_keywords
:
1142 print "Retrying the patch with RCS keywords cleaned up"
1143 if os
.system(tryPatchCmd
) == 0:
1144 patch_succeeded
= True
1146 if not patch_succeeded
:
1147 print "What do you want to do?"
1149 while response
!= "s" and response
!= "a" and response
!= "w":
1150 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1151 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1153 print "Skipping! Good luck with the next patches..."
1154 for f
in editedFiles
:
1156 for f
in filesToAdd
:
1159 elif response
== "a":
1160 os
.system(applyPatchCmd
)
1161 if len(filesToAdd
) > 0:
1162 print "You may also want to call p4 add on the following files:"
1163 print " ".join(filesToAdd
)
1164 if len(filesToDelete
):
1165 print "The following files should be scheduled for deletion with p4 delete:"
1166 print " ".join(filesToDelete
)
1167 die("Please resolve and submit the conflict manually and "
1168 + "continue afterwards with git p4 submit --continue")
1169 elif response
== "w":
1170 system(diffcmd
+ " > patch.txt")
1171 print "Patch saved to patch.txt in %s !" % self
.clientPath
1172 die("Please resolve and submit the conflict manually and "
1173 "continue afterwards with git p4 submit --continue")
1175 system(applyPatchCmd
)
1177 for f
in filesToAdd
:
1179 for f
in filesToDelete
:
1183 # Set/clear executable bits
1184 for f
in filesToChangeExecBit
.keys():
1185 mode
= filesToChangeExecBit
[f
]
1186 setP4ExecBit(f
, mode
)
1188 logMessage
= extractLogMessageFromGitCommit(id)
1189 logMessage
= logMessage
.strip()
1191 template
= self
.prepareSubmitTemplate()
1193 if self
.interactive
:
1194 submitTemplate
= self
.prepareLogMessage(template
, logMessage
)
1196 if self
.preserveUser
:
1197 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1199 if os
.environ
.has_key("P4DIFF"):
1200 del(os
.environ
["P4DIFF"])
1202 for editedFile
in editedFiles
:
1203 diff
+= p4_read_pipe(['diff', '-du',
1204 wildcard_encode(editedFile
)])
1207 for newFile
in filesToAdd
:
1208 newdiff
+= "==== new file ====\n"
1209 newdiff
+= "--- /dev/null\n"
1210 newdiff
+= "+++ %s\n" % newFile
1211 f
= open(newFile
, "r")
1212 for line
in f
.readlines():
1213 newdiff
+= "+" + line
1216 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1217 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1218 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1219 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1221 separatorLine
= "######## everything below this line is just the diff #######\n"
1223 (handle
, fileName
) = tempfile
.mkstemp()
1224 tmpFile
= os
.fdopen(handle
, "w+")
1226 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1227 separatorLine
= separatorLine
.replace("\n", "\r\n")
1228 newdiff
= newdiff
.replace("\n", "\r\n")
1229 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1232 if self
.edit_template(fileName
):
1233 # read the edited message and submit
1234 tmpFile
= open(fileName
, "rb")
1235 message
= tmpFile
.read()
1237 submitTemplate
= message
[:message
.index(separatorLine
)]
1239 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1240 p4_write_pipe(['submit', '-i'], submitTemplate
)
1242 if self
.preserveUser
:
1244 # Get last changelist number. Cannot easily get it from
1245 # the submit command output as the output is
1247 changelist
= self
.lastP4Changelist()
1248 self
.modifyChangelistUser(changelist
, p4User
)
1250 # The rename/copy happened by applying a patch that created a
1251 # new file. This leaves it writable, which confuses p4.
1252 for f
in pureRenameCopy
:
1257 print "Submission cancelled, undoing p4 changes."
1258 for f
in editedFiles
:
1260 for f
in filesToAdd
:
1266 fileName
= "submit.txt"
1267 file = open(fileName
, "w+")
1268 file.write(self
.prepareLogMessage(template
, logMessage
))
1270 print ("Perforce submit template written as %s. "
1271 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1272 % (fileName
, fileName
))
1274 # Export git tags as p4 labels. Create a p4 label and then tag
1276 def exportGitTags(self
, gitTags
):
1277 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1278 if len(validLabelRegexp
) == 0:
1279 validLabelRegexp
= defaultLabelRegexp
1280 m
= re
.compile(validLabelRegexp
)
1282 for name
in gitTags
:
1284 if not m
.match(name
):
1286 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1289 # Get the p4 commit this corresponds to
1290 logMessage
= extractLogMessageFromGitCommit(name
)
1291 values
= extractSettingsGitLog(logMessage
)
1293 if not values
.has_key('change'):
1294 # a tag pointing to something not sent to p4; ignore
1296 print "git tag %s does not give a p4 commit" % name
1299 changelist
= values
['change']
1301 # Get the tag details.
1305 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1308 if re
.match(r
'tag\s+', l
):
1310 elif re
.match(r
'\s*$', l
):
1317 body
= ["lightweight tag imported by git p4\n"]
1319 # Create the label - use the same view as the client spec we are using
1320 clientSpec
= getClientSpec()
1322 labelTemplate
= "Label: %s\n" % name
1323 labelTemplate
+= "Description:\n"
1325 labelTemplate
+= "\t" + b
+ "\n"
1326 labelTemplate
+= "View:\n"
1327 for mapping
in clientSpec
.mappings
:
1328 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1330 p4_write_pipe(["label", "-i"], labelTemplate
)
1333 p4_system(["tag", "-l", name
] +
1334 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1337 print "created p4 label for tag %s" % name
1339 def run(self
, args
):
1341 self
.master
= currentGitBranch()
1342 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1343 die("Detecting current git branch failed!")
1344 elif len(args
) == 1:
1345 self
.master
= args
[0]
1346 if not branchExists(self
.master
):
1347 die("Branch %s does not exist" % self
.master
)
1351 allowSubmit
= gitConfig("git-p4.allowSubmit")
1352 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1353 die("%s is not in git-p4.allowSubmit" % self
.master
)
1355 [upstream
, settings
] = findUpstreamBranchPoint()
1356 self
.depotPath
= settings
['depot-paths'][0]
1357 if len(self
.origin
) == 0:
1358 self
.origin
= upstream
1360 if self
.preserveUser
:
1361 if not self
.canChangeChangelists():
1362 die("Cannot preserve user names without p4 super-user or admin permissions")
1365 print "Origin branch is " + self
.origin
1367 if len(self
.depotPath
) == 0:
1368 print "Internal error: cannot locate perforce depot path from existing branches"
1371 self
.useClientSpec
= False
1372 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1373 self
.useClientSpec
= True
1374 if self
.useClientSpec
:
1375 self
.clientSpecDirs
= getClientSpec()
1377 if self
.useClientSpec
:
1378 # all files are relative to the client spec
1379 self
.clientPath
= getClientRoot()
1381 self
.clientPath
= p4Where(self
.depotPath
)
1383 if self
.clientPath
== "":
1384 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1386 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1387 self
.oldWorkingDirectory
= os
.getcwd()
1389 # ensure the clientPath exists
1390 new_client_dir
= False
1391 if not os
.path
.exists(self
.clientPath
):
1392 new_client_dir
= True
1393 os
.makedirs(self
.clientPath
)
1395 chdir(self
.clientPath
)
1396 print "Synchronizing p4 checkout..."
1398 # old one was destroyed, and maybe nobody told p4
1399 p4_sync("...", "-f")
1405 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1406 commits
.append(line
.strip())
1409 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1410 self
.checkAuthorship
= False
1412 self
.checkAuthorship
= True
1414 if self
.preserveUser
:
1415 self
.checkValidP4Users(commits
)
1418 # Build up a set of options to be passed to diff when
1419 # submitting each commit to p4.
1421 if self
.detectRenames
:
1422 # command-line -M arg
1423 self
.diffOpts
= "-M"
1425 # If not explicitly set check the config variable
1426 detectRenames
= gitConfig("git-p4.detectRenames")
1428 if detectRenames
.lower() == "false" or detectRenames
== "":
1430 elif detectRenames
.lower() == "true":
1431 self
.diffOpts
= "-M"
1433 self
.diffOpts
= "-M%s" % detectRenames
1435 # no command-line arg for -C or --find-copies-harder, just
1437 detectCopies
= gitConfig("git-p4.detectCopies")
1438 if detectCopies
.lower() == "false" or detectCopies
== "":
1440 elif detectCopies
.lower() == "true":
1441 self
.diffOpts
+= " -C"
1443 self
.diffOpts
+= " -C%s" % detectCopies
1445 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1446 self
.diffOpts
+= " --find-copies-harder"
1448 while len(commits
) > 0:
1450 commits
= commits
[1:]
1451 self
.applyCommit(commit
)
1452 if not self
.interactive
:
1455 if len(commits
) == 0:
1456 print "All changes applied!"
1457 chdir(self
.oldWorkingDirectory
)
1465 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1466 self
.exportLabels
= True
1468 if self
.exportLabels
:
1469 p4Labels
= getP4Labels(self
.depotPath
)
1470 gitTags
= getGitTags()
1472 missingGitTags
= gitTags
- p4Labels
1473 self
.exportGitTags(missingGitTags
)
1478 """Represent a p4 view ("p4 help views"), and map files in a
1479 repo according to the view."""
1482 """A depot or client path, possibly containing wildcards.
1483 The only one supported is ... at the end, currently.
1484 Initialize with the full path, with //depot or //client."""
1486 def __init__(self
, path
, is_depot
):
1488 self
.is_depot
= is_depot
1489 self
.find_wildcards()
1490 # remember the prefix bit, useful for relative mappings
1491 m
= re
.match("(//[^/]+/)", self
.path
)
1493 die("Path %s does not start with //prefix/" % self
.path
)
1495 if not self
.is_depot
:
1496 # strip //client/ on client paths
1497 self
.path
= self
.path
[len(prefix
):]
1499 def find_wildcards(self
):
1500 """Make sure wildcards are valid, and set up internal
1503 self
.ends_triple_dot
= False
1504 # There are three wildcards allowed in p4 views
1505 # (see "p4 help views"). This code knows how to
1506 # handle "..." (only at the end), but cannot deal with
1507 # "%%n" or "*". Only check the depot_side, as p4 should
1508 # validate that the client_side matches too.
1509 if re
.search(r
'%%[1-9]', self
.path
):
1510 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1511 if self
.path
.find("*") >= 0:
1512 die("Can't handle * wildcards in view: %s" % self
.path
)
1513 triple_dot_index
= self
.path
.find("...")
1514 if triple_dot_index
>= 0:
1515 if triple_dot_index
!= len(self
.path
) - 3:
1516 die("Can handle only single ... wildcard, at end: %s" %
1518 self
.ends_triple_dot
= True
1520 def ensure_compatible(self
, other_path
):
1521 """Make sure the wildcards agree."""
1522 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1523 die("Both paths must end with ... if either does;\n" +
1524 "paths: %s %s" % (self
.path
, other_path
.path
))
1526 def match_wildcards(self
, test_path
):
1527 """See if this test_path matches us, and fill in the value
1528 of the wildcards if so. Returns a tuple of
1529 (True|False, wildcards[]). For now, only the ... at end
1530 is supported, so at most one wildcard."""
1531 if self
.ends_triple_dot
:
1532 dotless
= self
.path
[:-3]
1533 if test_path
.startswith(dotless
):
1534 wildcard
= test_path
[len(dotless
):]
1535 return (True, [ wildcard
])
1537 if test_path
== self
.path
:
1541 def match(self
, test_path
):
1542 """Just return if it matches; don't bother with the wildcards."""
1543 b
, _
= self
.match_wildcards(test_path
)
1546 def fill_in_wildcards(self
, wildcards
):
1547 """Return the relative path, with the wildcards filled in
1548 if there are any."""
1549 if self
.ends_triple_dot
:
1550 return self
.path
[:-3] + wildcards
[0]
1554 class Mapping(object):
1555 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1556 # depot_side is without the trailing /... if it had one
1557 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1558 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1559 self
.overlay
= overlay
# started with "+"
1560 self
.exclude
= exclude
# started with "-"
1561 assert not (self
.overlay
and self
.exclude
)
1562 self
.depot_side
.ensure_compatible(self
.client_side
)
1570 return "View.Mapping: %s%s -> %s" % \
1571 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1573 def map_depot_to_client(self
, depot_path
):
1574 """Calculate the client path if using this mapping on the
1575 given depot path; does not consider the effect of other
1576 mappings in a view. Even excluded mappings are returned."""
1577 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1580 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1589 def append(self
, view_line
):
1590 """Parse a view line, splitting it into depot and client
1591 sides. Append to self.mappings, preserving order."""
1593 # Split the view line into exactly two words. P4 enforces
1594 # structure on these lines that simplifies this quite a bit.
1596 # Either or both words may be double-quoted.
1597 # Single quotes do not matter.
1598 # Double-quote marks cannot occur inside the words.
1599 # A + or - prefix is also inside the quotes.
1600 # There are no quotes unless they contain a space.
1601 # The line is already white-space stripped.
1602 # The two words are separated by a single space.
1604 if view_line
[0] == '"':
1605 # First word is double quoted. Find its end.
1606 close_quote_index
= view_line
.find('"', 1)
1607 if close_quote_index
<= 0:
1608 die("No first-word closing quote found: %s" % view_line
)
1609 depot_side
= view_line
[1:close_quote_index
]
1610 # skip closing quote and space
1611 rhs_index
= close_quote_index
+ 1 + 1
1613 space_index
= view_line
.find(" ")
1614 if space_index
<= 0:
1615 die("No word-splitting space found: %s" % view_line
)
1616 depot_side
= view_line
[0:space_index
]
1617 rhs_index
= space_index
+ 1
1619 if view_line
[rhs_index
] == '"':
1620 # Second word is double quoted. Make sure there is a
1621 # double quote at the end too.
1622 if not view_line
.endswith('"'):
1623 die("View line with rhs quote should end with one: %s" %
1626 client_side
= view_line
[rhs_index
+1:-1]
1628 client_side
= view_line
[rhs_index
:]
1630 # prefix + means overlay on previous mapping
1632 if depot_side
.startswith("+"):
1634 depot_side
= depot_side
[1:]
1636 # prefix - means exclude this path
1638 if depot_side
.startswith("-"):
1640 depot_side
= depot_side
[1:]
1642 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1643 self
.mappings
.append(m
)
1645 def map_in_client(self
, depot_path
):
1646 """Return the relative location in the client where this
1647 depot file should live. Returns "" if the file should
1648 not be mapped in the client."""
1653 # look at later entries first
1654 for m
in self
.mappings
[::-1]:
1656 # see where will this path end up in the client
1657 p
= m
.map_depot_to_client(depot_path
)
1660 # Depot path does not belong in client. Must remember
1661 # this, as previous items should not cause files to
1662 # exist in this path either. Remember that the list is
1663 # being walked from the end, which has higher precedence.
1664 # Overlap mappings do not exclude previous mappings.
1666 paths_filled
.append(m
.client_side
)
1669 # This mapping matched; no need to search any further.
1670 # But, the mapping could be rejected if the client path
1671 # has already been claimed by an earlier mapping (i.e.
1672 # one later in the list, which we are walking backwards).
1673 already_mapped_in_client
= False
1674 for f
in paths_filled
:
1675 # this is View.Path.match
1677 already_mapped_in_client
= True
1679 if not already_mapped_in_client
:
1680 # Include this file, unless it is from a line that
1681 # explicitly said to exclude it.
1685 # a match, even if rejected, always stops the search
1690 class P4Sync(Command
, P4UserMap
):
1691 delete_actions
= ( "delete", "move/delete", "purge" )
1694 Command
.__init
__(self
)
1695 P4UserMap
.__init
__(self
)
1697 optparse
.make_option("--branch", dest
="branch"),
1698 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1699 optparse
.make_option("--changesfile", dest
="changesFile"),
1700 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1701 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1702 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1703 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1704 help="Import into refs/heads/ , not refs/remotes"),
1705 optparse
.make_option("--max-changes", dest
="maxChanges"),
1706 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1707 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1708 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1709 help="Only sync files that are included in the Perforce Client Spec")
1711 self
.description
= """Imports from Perforce into a git repository.\n
1713 //depot/my/project/ -- to import the current head
1714 //depot/my/project/@all -- to import everything
1715 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1717 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1719 self
.usage
+= " //depot/path[@revRange]"
1721 self
.createdBranches
= set()
1722 self
.committedChanges
= set()
1724 self
.detectBranches
= False
1725 self
.detectLabels
= False
1726 self
.importLabels
= False
1727 self
.changesFile
= ""
1728 self
.syncWithOrigin
= True
1729 self
.importIntoRemotes
= True
1730 self
.maxChanges
= ""
1731 self
.isWindows
= (platform
.system() == "Windows")
1732 self
.keepRepoPath
= False
1733 self
.depotPaths
= None
1734 self
.p4BranchesInGit
= []
1735 self
.cloneExclude
= []
1736 self
.useClientSpec
= False
1737 self
.useClientSpec_from_options
= False
1738 self
.clientSpecDirs
= None
1739 self
.tempBranches
= []
1740 self
.tempBranchLocation
= "git-p4-tmp"
1742 if gitConfig("git-p4.syncFromOrigin") == "false":
1743 self
.syncWithOrigin
= False
1745 # Force a checkpoint in fast-import and wait for it to finish
1746 def checkpoint(self
):
1747 self
.gitStream
.write("checkpoint\n\n")
1748 self
.gitStream
.write("progress checkpoint\n\n")
1749 out
= self
.gitOutput
.readline()
1751 print "checkpoint finished: " + out
1753 def extractFilesFromCommit(self
, commit
):
1754 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1755 for path
in self
.cloneExclude
]
1758 while commit
.has_key("depotFile%s" % fnum
):
1759 path
= commit
["depotFile%s" % fnum
]
1761 if [p
for p
in self
.cloneExclude
1762 if p4PathStartsWith(path
, p
)]:
1765 found
= [p
for p
in self
.depotPaths
1766 if p4PathStartsWith(path
, p
)]
1773 file["rev"] = commit
["rev%s" % fnum
]
1774 file["action"] = commit
["action%s" % fnum
]
1775 file["type"] = commit
["type%s" % fnum
]
1780 def stripRepoPath(self
, path
, prefixes
):
1781 if self
.useClientSpec
:
1782 return self
.clientSpecDirs
.map_in_client(path
)
1784 if self
.keepRepoPath
:
1785 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1788 if p4PathStartsWith(path
, p
):
1789 path
= path
[len(p
):]
1793 def splitFilesIntoBranches(self
, commit
):
1796 while commit
.has_key("depotFile%s" % fnum
):
1797 path
= commit
["depotFile%s" % fnum
]
1798 found
= [p
for p
in self
.depotPaths
1799 if p4PathStartsWith(path
, p
)]
1806 file["rev"] = commit
["rev%s" % fnum
]
1807 file["action"] = commit
["action%s" % fnum
]
1808 file["type"] = commit
["type%s" % fnum
]
1811 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1812 relPath
= wildcard_decode(relPath
)
1814 for branch
in self
.knownBranches
.keys():
1816 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1817 if relPath
.startswith(branch
+ "/"):
1818 if branch
not in branches
:
1819 branches
[branch
] = []
1820 branches
[branch
].append(file)
1825 # output one file from the P4 stream
1826 # - helper for streamP4Files
1828 def streamOneP4File(self
, file, contents
):
1829 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1830 relPath
= wildcard_decode(relPath
)
1832 sys
.stderr
.write("%s\n" % relPath
)
1834 (type_base
, type_mods
) = split_p4_type(file["type"])
1837 if "x" in type_mods
:
1839 if type_base
== "symlink":
1841 # p4 print on a symlink contains "target\n"; remove the newline
1842 data
= ''.join(contents
)
1843 contents
= [data
[:-1]]
1845 if type_base
== "utf16":
1846 # p4 delivers different text in the python output to -G
1847 # than it does when using "print -o", or normal p4 client
1848 # operations. utf16 is converted to ascii or utf8, perhaps.
1849 # But ascii text saved as -t utf16 is completely mangled.
1850 # Invoke print -o to get the real contents.
1851 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1854 if type_base
== "apple":
1855 # Apple filetype files will be streamed as a concatenation of
1856 # its appledouble header and the contents. This is useless
1857 # on both macs and non-macs. If using "print -q -o xx", it
1858 # will create "xx" with the data, and "%xx" with the header.
1859 # This is also not very useful.
1861 # Ideally, someday, this script can learn how to generate
1862 # appledouble files directly and import those to git, but
1863 # non-mac machines can never find a use for apple filetype.
1864 print "\nIgnoring apple filetype file %s" % file['depotFile']
1867 # Perhaps windows wants unicode, utf16 newlines translated too;
1868 # but this is not doing it.
1869 if self
.isWindows
and type_base
== "text":
1871 for data
in contents
:
1872 data
= data
.replace("\r\n", "\n")
1873 mangled
.append(data
)
1876 # Note that we do not try to de-mangle keywords on utf16 files,
1877 # even though in theory somebody may want that.
1878 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1880 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1881 text
= ''.join(contents
)
1882 text
= regexp
.sub(r
'$\1$', text
)
1885 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1890 length
= length
+ len(d
)
1892 self
.gitStream
.write("data %d\n" % length
)
1894 self
.gitStream
.write(d
)
1895 self
.gitStream
.write("\n")
1897 def streamOneP4Deletion(self
, file):
1898 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1899 relPath
= wildcard_decode(relPath
)
1901 sys
.stderr
.write("delete %s\n" % relPath
)
1902 self
.gitStream
.write("D %s\n" % relPath
)
1904 # handle another chunk of streaming data
1905 def streamP4FilesCb(self
, marshalled
):
1907 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1908 # start of a new file - output the old one first
1909 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1910 self
.stream_file
= {}
1911 self
.stream_contents
= []
1912 self
.stream_have_file_info
= False
1914 # pick up the new file information... for the
1915 # 'data' field we need to append to our array
1916 for k
in marshalled
.keys():
1918 self
.stream_contents
.append(marshalled
['data'])
1920 self
.stream_file
[k
] = marshalled
[k
]
1922 self
.stream_have_file_info
= True
1924 # Stream directly from "p4 files" into "git fast-import"
1925 def streamP4Files(self
, files
):
1931 # if using a client spec, only add the files that have
1932 # a path in the client
1933 if self
.clientSpecDirs
:
1934 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
1937 filesForCommit
.append(f
)
1938 if f
['action'] in self
.delete_actions
:
1939 filesToDelete
.append(f
)
1941 filesToRead
.append(f
)
1944 for f
in filesToDelete
:
1945 self
.streamOneP4Deletion(f
)
1947 if len(filesToRead
) > 0:
1948 self
.stream_file
= {}
1949 self
.stream_contents
= []
1950 self
.stream_have_file_info
= False
1952 # curry self argument
1953 def streamP4FilesCbSelf(entry
):
1954 self
.streamP4FilesCb(entry
)
1956 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
1958 p4CmdList(["-x", "-", "print"],
1960 cb
=streamP4FilesCbSelf
)
1963 if self
.stream_file
.has_key('depotFile'):
1964 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1966 def make_email(self
, userid
):
1967 if userid
in self
.users
:
1968 return self
.users
[userid
]
1970 return "%s <a@b>" % userid
1973 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
1975 print "writing tag %s for commit %s" % (labelName
, commit
)
1976 gitStream
.write("tag %s\n" % labelName
)
1977 gitStream
.write("from %s\n" % commit
)
1979 if labelDetails
.has_key('Owner'):
1980 owner
= labelDetails
["Owner"]
1984 # Try to use the owner of the p4 label, or failing that,
1985 # the current p4 user id.
1987 email
= self
.make_email(owner
)
1989 email
= self
.make_email(self
.p4UserId())
1990 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
1992 gitStream
.write("tagger %s\n" % tagger
)
1994 print "labelDetails=",labelDetails
1995 if labelDetails
.has_key('Description'):
1996 description
= labelDetails
['Description']
1998 description
= 'Label from git p4'
2000 gitStream
.write("data %d\n" % len(description
))
2001 gitStream
.write(description
)
2002 gitStream
.write("\n")
2004 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
2005 epoch
= details
["time"]
2006 author
= details
["user"]
2007 self
.branchPrefixes
= branchPrefixes
2010 print "commit into %s" % branch
2012 # start with reading files; if that fails, we should not
2016 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2017 new_files
.append (f
)
2019 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2021 self
.gitStream
.write("commit %s\n" % branch
)
2022 # gitStream.write("mark :%s\n" % details["change"])
2023 self
.committedChanges
.add(int(details
["change"]))
2025 if author
not in self
.users
:
2026 self
.getUserMapFromPerforceServer()
2027 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2029 self
.gitStream
.write("committer %s\n" % committer
)
2031 self
.gitStream
.write("data <<EOT\n")
2032 self
.gitStream
.write(details
["desc"])
2033 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2034 % (','.join (branchPrefixes
), details
["change"]))
2035 if len(details
['options']) > 0:
2036 self
.gitStream
.write(": options = %s" % details
['options'])
2037 self
.gitStream
.write("]\nEOT\n\n")
2041 print "parent %s" % parent
2042 self
.gitStream
.write("from %s\n" % parent
)
2044 self
.streamP4Files(new_files
)
2045 self
.gitStream
.write("\n")
2047 change
= int(details
["change"])
2049 if self
.labels
.has_key(change
):
2050 label
= self
.labels
[change
]
2051 labelDetails
= label
[0]
2052 labelRevisions
= label
[1]
2054 print "Change %s is labelled %s" % (change
, labelDetails
)
2056 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2057 for p
in branchPrefixes
])
2059 if len(files
) == len(labelRevisions
):
2063 if info
["action"] in self
.delete_actions
:
2065 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2067 if cleanedFiles
== labelRevisions
:
2068 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2072 print ("Tag %s does not match with change %s: files do not match."
2073 % (labelDetails
["label"], change
))
2077 print ("Tag %s does not match with change %s: file count is different."
2078 % (labelDetails
["label"], change
))
2080 # Build a dictionary of changelists and labels, for "detect-labels" option.
2081 def getLabels(self
):
2084 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2085 if len(l
) > 0 and not self
.silent
:
2086 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2089 label
= output
["label"]
2093 print "Querying files for label %s" % label
2094 for file in p4CmdList(["files"] +
2095 ["%s...@%s" % (p
, label
)
2096 for p
in self
.depotPaths
]):
2097 revisions
[file["depotFile"]] = file["rev"]
2098 change
= int(file["change"])
2099 if change
> newestChange
:
2100 newestChange
= change
2102 self
.labels
[newestChange
] = [output
, revisions
]
2105 print "Label changes: %s" % self
.labels
.keys()
2107 # Import p4 labels as git tags. A direct mapping does not
2108 # exist, so assume that if all the files are at the same revision
2109 # then we can use that, or it's something more complicated we should
2111 def importP4Labels(self
, stream
, p4Labels
):
2113 print "import p4 labels: " + ' '.join(p4Labels
)
2115 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2116 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2117 if len(validLabelRegexp
) == 0:
2118 validLabelRegexp
= defaultLabelRegexp
2119 m
= re
.compile(validLabelRegexp
)
2121 for name
in p4Labels
:
2124 if not m
.match(name
):
2126 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2129 if name
in ignoredP4Labels
:
2132 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2134 # get the most recent changelist for each file in this label
2135 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2136 for p
in self
.depotPaths
])
2138 if change
.has_key('change'):
2139 # find the corresponding git commit; take the oldest commit
2140 changelist
= int(change
['change'])
2141 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2142 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2143 if len(gitCommit
) == 0:
2144 print "could not find git commit for changelist %d" % changelist
2146 gitCommit
= gitCommit
.strip()
2148 # Convert from p4 time format
2150 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2152 print "Could not convert label time %s" % labelDetail
['Update']
2155 when
= int(time
.mktime(tmwhen
))
2156 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2158 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2161 print "Label %s has no changelists - possibly deleted?" % name
2164 # We can't import this label; don't try again as it will get very
2165 # expensive repeatedly fetching all the files for labels that will
2166 # never be imported. If the label is moved in the future, the
2167 # ignore will need to be removed manually.
2168 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2170 def guessProjectName(self
):
2171 for p
in self
.depotPaths
:
2174 p
= p
[p
.strip().rfind("/") + 1:]
2175 if not p
.endswith("/"):
2179 def getBranchMapping(self
):
2180 lostAndFoundBranches
= set()
2182 user
= gitConfig("git-p4.branchUser")
2184 command
= "branches -u %s" % user
2186 command
= "branches"
2188 for info
in p4CmdList(command
):
2189 details
= p4Cmd(["branch", "-o", info
["branch"]])
2191 while details
.has_key("View%s" % viewIdx
):
2192 paths
= details
["View%s" % viewIdx
].split(" ")
2193 viewIdx
= viewIdx
+ 1
2194 # require standard //depot/foo/... //depot/bar/... mapping
2195 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2198 destination
= paths
[1]
2200 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2201 source
= source
[len(self
.depotPaths
[0]):-4]
2202 destination
= destination
[len(self
.depotPaths
[0]):-4]
2204 if destination
in self
.knownBranches
:
2206 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2207 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2210 self
.knownBranches
[destination
] = source
2212 lostAndFoundBranches
.discard(destination
)
2214 if source
not in self
.knownBranches
:
2215 lostAndFoundBranches
.add(source
)
2217 # Perforce does not strictly require branches to be defined, so we also
2218 # check git config for a branch list.
2220 # Example of branch definition in git config file:
2222 # branchList=main:branchA
2223 # branchList=main:branchB
2224 # branchList=branchA:branchC
2225 configBranches
= gitConfigList("git-p4.branchList")
2226 for branch
in configBranches
:
2228 (source
, destination
) = branch
.split(":")
2229 self
.knownBranches
[destination
] = source
2231 lostAndFoundBranches
.discard(destination
)
2233 if source
not in self
.knownBranches
:
2234 lostAndFoundBranches
.add(source
)
2237 for branch
in lostAndFoundBranches
:
2238 self
.knownBranches
[branch
] = branch
2240 def getBranchMappingFromGitBranches(self
):
2241 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2242 for branch
in branches
.keys():
2243 if branch
== "master":
2246 branch
= branch
[len(self
.projectName
):]
2247 self
.knownBranches
[branch
] = branch
2249 def listExistingP4GitBranches(self
):
2250 # branches holds mapping from name to commit
2251 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2252 self
.p4BranchesInGit
= branches
.keys()
2253 for branch
in branches
.keys():
2254 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2256 def updateOptionDict(self
, d
):
2258 if self
.keepRepoPath
:
2259 option_keys
['keepRepoPath'] = 1
2261 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2263 def readOptions(self
, d
):
2264 self
.keepRepoPath
= (d
.has_key('options')
2265 and ('keepRepoPath' in d
['options']))
2267 def gitRefForBranch(self
, branch
):
2268 if branch
== "main":
2269 return self
.refPrefix
+ "master"
2271 if len(branch
) <= 0:
2274 return self
.refPrefix
+ self
.projectName
+ branch
2276 def gitCommitByP4Change(self
, ref
, change
):
2278 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2281 latestCommit
= parseRevision(ref
)
2285 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2286 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2291 log
= extractLogMessageFromGitCommit(next
)
2292 settings
= extractSettingsGitLog(log
)
2293 currentChange
= int(settings
['change'])
2295 print "current change %s" % currentChange
2297 if currentChange
== change
:
2299 print "found %s" % next
2302 if currentChange
< change
:
2303 earliestCommit
= "^%s" % next
2305 latestCommit
= "%s" % next
2309 def importNewBranch(self
, branch
, maxChange
):
2310 # make fast-import flush all changes to disk and update the refs using the checkpoint
2311 # command so that we can try to find the branch parent in the git history
2312 self
.gitStream
.write("checkpoint\n\n");
2313 self
.gitStream
.flush();
2314 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2315 range = "@1,%s" % maxChange
2316 #print "prefix" + branchPrefix
2317 changes
= p4ChangesForPaths([branchPrefix
], range)
2318 if len(changes
) <= 0:
2320 firstChange
= changes
[0]
2321 #print "first change in branch: %s" % firstChange
2322 sourceBranch
= self
.knownBranches
[branch
]
2323 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2324 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2325 #print "source " + sourceBranch
2327 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2328 #print "branch parent: %s" % branchParentChange
2329 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2330 if len(gitParent
) > 0:
2331 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2332 #print "parent git commit: %s" % gitParent
2334 self
.importChanges(changes
)
2337 def searchParent(self
, parent
, branch
, target
):
2339 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2341 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2344 print "Found parent of %s in commit %s" % (branch
, blob
)
2351 def importChanges(self
, changes
):
2353 for change
in changes
:
2354 description
= p4Cmd(["describe", str(change
)])
2355 self
.updateOptionDict(description
)
2358 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2363 if self
.detectBranches
:
2364 branches
= self
.splitFilesIntoBranches(description
)
2365 for branch
in branches
.keys():
2367 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2371 filesForCommit
= branches
[branch
]
2374 print "branch is %s" % branch
2376 self
.updatedBranches
.add(branch
)
2378 if branch
not in self
.createdBranches
:
2379 self
.createdBranches
.add(branch
)
2380 parent
= self
.knownBranches
[branch
]
2381 if parent
== branch
:
2384 fullBranch
= self
.projectName
+ branch
2385 if fullBranch
not in self
.p4BranchesInGit
:
2387 print("\n Importing new branch %s" % fullBranch
);
2388 if self
.importNewBranch(branch
, change
- 1):
2390 self
.p4BranchesInGit
.append(fullBranch
)
2392 print("\n Resuming with change %s" % change
);
2395 print "parent determined through known branches: %s" % parent
2397 branch
= self
.gitRefForBranch(branch
)
2398 parent
= self
.gitRefForBranch(parent
)
2401 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2403 if len(parent
) == 0 and branch
in self
.initialParents
:
2404 parent
= self
.initialParents
[branch
]
2405 del self
.initialParents
[branch
]
2409 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2411 print "Creating temporary branch: " + tempBranch
2412 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2413 self
.tempBranches
.append(tempBranch
)
2415 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2417 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2420 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2421 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2423 files
= self
.extractFilesFromCommit(description
)
2424 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2426 self
.initialParent
= ""
2428 print self
.gitError
.read()
2431 def importHeadRevision(self
, revision
):
2432 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2435 details
["user"] = "git perforce import user"
2436 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2437 % (' '.join(self
.depotPaths
), revision
))
2438 details
["change"] = revision
2442 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2444 for info
in p4CmdList(["files"] + fileArgs
):
2446 if 'code' in info
and info
['code'] == 'error':
2447 sys
.stderr
.write("p4 returned an error: %s\n"
2449 if info
['data'].find("must refer to client") >= 0:
2450 sys
.stderr
.write("This particular p4 error is misleading.\n")
2451 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2452 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2454 if 'p4ExitCode' in info
:
2455 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2459 change
= int(info
["change"])
2460 if change
> newestRevision
:
2461 newestRevision
= change
2463 if info
["action"] in self
.delete_actions
:
2464 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2465 #fileCnt = fileCnt + 1
2468 for prop
in ["depotFile", "rev", "action", "type" ]:
2469 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2471 fileCnt
= fileCnt
+ 1
2473 details
["change"] = newestRevision
2475 # Use time from top-most change so that all git p4 clones of
2476 # the same p4 repo have the same commit SHA1s.
2477 res
= p4CmdList("describe -s %d" % newestRevision
)
2480 if r
.has_key('time'):
2481 newestTime
= int(r
['time'])
2482 if newestTime
is None:
2483 die("\"describe -s\" on newest change %d did not give a time")
2484 details
["time"] = newestTime
2486 self
.updateOptionDict(details
)
2488 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2490 print "IO error with git fast-import. Is your git version recent enough?"
2491 print self
.gitError
.read()
2494 def run(self
, args
):
2495 self
.depotPaths
= []
2496 self
.changeRange
= ""
2497 self
.initialParent
= ""
2498 self
.previousDepotPaths
= []
2500 # map from branch depot path to parent branch
2501 self
.knownBranches
= {}
2502 self
.initialParents
= {}
2503 self
.hasOrigin
= originP4BranchesExist()
2504 if not self
.syncWithOrigin
:
2505 self
.hasOrigin
= False
2507 if self
.importIntoRemotes
:
2508 self
.refPrefix
= "refs/remotes/p4/"
2510 self
.refPrefix
= "refs/heads/p4/"
2512 if self
.syncWithOrigin
and self
.hasOrigin
:
2514 print "Syncing with origin first by calling git fetch origin"
2515 system("git fetch origin")
2517 if len(self
.branch
) == 0:
2518 self
.branch
= self
.refPrefix
+ "master"
2519 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2520 system("git update-ref %s refs/heads/p4" % self
.branch
)
2521 system("git branch -D p4");
2522 # create it /after/ importing, when master exists
2523 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2524 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2526 # accept either the command-line option, or the configuration variable
2527 if self
.useClientSpec
:
2528 # will use this after clone to set the variable
2529 self
.useClientSpec_from_options
= True
2531 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2532 self
.useClientSpec
= True
2533 if self
.useClientSpec
:
2534 self
.clientSpecDirs
= getClientSpec()
2536 # TODO: should always look at previous commits,
2537 # merge with previous imports, if possible.
2540 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2541 self
.listExistingP4GitBranches()
2543 if len(self
.p4BranchesInGit
) > 1:
2545 print "Importing from/into multiple branches"
2546 self
.detectBranches
= True
2549 print "branches: %s" % self
.p4BranchesInGit
2552 for branch
in self
.p4BranchesInGit
:
2553 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2555 settings
= extractSettingsGitLog(logMsg
)
2557 self
.readOptions(settings
)
2558 if (settings
.has_key('depot-paths')
2559 and settings
.has_key ('change')):
2560 change
= int(settings
['change']) + 1
2561 p4Change
= max(p4Change
, change
)
2563 depotPaths
= sorted(settings
['depot-paths'])
2564 if self
.previousDepotPaths
== []:
2565 self
.previousDepotPaths
= depotPaths
2568 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2569 prev_list
= prev
.split("/")
2570 cur_list
= cur
.split("/")
2571 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2572 if cur_list
[i
] <> prev_list
[i
]:
2576 paths
.append ("/".join(cur_list
[:i
+ 1]))
2578 self
.previousDepotPaths
= paths
2581 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2582 self
.changeRange
= "@%s,#head" % p4Change
2583 if not self
.detectBranches
:
2584 self
.initialParent
= parseRevision(self
.branch
)
2585 if not self
.silent
and not self
.detectBranches
:
2586 print "Performing incremental import into %s git branch" % self
.branch
2588 if not self
.branch
.startswith("refs/"):
2589 self
.branch
= "refs/heads/" + self
.branch
2591 if len(args
) == 0 and self
.depotPaths
:
2593 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2595 if self
.depotPaths
and self
.depotPaths
!= args
:
2596 print ("previous import used depot path %s and now %s was specified. "
2597 "This doesn't work!" % (' '.join (self
.depotPaths
),
2601 self
.depotPaths
= sorted(args
)
2606 # Make sure no revision specifiers are used when --changesfile
2608 bad_changesfile
= False
2609 if len(self
.changesFile
) > 0:
2610 for p
in self
.depotPaths
:
2611 if p
.find("@") >= 0 or p
.find("#") >= 0:
2612 bad_changesfile
= True
2615 die("Option --changesfile is incompatible with revision specifiers")
2618 for p
in self
.depotPaths
:
2619 if p
.find("@") != -1:
2620 atIdx
= p
.index("@")
2621 self
.changeRange
= p
[atIdx
:]
2622 if self
.changeRange
== "@all":
2623 self
.changeRange
= ""
2624 elif ',' not in self
.changeRange
:
2625 revision
= self
.changeRange
2626 self
.changeRange
= ""
2628 elif p
.find("#") != -1:
2629 hashIdx
= p
.index("#")
2630 revision
= p
[hashIdx
:]
2632 elif self
.previousDepotPaths
== []:
2633 # pay attention to changesfile, if given, else import
2634 # the entire p4 tree at the head revision
2635 if len(self
.changesFile
) == 0:
2638 p
= re
.sub ("\.\.\.$", "", p
)
2639 if not p
.endswith("/"):
2644 self
.depotPaths
= newPaths
2646 self
.loadUserMapFromCache()
2648 if self
.detectLabels
:
2651 if self
.detectBranches
:
2652 ## FIXME - what's a P4 projectName ?
2653 self
.projectName
= self
.guessProjectName()
2656 self
.getBranchMappingFromGitBranches()
2658 self
.getBranchMapping()
2660 print "p4-git branches: %s" % self
.p4BranchesInGit
2661 print "initial parents: %s" % self
.initialParents
2662 for b
in self
.p4BranchesInGit
:
2666 b
= b
[len(self
.projectName
):]
2667 self
.createdBranches
.add(b
)
2669 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2671 importProcess
= subprocess
.Popen(["git", "fast-import"],
2672 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2673 stderr
=subprocess
.PIPE
);
2674 self
.gitOutput
= importProcess
.stdout
2675 self
.gitStream
= importProcess
.stdin
2676 self
.gitError
= importProcess
.stderr
2679 self
.importHeadRevision(revision
)
2683 if len(self
.changesFile
) > 0:
2684 output
= open(self
.changesFile
).readlines()
2687 changeSet
.add(int(line
))
2689 for change
in changeSet
:
2690 changes
.append(change
)
2694 # catch "git p4 sync" with no new branches, in a repo that
2695 # does not have any existing p4 branches
2696 if len(args
) == 0 and not self
.p4BranchesInGit
:
2697 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2699 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2701 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2703 if len(self
.maxChanges
) > 0:
2704 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2706 if len(changes
) == 0:
2708 print "No changes to import!"
2710 if not self
.silent
and not self
.detectBranches
:
2711 print "Import destination: %s" % self
.branch
2713 self
.updatedBranches
= set()
2715 self
.importChanges(changes
)
2719 if len(self
.updatedBranches
) > 0:
2720 sys
.stdout
.write("Updated branches: ")
2721 for b
in self
.updatedBranches
:
2722 sys
.stdout
.write("%s " % b
)
2723 sys
.stdout
.write("\n")
2725 if gitConfig("git-p4.importLabels", "--bool") == "true":
2726 self
.importLabels
= True
2728 if self
.importLabels
:
2729 p4Labels
= getP4Labels(self
.depotPaths
)
2730 gitTags
= getGitTags()
2732 missingP4Labels
= p4Labels
- gitTags
2733 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2735 self
.gitStream
.close()
2736 if importProcess
.wait() != 0:
2737 die("fast-import failed: %s" % self
.gitError
.read())
2738 self
.gitOutput
.close()
2739 self
.gitError
.close()
2741 # Cleanup temporary branches created during import
2742 if self
.tempBranches
!= []:
2743 for branch
in self
.tempBranches
:
2744 read_pipe("git update-ref -d %s" % branch
)
2745 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2749 class P4Rebase(Command
):
2751 Command
.__init
__(self
)
2753 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2755 self
.importLabels
= False
2756 self
.description
= ("Fetches the latest revision from perforce and "
2757 + "rebases the current work (branch) against it")
2759 def run(self
, args
):
2761 sync
.importLabels
= self
.importLabels
2764 return self
.rebase()
2767 if os
.system("git update-index --refresh") != 0:
2768 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2769 if len(read_pipe("git diff-index HEAD --")) > 0:
2770 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2772 [upstream
, settings
] = findUpstreamBranchPoint()
2773 if len(upstream
) == 0:
2774 die("Cannot find upstream branchpoint for rebase")
2776 # the branchpoint may be p4/foo~3, so strip off the parent
2777 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2779 print "Rebasing the current branch onto %s" % upstream
2780 oldHead
= read_pipe("git rev-parse HEAD").strip()
2781 system("git rebase %s" % upstream
)
2782 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2785 class P4Clone(P4Sync
):
2787 P4Sync
.__init
__(self
)
2788 self
.description
= "Creates a new git repository and imports from Perforce into it"
2789 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2791 optparse
.make_option("--destination", dest
="cloneDestination",
2792 action
='store', default
=None,
2793 help="where to leave result of the clone"),
2794 optparse
.make_option("-/", dest
="cloneExclude",
2795 action
="append", type="string",
2796 help="exclude depot path"),
2797 optparse
.make_option("--bare", dest
="cloneBare",
2798 action
="store_true", default
=False),
2800 self
.cloneDestination
= None
2801 self
.needsGit
= False
2802 self
.cloneBare
= False
2804 # This is required for the "append" cloneExclude action
2805 def ensure_value(self
, attr
, value
):
2806 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2807 setattr(self
, attr
, value
)
2808 return getattr(self
, attr
)
2810 def defaultDestination(self
, args
):
2811 ## TODO: use common prefix of args?
2813 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2814 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2815 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2816 depotDir
= re
.sub(r
"/$", "", depotDir
)
2817 return os
.path
.split(depotDir
)[1]
2819 def run(self
, args
):
2823 if self
.keepRepoPath
and not self
.cloneDestination
:
2824 sys
.stderr
.write("Must specify destination for --keep-path\n")
2829 if not self
.cloneDestination
and len(depotPaths
) > 1:
2830 self
.cloneDestination
= depotPaths
[-1]
2831 depotPaths
= depotPaths
[:-1]
2833 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2834 for p
in depotPaths
:
2835 if not p
.startswith("//"):
2838 if not self
.cloneDestination
:
2839 self
.cloneDestination
= self
.defaultDestination(args
)
2841 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2843 if not os
.path
.exists(self
.cloneDestination
):
2844 os
.makedirs(self
.cloneDestination
)
2845 chdir(self
.cloneDestination
)
2847 init_cmd
= [ "git", "init" ]
2849 init_cmd
.append("--bare")
2850 subprocess
.check_call(init_cmd
)
2852 if not P4Sync
.run(self
, depotPaths
):
2854 if self
.branch
!= "master":
2855 if self
.importIntoRemotes
:
2856 masterbranch
= "refs/remotes/p4/master"
2858 masterbranch
= "refs/heads/p4/master"
2859 if gitBranchExists(masterbranch
):
2860 system("git branch master %s" % masterbranch
)
2861 if not self
.cloneBare
:
2862 system("git checkout -f")
2864 print "Could not detect main branch. No checkout/master branch created."
2866 # auto-set this variable if invoked with --use-client-spec
2867 if self
.useClientSpec_from_options
:
2868 system("git config --bool git-p4.useclientspec true")
2872 class P4Branches(Command
):
2874 Command
.__init
__(self
)
2876 self
.description
= ("Shows the git branches that hold imports and their "
2877 + "corresponding perforce depot paths")
2878 self
.verbose
= False
2880 def run(self
, args
):
2881 if originP4BranchesExist():
2882 createOrUpdateBranchesFromOrigin()
2884 cmdline
= "git rev-parse --symbolic "
2885 cmdline
+= " --remotes"
2887 for line
in read_pipe_lines(cmdline
):
2890 if not line
.startswith('p4/') or line
== "p4/HEAD":
2894 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2895 settings
= extractSettingsGitLog(log
)
2897 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2900 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2902 optparse
.IndentedHelpFormatter
.__init
__(self
)
2904 def format_description(self
, description
):
2906 return description
+ "\n"
2910 def printUsage(commands
):
2911 print "usage: %s <command> [options]" % sys
.argv
[0]
2913 print "valid commands: %s" % ", ".join(commands
)
2915 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2920 "submit" : P4Submit
,
2921 "commit" : P4Submit
,
2923 "rebase" : P4Rebase
,
2925 "rollback" : P4RollBack
,
2926 "branches" : P4Branches
2931 if len(sys
.argv
[1:]) == 0:
2932 printUsage(commands
.keys())
2936 cmdName
= sys
.argv
[1]
2938 klass
= commands
[cmdName
]
2941 print "unknown command %s" % cmdName
2943 printUsage(commands
.keys())
2946 options
= cmd
.options
2947 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2951 options
.append(optparse
.make_option("--verbose", dest
="verbose", action
="store_true"))
2953 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2955 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2957 description
= cmd
.description
,
2958 formatter
= HelpFormatter())
2960 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2962 verbose
= cmd
.verbose
2964 if cmd
.gitdir
== None:
2965 cmd
.gitdir
= os
.path
.abspath(".git")
2966 if not isValidGitDir(cmd
.gitdir
):
2967 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2968 if os
.path
.exists(cmd
.gitdir
):
2969 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2973 if not isValidGitDir(cmd
.gitdir
):
2974 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2975 cmd
.gitdir
+= "/.git"
2977 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2979 os
.environ
["GIT_DIR"] = cmd
.gitdir
2981 if not cmd
.run(args
):
2986 if __name__
== '__main__':