3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
18 def p4_build_cmd(cmd
):
19 """Build a suitable p4 command line.
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
25 real_cmd
= "%s " % "p4"
27 user
= gitConfig("git-p4.user")
29 real_cmd
+= "-u %s " % user
31 password
= gitConfig("git-p4.password")
33 real_cmd
+= "-P %s " % password
35 port
= gitConfig("git-p4.port")
37 real_cmd
+= "-p %s " % port
39 host
= gitConfig("git-p4.host")
41 real_cmd
+= "-h %s " % host
43 client
= gitConfig("git-p4.client")
45 real_cmd
+= "-c %s " % client
47 real_cmd
+= "%s" % (cmd
)
61 sys
.stderr
.write(msg
+ "\n")
64 def write_pipe(c
, str):
66 sys
.stderr
.write('Writing pipe: %s\n' % c
)
68 pipe
= os
.popen(c
, 'w')
71 die('Command failed: %s' % c
)
75 def p4_write_pipe(c
, str):
76 real_cmd
= p4_build_cmd(c
)
77 return write_pipe(real_cmd
, str)
79 def read_pipe(c
, ignore_error
=False):
81 sys
.stderr
.write('Reading pipe: %s\n' % c
)
83 pipe
= os
.popen(c
, 'rb')
85 if pipe
.close() and not ignore_error
:
86 die('Command failed: %s' % c
)
90 def p4_read_pipe(c
, ignore_error
=False):
91 real_cmd
= p4_build_cmd(c
)
92 return read_pipe(real_cmd
, ignore_error
)
94 def read_pipe_lines(c
):
96 sys
.stderr
.write('Reading pipe: %s\n' % c
)
97 ## todo: check return status
98 pipe
= os
.popen(c
, 'rb')
99 val
= pipe
.readlines()
101 die('Command failed: %s' % c
)
105 def p4_read_pipe_lines(c
):
106 """Specifically invoke p4 on the command supplied. """
107 real_cmd
= p4_build_cmd(c
)
108 return read_pipe_lines(real_cmd
)
112 sys
.stderr
.write("executing %s\n" % cmd
)
113 if os
.system(cmd
) != 0:
114 die("command failed: %s" % cmd
)
117 """Specifically invoke p4 as the system command. """
118 real_cmd
= p4_build_cmd(cmd
)
119 return system(real_cmd
)
122 # Canonicalize the p4 type and return a tuple of the
123 # base type, plus any modifiers. See "p4 help filetypes"
124 # for a list and explanation.
126 def split_p4_type(p4type
):
128 p4_filetypes_historical
= {
129 "ctempobj": "binary+Sw",
135 "tempobj": "binary+FSw",
136 "ubinary": "binary+F",
137 "uresource": "resource+F",
138 "uxbinary": "binary+Fx",
139 "xbinary": "binary+x",
141 "xtempobj": "binary+Swx",
143 "xunicode": "unicode+x",
146 if p4type
in p4_filetypes_historical
:
147 p4type
= p4_filetypes_historical
[p4type
]
149 s
= p4type
.split("+")
157 def setP4ExecBit(file, mode
):
158 # Reopens an already open file and changes the execute bit to match
159 # the execute bit setting in the passed in mode.
163 if not isModeExec(mode
):
164 p4Type
= getP4OpenedType(file)
165 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
166 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
167 if p4Type
[-1] == "+":
168 p4Type
= p4Type
[0:-1]
170 p4_system("reopen -t %s %s" % (p4Type
, file))
172 def getP4OpenedType(file):
173 # Returns the perforce file type for the given file.
175 result
= p4_read_pipe("opened %s" % file)
176 match
= re
.match(".*\((.+)\)\r?$", result
)
178 return match
.group(1)
180 die("Could not determine file type for %s (result: '%s')" % (file, result
))
182 def diffTreePattern():
183 # This is a simple generator for the diff tree regex pattern. This could be
184 # a class variable if this and parseDiffTreeEntry were a part of a class.
185 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
189 def parseDiffTreeEntry(entry
):
190 """Parses a single diff tree entry into its component elements.
192 See git-diff-tree(1) manpage for details about the format of the diff
193 output. This method returns a dictionary with the following elements:
195 src_mode - The mode of the source file
196 dst_mode - The mode of the destination file
197 src_sha1 - The sha1 for the source file
198 dst_sha1 - The sha1 fr the destination file
199 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
200 status_score - The score for the status (applicable for 'C' and 'R'
201 statuses). This is None if there is no score.
202 src - The path for the source file.
203 dst - The path for the destination file. This is only present for
204 copy or renames. If it is not present, this is None.
206 If the pattern is not matched, None is returned."""
208 match
= diffTreePattern().next().match(entry
)
211 'src_mode': match
.group(1),
212 'dst_mode': match
.group(2),
213 'src_sha1': match
.group(3),
214 'dst_sha1': match
.group(4),
215 'status': match
.group(5),
216 'status_score': match
.group(6),
217 'src': match
.group(7),
218 'dst': match
.group(10)
222 def isModeExec(mode
):
223 # Returns True if the given git mode represents an executable file,
225 return mode
[-3:] == "755"
227 def isModeExecChanged(src_mode
, dst_mode
):
228 return isModeExec(src_mode
) != isModeExec(dst_mode
)
230 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
231 cmd
= p4_build_cmd("-G %s" % (cmd
))
233 sys
.stderr
.write("Opening pipe: %s\n" % cmd
)
235 # Use a temporary file to avoid deadlocks without
236 # subprocess.communicate(), which would put another copy
237 # of stdout into memory.
239 if stdin
is not None:
240 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
241 stdin_file
.write(stdin
)
245 p4
= subprocess
.Popen(cmd
, shell
=True,
247 stdout
=subprocess
.PIPE
)
252 entry
= marshal
.load(p4
.stdout
)
262 entry
["p4ExitCode"] = exitCode
268 list = p4CmdList(cmd
)
274 def p4Where(depotPath
):
275 if not depotPath
.endswith("/"):
277 depotPath
= depotPath
+ "..."
278 outputList
= p4CmdList("where %s" % depotPath
)
280 for entry
in outputList
:
281 if "depotFile" in entry
:
282 if entry
["depotFile"] == depotPath
:
285 elif "data" in entry
:
286 data
= entry
.get("data")
287 space
= data
.find(" ")
288 if data
[:space
] == depotPath
:
293 if output
["code"] == "error":
297 clientPath
= output
.get("path")
298 elif "data" in output
:
299 data
= output
.get("data")
300 lastSpace
= data
.rfind(" ")
301 clientPath
= data
[lastSpace
+ 1:]
303 if clientPath
.endswith("..."):
304 clientPath
= clientPath
[:-3]
307 def currentGitBranch():
308 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
310 def isValidGitDir(path
):
311 if (os
.path
.exists(path
+ "/HEAD")
312 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
316 def parseRevision(ref
):
317 return read_pipe("git rev-parse %s" % ref
).strip()
319 def extractLogMessageFromGitCommit(commit
):
322 ## fixme: title is first line of commit, not 1st paragraph.
324 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
333 def extractSettingsGitLog(log
):
335 for line
in log
.split("\n"):
337 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
341 assignments
= m
.group(1).split (':')
342 for a
in assignments
:
344 key
= vals
[0].strip()
345 val
= ('='.join (vals
[1:])).strip()
346 if val
.endswith ('\"') and val
.startswith('"'):
351 paths
= values
.get("depot-paths")
353 paths
= values
.get("depot-path")
355 values
['depot-paths'] = paths
.split(',')
358 def gitBranchExists(branch
):
359 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
360 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
361 return proc
.wait() == 0;
364 def gitConfig(key
, args
= None): # set args to "--bool", for instance
365 if not _gitConfig
.has_key(key
):
368 argsFilter
= "%s " % args
369 cmd
= "git config %s%s" % (argsFilter
, key
)
370 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
371 return _gitConfig
[key
]
373 def gitConfigList(key
):
374 if not _gitConfig
.has_key(key
):
375 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
376 return _gitConfig
[key
]
378 def p4BranchesInGit(branchesAreInRemotes
= True):
381 cmdline
= "git rev-parse --symbolic "
382 if branchesAreInRemotes
:
383 cmdline
+= " --remotes"
385 cmdline
+= " --branches"
387 for line
in read_pipe_lines(cmdline
):
390 ## only import to p4/
391 if not line
.startswith('p4/') or line
== "p4/HEAD":
396 branch
= re
.sub ("^p4/", "", line
)
398 branches
[branch
] = parseRevision(line
)
401 def findUpstreamBranchPoint(head
= "HEAD"):
402 branches
= p4BranchesInGit()
403 # map from depot-path to branch name
404 branchByDepotPath
= {}
405 for branch
in branches
.keys():
406 tip
= branches
[branch
]
407 log
= extractLogMessageFromGitCommit(tip
)
408 settings
= extractSettingsGitLog(log
)
409 if settings
.has_key("depot-paths"):
410 paths
= ",".join(settings
["depot-paths"])
411 branchByDepotPath
[paths
] = "remotes/p4/" + branch
415 while parent
< 65535:
416 commit
= head
+ "~%s" % parent
417 log
= extractLogMessageFromGitCommit(commit
)
418 settings
= extractSettingsGitLog(log
)
419 if settings
.has_key("depot-paths"):
420 paths
= ",".join(settings
["depot-paths"])
421 if branchByDepotPath
.has_key(paths
):
422 return [branchByDepotPath
[paths
], settings
]
426 return ["", settings
]
428 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
430 print ("Creating/updating branch(es) in %s based on origin branch(es)"
433 originPrefix
= "origin/p4/"
435 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
437 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
440 headName
= line
[len(originPrefix
):]
441 remoteHead
= localRefPrefix
+ headName
444 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
445 if (not original
.has_key('depot-paths')
446 or not original
.has_key('change')):
450 if not gitBranchExists(remoteHead
):
452 print "creating %s" % remoteHead
455 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
456 if settings
.has_key('change') > 0:
457 if settings
['depot-paths'] == original
['depot-paths']:
458 originP4Change
= int(original
['change'])
459 p4Change
= int(settings
['change'])
460 if originP4Change
> p4Change
:
461 print ("%s (%s) is newer than %s (%s). "
462 "Updating p4 branch from origin."
463 % (originHead
, originP4Change
,
464 remoteHead
, p4Change
))
467 print ("Ignoring: %s was imported from %s while "
468 "%s was imported from %s"
469 % (originHead
, ','.join(original
['depot-paths']),
470 remoteHead
, ','.join(settings
['depot-paths'])))
473 system("git update-ref %s %s" % (remoteHead
, originHead
))
475 def originP4BranchesExist():
476 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
478 def p4ChangesForPaths(depotPaths
, changeRange
):
480 output
= p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p
, changeRange
)
481 for p
in depotPaths
]))
485 changeNum
= int(line
.split(" ")[1])
486 changes
[changeNum
] = True
488 changelist
= changes
.keys()
492 def p4PathStartsWith(path
, prefix
):
493 # This method tries to remedy a potential mixed-case issue:
495 # If UserA adds //depot/DirA/file1
496 # and UserB adds //depot/dira/file2
498 # we may or may not have a problem. If you have core.ignorecase=true,
499 # we treat DirA and dira as the same directory
500 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
502 return path
.lower().startswith(prefix
.lower())
503 return path
.startswith(prefix
)
507 self
.usage
= "usage: %prog [options]"
512 self
.userMapFromPerforceServer
= False
514 def getUserCacheFilename(self
):
515 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
516 return home
+ "/.gitp4-usercache.txt"
518 def getUserMapFromPerforceServer(self
):
519 if self
.userMapFromPerforceServer
:
524 for output
in p4CmdList("users"):
525 if not output
.has_key("User"):
527 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
528 self
.emails
[output
["Email"]] = output
["User"]
532 for (key
, val
) in self
.users
.items():
533 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
535 open(self
.getUserCacheFilename(), "wb").write(s
)
536 self
.userMapFromPerforceServer
= True
538 def loadUserMapFromCache(self
):
540 self
.userMapFromPerforceServer
= False
542 cache
= open(self
.getUserCacheFilename(), "rb")
543 lines
= cache
.readlines()
546 entry
= line
.strip().split("\t")
547 self
.users
[entry
[0]] = entry
[1]
549 self
.getUserMapFromPerforceServer()
551 class P4Debug(Command
):
553 Command
.__init
__(self
)
555 optparse
.make_option("--verbose", dest
="verbose", action
="store_true",
558 self
.description
= "A tool to debug the output of p4 -G."
559 self
.needsGit
= False
564 for output
in p4CmdList(" ".join(args
)):
565 print 'Element: %d' % j
570 class P4RollBack(Command
):
572 Command
.__init
__(self
)
574 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
575 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
577 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
579 self
.rollbackLocalBranches
= False
584 maxChange
= int(args
[0])
586 if "p4ExitCode" in p4Cmd("changes -m 1"):
587 die("Problems executing p4");
589 if self
.rollbackLocalBranches
:
590 refPrefix
= "refs/heads/"
591 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
593 refPrefix
= "refs/remotes/"
594 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
597 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
599 ref
= refPrefix
+ line
600 log
= extractLogMessageFromGitCommit(ref
)
601 settings
= extractSettingsGitLog(log
)
603 depotPaths
= settings
['depot-paths']
604 change
= settings
['change']
608 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
609 for p
in depotPaths
]))) == 0:
610 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
611 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
614 while change
and int(change
) > maxChange
:
617 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
618 system("git update-ref %s \"%s^\"" % (ref
, ref
))
619 log
= extractLogMessageFromGitCommit(ref
)
620 settings
= extractSettingsGitLog(log
)
623 depotPaths
= settings
['depot-paths']
624 change
= settings
['change']
627 print "%s rewound to %s" % (ref
, change
)
631 class P4Submit(Command
, P4UserMap
):
633 Command
.__init
__(self
)
634 P4UserMap
.__init
__(self
)
636 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
637 optparse
.make_option("--origin", dest
="origin"),
638 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
639 # preserve the user, requires relevant p4 permissions
640 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
642 self
.description
= "Submit changes from git to the perforce depot."
643 self
.usage
+= " [name of git branch to submit into perforce depot]"
644 self
.interactive
= True
646 self
.detectRenames
= False
648 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
649 self
.isWindows
= (platform
.system() == "Windows")
650 self
.myP4UserId
= None
653 if len(p4CmdList("opened ...")) > 0:
654 die("You have files opened with perforce! Close them before starting the sync.")
656 # replaces everything between 'Description:' and the next P4 submit template field with the
658 def prepareLogMessage(self
, template
, message
):
661 inDescriptionSection
= False
663 for line
in template
.split("\n"):
664 if line
.startswith("#"):
665 result
+= line
+ "\n"
668 if inDescriptionSection
:
669 if line
.startswith("Files:") or line
.startswith("Jobs:"):
670 inDescriptionSection
= False
674 if line
.startswith("Description:"):
675 inDescriptionSection
= True
677 for messageLine
in message
.split("\n"):
678 line
+= "\t" + messageLine
+ "\n"
680 result
+= line
+ "\n"
684 def p4UserForCommit(self
,id):
685 # Return the tuple (perforce user,git email) for a given git commit id
686 self
.getUserMapFromPerforceServer()
687 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
688 gitEmail
= gitEmail
.strip()
689 if not self
.emails
.has_key(gitEmail
):
690 return (None,gitEmail
)
692 return (self
.emails
[gitEmail
],gitEmail
)
694 def checkValidP4Users(self
,commits
):
695 # check if any git authors cannot be mapped to p4 users
697 (user
,email
) = self
.p4UserForCommit(id)
699 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
700 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
703 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
705 def lastP4Changelist(self
):
706 # Get back the last changelist number submitted in this client spec. This
707 # then gets used to patch up the username in the change. If the same
708 # client spec is being used by multiple processes then this might go
710 results
= p4CmdList("client -o") # find the current client
713 if r
.has_key('Client'):
717 die("could not get client spec")
718 results
= p4CmdList("changes -c %s -m 1" % client
)
720 if r
.has_key('change'):
722 die("Could not get changelist number for last submit - cannot patch up user details")
724 def modifyChangelistUser(self
, changelist
, newUser
):
725 # fixup the user field of a changelist after it has been submitted.
726 changes
= p4CmdList("change -o %s" % changelist
)
727 if len(changes
) != 1:
728 die("Bad output from p4 change modifying %s to user %s" %
729 (changelist
, newUser
))
732 if c
['User'] == newUser
: return # nothing to do
734 input = marshal
.dumps(c
)
736 result
= p4CmdList("change -f -i", stdin
=input)
738 if r
.has_key('code'):
739 if r
['code'] == 'error':
740 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
741 if r
.has_key('data'):
742 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
744 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
746 def canChangeChangelists(self
):
747 # check to see if we have p4 admin or super-user permissions, either of
748 # which are required to modify changelists.
749 results
= p4CmdList("protects %s" % self
.depotPath
)
751 if r
.has_key('perm'):
752 if r
['perm'] == 'admin':
754 if r
['perm'] == 'super':
760 return self
.myP4UserId
762 results
= p4CmdList("user -o")
764 if r
.has_key('User'):
765 self
.myP4UserId
= r
['User']
767 die("Could not find your p4 user id")
769 def p4UserIsMe(self
, p4User
):
770 # return True if the given p4 user is actually me
772 if not p4User
or p4User
!= me
:
777 def prepareSubmitTemplate(self
):
778 # remove lines in the Files section that show changes to files outside the depot path we're committing into
780 inFilesSection
= False
781 for line
in p4_read_pipe_lines("change -o"):
782 if line
.endswith("\r\n"):
783 line
= line
[:-2] + "\n"
785 if line
.startswith("\t"):
786 # path starts and ends with a tab
788 lastTab
= path
.rfind("\t")
790 path
= path
[:lastTab
]
791 if not p4PathStartsWith(path
, self
.depotPath
):
794 inFilesSection
= False
796 if line
.startswith("Files:"):
797 inFilesSection
= True
803 def applyCommit(self
, id):
804 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
806 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
808 if not self
.detectRenames
:
809 # If not explicitly set check the config variable
810 self
.detectRenames
= gitConfig("git-p4.detectRenames")
812 if self
.detectRenames
.lower() == "false" or self
.detectRenames
== "":
814 elif self
.detectRenames
.lower() == "true":
817 diffOpts
= "-M%s" % self
.detectRenames
819 detectCopies
= gitConfig("git-p4.detectCopies")
820 if detectCopies
.lower() == "true":
822 elif detectCopies
!= "" and detectCopies
.lower() != "false":
823 diffOpts
+= " -C%s" % detectCopies
825 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
826 diffOpts
+= " --find-copies-harder"
828 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts
, id, id))
830 filesToDelete
= set()
832 filesToChangeExecBit
= {}
834 diff
= parseDiffTreeEntry(line
)
835 modifier
= diff
['status']
838 p4_system("edit \"%s\"" % path
)
839 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
840 filesToChangeExecBit
[path
] = diff
['dst_mode']
841 editedFiles
.add(path
)
842 elif modifier
== "A":
844 filesToChangeExecBit
[path
] = diff
['dst_mode']
845 if path
in filesToDelete
:
846 filesToDelete
.remove(path
)
847 elif modifier
== "D":
848 filesToDelete
.add(path
)
849 if path
in filesToAdd
:
850 filesToAdd
.remove(path
)
851 elif modifier
== "C":
852 src
, dest
= diff
['src'], diff
['dst']
853 p4_system("integrate -Dt \"%s\" \"%s\"" % (src
, dest
))
854 if diff
['src_sha1'] != diff
['dst_sha1']:
855 p4_system("edit \"%s\"" % (dest
))
856 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
857 p4_system("edit \"%s\"" % (dest
))
858 filesToChangeExecBit
[dest
] = diff
['dst_mode']
860 editedFiles
.add(dest
)
861 elif modifier
== "R":
862 src
, dest
= diff
['src'], diff
['dst']
863 p4_system("integrate -Dt \"%s\" \"%s\"" % (src
, dest
))
864 if diff
['src_sha1'] != diff
['dst_sha1']:
865 p4_system("edit \"%s\"" % (dest
))
866 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
867 p4_system("edit \"%s\"" % (dest
))
868 filesToChangeExecBit
[dest
] = diff
['dst_mode']
870 editedFiles
.add(dest
)
871 filesToDelete
.add(src
)
873 die("unknown modifier %s for %s" % (modifier
, path
))
875 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
876 patchcmd
= diffcmd
+ " | git apply "
877 tryPatchCmd
= patchcmd
+ "--check -"
878 applyPatchCmd
= patchcmd
+ "--check --apply -"
880 if os
.system(tryPatchCmd
) != 0:
881 print "Unfortunately applying the change failed!"
882 print "What do you want to do?"
884 while response
!= "s" and response
!= "a" and response
!= "w":
885 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
886 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
888 print "Skipping! Good luck with the next patches..."
889 for f
in editedFiles
:
890 p4_system("revert \"%s\"" % f
);
894 elif response
== "a":
895 os
.system(applyPatchCmd
)
896 if len(filesToAdd
) > 0:
897 print "You may also want to call p4 add on the following files:"
898 print " ".join(filesToAdd
)
899 if len(filesToDelete
):
900 print "The following files should be scheduled for deletion with p4 delete:"
901 print " ".join(filesToDelete
)
902 die("Please resolve and submit the conflict manually and "
903 + "continue afterwards with git-p4 submit --continue")
904 elif response
== "w":
905 system(diffcmd
+ " > patch.txt")
906 print "Patch saved to patch.txt in %s !" % self
.clientPath
907 die("Please resolve and submit the conflict manually and "
908 "continue afterwards with git-p4 submit --continue")
910 system(applyPatchCmd
)
913 p4_system("add \"%s\"" % f
)
914 for f
in filesToDelete
:
915 p4_system("revert \"%s\"" % f
)
916 p4_system("delete \"%s\"" % f
)
918 # Set/clear executable bits
919 for f
in filesToChangeExecBit
.keys():
920 mode
= filesToChangeExecBit
[f
]
921 setP4ExecBit(f
, mode
)
923 logMessage
= extractLogMessageFromGitCommit(id)
924 logMessage
= logMessage
.strip()
926 template
= self
.prepareSubmitTemplate()
929 submitTemplate
= self
.prepareLogMessage(template
, logMessage
)
931 if self
.preserveUser
:
932 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
934 if os
.environ
.has_key("P4DIFF"):
935 del(os
.environ
["P4DIFF"])
937 for editedFile
in editedFiles
:
938 diff
+= p4_read_pipe("diff -du %r" % editedFile
)
941 for newFile
in filesToAdd
:
942 newdiff
+= "==== new file ====\n"
943 newdiff
+= "--- /dev/null\n"
944 newdiff
+= "+++ %s\n" % newFile
945 f
= open(newFile
, "r")
946 for line
in f
.readlines():
947 newdiff
+= "+" + line
950 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
951 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
952 submitTemplate
+= "######## Use git-p4 option --preserve-user to modify authorship\n"
953 submitTemplate
+= "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
955 separatorLine
= "######## everything below this line is just the diff #######\n"
957 [handle
, fileName
] = tempfile
.mkstemp()
958 tmpFile
= os
.fdopen(handle
, "w+")
960 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
961 separatorLine
= separatorLine
.replace("\n", "\r\n")
962 newdiff
= newdiff
.replace("\n", "\r\n")
963 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
965 mtime
= os
.stat(fileName
).st_mtime
966 if os
.environ
.has_key("P4EDITOR"):
967 editor
= os
.environ
.get("P4EDITOR")
969 editor
= read_pipe("git var GIT_EDITOR").strip()
970 system(editor
+ " " + fileName
)
972 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
978 if checkModTime
and (os
.stat(fileName
).st_mtime
<= mtime
):
980 while response
!= "y" and response
!= "n":
981 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
984 tmpFile
= open(fileName
, "rb")
985 message
= tmpFile
.read()
987 submitTemplate
= message
[:message
.index(separatorLine
)]
989 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
990 p4_write_pipe("submit -i", submitTemplate
)
992 if self
.preserveUser
:
994 # Get last changelist number. Cannot easily get it from
995 # the submit command output as the output is unmarshalled.
996 changelist
= self
.lastP4Changelist()
997 self
.modifyChangelistUser(changelist
, p4User
)
1000 for f
in editedFiles
:
1001 p4_system("revert \"%s\"" % f
);
1002 for f
in filesToAdd
:
1003 p4_system("revert \"%s\"" % f
);
1008 fileName
= "submit.txt"
1009 file = open(fileName
, "w+")
1010 file.write(self
.prepareLogMessage(template
, logMessage
))
1012 print ("Perforce submit template written as %s. "
1013 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1014 % (fileName
, fileName
))
1016 def run(self
, args
):
1018 self
.master
= currentGitBranch()
1019 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1020 die("Detecting current git branch failed!")
1021 elif len(args
) == 1:
1022 self
.master
= args
[0]
1026 allowSubmit
= gitConfig("git-p4.allowSubmit")
1027 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1028 die("%s is not in git-p4.allowSubmit" % self
.master
)
1030 [upstream
, settings
] = findUpstreamBranchPoint()
1031 self
.depotPath
= settings
['depot-paths'][0]
1032 if len(self
.origin
) == 0:
1033 self
.origin
= upstream
1035 if self
.preserveUser
:
1036 if not self
.canChangeChangelists():
1037 die("Cannot preserve user names without p4 super-user or admin permissions")
1040 print "Origin branch is " + self
.origin
1042 if len(self
.depotPath
) == 0:
1043 print "Internal error: cannot locate perforce depot path from existing branches"
1046 self
.clientPath
= p4Where(self
.depotPath
)
1048 if len(self
.clientPath
) == 0:
1049 print "Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
1052 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1053 self
.oldWorkingDirectory
= os
.getcwd()
1055 chdir(self
.clientPath
)
1056 print "Synchronizing p4 checkout..."
1057 p4_system("sync ...")
1062 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1063 commits
.append(line
.strip())
1066 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1067 self
.checkAuthorship
= False
1069 self
.checkAuthorship
= True
1071 if self
.preserveUser
:
1072 self
.checkValidP4Users(commits
)
1074 while len(commits
) > 0:
1076 commits
= commits
[1:]
1077 self
.applyCommit(commit
)
1078 if not self
.interactive
:
1081 if len(commits
) == 0:
1082 print "All changes applied!"
1083 chdir(self
.oldWorkingDirectory
)
1093 class P4Sync(Command
, P4UserMap
):
1094 delete_actions
= ( "delete", "move/delete", "purge" )
1097 Command
.__init
__(self
)
1098 P4UserMap
.__init
__(self
)
1100 optparse
.make_option("--branch", dest
="branch"),
1101 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1102 optparse
.make_option("--changesfile", dest
="changesFile"),
1103 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1104 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1105 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
1106 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1107 help="Import into refs/heads/ , not refs/remotes"),
1108 optparse
.make_option("--max-changes", dest
="maxChanges"),
1109 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1110 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1111 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1112 help="Only sync files that are included in the Perforce Client Spec")
1114 self
.description
= """Imports from Perforce into a git repository.\n
1116 //depot/my/project/ -- to import the current head
1117 //depot/my/project/@all -- to import everything
1118 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1120 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1122 self
.usage
+= " //depot/path[@revRange]"
1124 self
.createdBranches
= set()
1125 self
.committedChanges
= set()
1127 self
.detectBranches
= False
1128 self
.detectLabels
= False
1129 self
.changesFile
= ""
1130 self
.syncWithOrigin
= True
1131 self
.verbose
= False
1132 self
.importIntoRemotes
= True
1133 self
.maxChanges
= ""
1134 self
.isWindows
= (platform
.system() == "Windows")
1135 self
.keepRepoPath
= False
1136 self
.depotPaths
= None
1137 self
.p4BranchesInGit
= []
1138 self
.cloneExclude
= []
1139 self
.useClientSpec
= False
1140 self
.clientSpecDirs
= []
1142 if gitConfig("git-p4.syncFromOrigin") == "false":
1143 self
.syncWithOrigin
= False
1146 # P4 wildcards are not allowed in filenames. P4 complains
1147 # if you simply add them, but you can force it with "-f", in
1148 # which case it translates them into %xx encoding internally.
1149 # Search for and fix just these four characters. Do % last so
1150 # that fixing it does not inadvertently create new %-escapes.
1152 def wildcard_decode(self
, path
):
1153 # Cannot have * in a filename in windows; untested as to
1154 # what p4 would do in such a case.
1155 if not self
.isWindows
:
1156 path
= path
.replace("%2A", "*")
1157 path
= path
.replace("%23", "#") \
1158 .replace("%40", "@") \
1159 .replace("%25", "%")
1162 def extractFilesFromCommit(self
, commit
):
1163 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1164 for path
in self
.cloneExclude
]
1167 while commit
.has_key("depotFile%s" % fnum
):
1168 path
= commit
["depotFile%s" % fnum
]
1170 if [p
for p
in self
.cloneExclude
1171 if p4PathStartsWith(path
, p
)]:
1174 found
= [p
for p
in self
.depotPaths
1175 if p4PathStartsWith(path
, p
)]
1182 file["rev"] = commit
["rev%s" % fnum
]
1183 file["action"] = commit
["action%s" % fnum
]
1184 file["type"] = commit
["type%s" % fnum
]
1189 def stripRepoPath(self
, path
, prefixes
):
1190 if self
.useClientSpec
:
1192 # if using the client spec, we use the output directory
1193 # specified in the client. For example, a view
1194 # //depot/foo/branch/... //client/branch/foo/...
1195 # will end up putting all foo/branch files into
1197 for val
in self
.clientSpecDirs
:
1198 if path
.startswith(val
[0]):
1199 # replace the depot path with the client path
1200 path
= path
.replace(val
[0], val
[1][1])
1201 # now strip out the client (//client/...)
1202 path
= re
.sub("^(//[^/]+/)", '', path
)
1203 # the rest is all path
1206 if self
.keepRepoPath
:
1207 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1210 if p4PathStartsWith(path
, p
):
1211 path
= path
[len(p
):]
1215 def splitFilesIntoBranches(self
, commit
):
1218 while commit
.has_key("depotFile%s" % fnum
):
1219 path
= commit
["depotFile%s" % fnum
]
1220 found
= [p
for p
in self
.depotPaths
1221 if p4PathStartsWith(path
, p
)]
1228 file["rev"] = commit
["rev%s" % fnum
]
1229 file["action"] = commit
["action%s" % fnum
]
1230 file["type"] = commit
["type%s" % fnum
]
1233 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1235 for branch
in self
.knownBranches
.keys():
1237 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1238 if relPath
.startswith(branch
+ "/"):
1239 if branch
not in branches
:
1240 branches
[branch
] = []
1241 branches
[branch
].append(file)
1246 # output one file from the P4 stream
1247 # - helper for streamP4Files
1249 def streamOneP4File(self
, file, contents
):
1250 if file["type"] == "apple":
1251 print "\nfile %s is a strange apple file that forks. Ignoring" % \
1255 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1256 relPath
= self
.wildcard_decode(relPath
)
1258 sys
.stderr
.write("%s\n" % relPath
)
1260 (type_base
, type_mods
) = split_p4_type(file["type"])
1263 if "x" in type_mods
:
1265 if type_base
== "symlink":
1267 # p4 print on a symlink contains "target\n"; remove the newline
1268 data
= ''.join(contents
)
1269 contents
= [data
[:-1]]
1271 if type_base
== "utf16":
1272 # p4 delivers different text in the python output to -G
1273 # than it does when using "print -o", or normal p4 client
1274 # operations. utf16 is converted to ascii or utf8, perhaps.
1275 # But ascii text saved as -t utf16 is completely mangled.
1276 # Invoke print -o to get the real contents.
1277 text
= p4_read_pipe('print -q -o - "%s"' % file['depotFile'])
1280 # Perhaps windows wants unicode, utf16 newlines translated too;
1281 # but this is not doing it.
1282 if self
.isWindows
and type_base
== "text":
1284 for data
in contents
:
1285 data
= data
.replace("\r\n", "\n")
1286 mangled
.append(data
)
1289 # Note that we do not try to de-mangle keywords on utf16 files,
1290 # even though in theory somebody may want that.
1291 if type_base
in ("text", "unicode", "binary"):
1292 if "ko" in type_mods
:
1293 contents
= map(lambda text
: re
.sub(r
'(?i)\$(Id|Header):[^$]*\$', r
'$\1$', text
), contents
)
1294 elif "k" in type_mods
:
1295 contents
= map(lambda text
: re
.sub(r
'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$', r
'$\1$', text
), contents
)
1297 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1302 length
= length
+ len(d
)
1304 self
.gitStream
.write("data %d\n" % length
)
1306 self
.gitStream
.write(d
)
1307 self
.gitStream
.write("\n")
1309 def streamOneP4Deletion(self
, file):
1310 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1312 sys
.stderr
.write("delete %s\n" % relPath
)
1313 self
.gitStream
.write("D %s\n" % relPath
)
1315 # handle another chunk of streaming data
1316 def streamP4FilesCb(self
, marshalled
):
1318 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1319 # start of a new file - output the old one first
1320 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1321 self
.stream_file
= {}
1322 self
.stream_contents
= []
1323 self
.stream_have_file_info
= False
1325 # pick up the new file information... for the
1326 # 'data' field we need to append to our array
1327 for k
in marshalled
.keys():
1329 self
.stream_contents
.append(marshalled
['data'])
1331 self
.stream_file
[k
] = marshalled
[k
]
1333 self
.stream_have_file_info
= True
1335 # Stream directly from "p4 files" into "git fast-import"
1336 def streamP4Files(self
, files
):
1343 for val
in self
.clientSpecDirs
:
1344 if f
['path'].startswith(val
[0]):
1350 filesForCommit
.append(f
)
1351 if f
['action'] in self
.delete_actions
:
1352 filesToDelete
.append(f
)
1354 filesToRead
.append(f
)
1357 for f
in filesToDelete
:
1358 self
.streamOneP4Deletion(f
)
1360 if len(filesToRead
) > 0:
1361 self
.stream_file
= {}
1362 self
.stream_contents
= []
1363 self
.stream_have_file_info
= False
1365 # curry self argument
1366 def streamP4FilesCbSelf(entry
):
1367 self
.streamP4FilesCb(entry
)
1369 p4CmdList("-x - print",
1370 '\n'.join(['%s#%s' % (f
['path'], f
['rev'])
1371 for f
in filesToRead
]),
1372 cb
=streamP4FilesCbSelf
)
1375 if self
.stream_file
.has_key('depotFile'):
1376 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1378 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
1379 epoch
= details
["time"]
1380 author
= details
["user"]
1381 self
.branchPrefixes
= branchPrefixes
1384 print "commit into %s" % branch
1386 # start with reading files; if that fails, we should not
1390 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
1391 new_files
.append (f
)
1393 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
1395 self
.gitStream
.write("commit %s\n" % branch
)
1396 # gitStream.write("mark :%s\n" % details["change"])
1397 self
.committedChanges
.add(int(details
["change"]))
1399 if author
not in self
.users
:
1400 self
.getUserMapFromPerforceServer()
1401 if author
in self
.users
:
1402 committer
= "%s %s %s" % (self
.users
[author
], epoch
, self
.tz
)
1404 committer
= "%s <a@b> %s %s" % (author
, epoch
, self
.tz
)
1406 self
.gitStream
.write("committer %s\n" % committer
)
1408 self
.gitStream
.write("data <<EOT\n")
1409 self
.gitStream
.write(details
["desc"])
1410 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1411 % (','.join (branchPrefixes
), details
["change"]))
1412 if len(details
['options']) > 0:
1413 self
.gitStream
.write(": options = %s" % details
['options'])
1414 self
.gitStream
.write("]\nEOT\n\n")
1418 print "parent %s" % parent
1419 self
.gitStream
.write("from %s\n" % parent
)
1421 self
.streamP4Files(new_files
)
1422 self
.gitStream
.write("\n")
1424 change
= int(details
["change"])
1426 if self
.labels
.has_key(change
):
1427 label
= self
.labels
[change
]
1428 labelDetails
= label
[0]
1429 labelRevisions
= label
[1]
1431 print "Change %s is labelled %s" % (change
, labelDetails
)
1433 files
= p4CmdList("files " + ' '.join (["%s...@%s" % (p
, change
)
1434 for p
in branchPrefixes
]))
1436 if len(files
) == len(labelRevisions
):
1440 if info
["action"] in self
.delete_actions
:
1442 cleanedFiles
[info
["depotFile"]] = info
["rev"]
1444 if cleanedFiles
== labelRevisions
:
1445 self
.gitStream
.write("tag tag_%s\n" % labelDetails
["label"])
1446 self
.gitStream
.write("from %s\n" % branch
)
1448 owner
= labelDetails
["Owner"]
1450 if author
in self
.users
:
1451 tagger
= "%s %s %s" % (self
.users
[owner
], epoch
, self
.tz
)
1453 tagger
= "%s <a@b> %s %s" % (owner
, epoch
, self
.tz
)
1454 self
.gitStream
.write("tagger %s\n" % tagger
)
1455 self
.gitStream
.write("data <<EOT\n")
1456 self
.gitStream
.write(labelDetails
["Description"])
1457 self
.gitStream
.write("EOT\n\n")
1461 print ("Tag %s does not match with change %s: files do not match."
1462 % (labelDetails
["label"], change
))
1466 print ("Tag %s does not match with change %s: file count is different."
1467 % (labelDetails
["label"], change
))
1469 def getLabels(self
):
1472 l
= p4CmdList("labels %s..." % ' '.join (self
.depotPaths
))
1473 if len(l
) > 0 and not self
.silent
:
1474 print "Finding files belonging to labels in %s" % `self
.depotPaths`
1477 label
= output
["label"]
1481 print "Querying files for label %s" % label
1482 for file in p4CmdList("files "
1483 + ' '.join (["%s...@%s" % (p
, label
)
1484 for p
in self
.depotPaths
])):
1485 revisions
[file["depotFile"]] = file["rev"]
1486 change
= int(file["change"])
1487 if change
> newestChange
:
1488 newestChange
= change
1490 self
.labels
[newestChange
] = [output
, revisions
]
1493 print "Label changes: %s" % self
.labels
.keys()
1495 def guessProjectName(self
):
1496 for p
in self
.depotPaths
:
1499 p
= p
[p
.strip().rfind("/") + 1:]
1500 if not p
.endswith("/"):
1504 def getBranchMapping(self
):
1505 lostAndFoundBranches
= set()
1507 user
= gitConfig("git-p4.branchUser")
1509 command
= "branches -u %s" % user
1511 command
= "branches"
1513 for info
in p4CmdList(command
):
1514 details
= p4Cmd("branch -o %s" % info
["branch"])
1516 while details
.has_key("View%s" % viewIdx
):
1517 paths
= details
["View%s" % viewIdx
].split(" ")
1518 viewIdx
= viewIdx
+ 1
1519 # require standard //depot/foo/... //depot/bar/... mapping
1520 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
1523 destination
= paths
[1]
1525 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
1526 source
= source
[len(self
.depotPaths
[0]):-4]
1527 destination
= destination
[len(self
.depotPaths
[0]):-4]
1529 if destination
in self
.knownBranches
:
1531 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
1532 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
1535 self
.knownBranches
[destination
] = source
1537 lostAndFoundBranches
.discard(destination
)
1539 if source
not in self
.knownBranches
:
1540 lostAndFoundBranches
.add(source
)
1542 # Perforce does not strictly require branches to be defined, so we also
1543 # check git config for a branch list.
1545 # Example of branch definition in git config file:
1547 # branchList=main:branchA
1548 # branchList=main:branchB
1549 # branchList=branchA:branchC
1550 configBranches
= gitConfigList("git-p4.branchList")
1551 for branch
in configBranches
:
1553 (source
, destination
) = branch
.split(":")
1554 self
.knownBranches
[destination
] = source
1556 lostAndFoundBranches
.discard(destination
)
1558 if source
not in self
.knownBranches
:
1559 lostAndFoundBranches
.add(source
)
1562 for branch
in lostAndFoundBranches
:
1563 self
.knownBranches
[branch
] = branch
1565 def getBranchMappingFromGitBranches(self
):
1566 branches
= p4BranchesInGit(self
.importIntoRemotes
)
1567 for branch
in branches
.keys():
1568 if branch
== "master":
1571 branch
= branch
[len(self
.projectName
):]
1572 self
.knownBranches
[branch
] = branch
1574 def listExistingP4GitBranches(self
):
1575 # branches holds mapping from name to commit
1576 branches
= p4BranchesInGit(self
.importIntoRemotes
)
1577 self
.p4BranchesInGit
= branches
.keys()
1578 for branch
in branches
.keys():
1579 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
1581 def updateOptionDict(self
, d
):
1583 if self
.keepRepoPath
:
1584 option_keys
['keepRepoPath'] = 1
1586 d
["options"] = ' '.join(sorted(option_keys
.keys()))
1588 def readOptions(self
, d
):
1589 self
.keepRepoPath
= (d
.has_key('options')
1590 and ('keepRepoPath' in d
['options']))
1592 def gitRefForBranch(self
, branch
):
1593 if branch
== "main":
1594 return self
.refPrefix
+ "master"
1596 if len(branch
) <= 0:
1599 return self
.refPrefix
+ self
.projectName
+ branch
1601 def gitCommitByP4Change(self
, ref
, change
):
1603 print "looking in ref " + ref
+ " for change %s using bisect..." % change
1606 latestCommit
= parseRevision(ref
)
1610 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
1611 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
1616 log
= extractLogMessageFromGitCommit(next
)
1617 settings
= extractSettingsGitLog(log
)
1618 currentChange
= int(settings
['change'])
1620 print "current change %s" % currentChange
1622 if currentChange
== change
:
1624 print "found %s" % next
1627 if currentChange
< change
:
1628 earliestCommit
= "^%s" % next
1630 latestCommit
= "%s" % next
1634 def importNewBranch(self
, branch
, maxChange
):
1635 # make fast-import flush all changes to disk and update the refs using the checkpoint
1636 # command so that we can try to find the branch parent in the git history
1637 self
.gitStream
.write("checkpoint\n\n");
1638 self
.gitStream
.flush();
1639 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
1640 range = "@1,%s" % maxChange
1641 #print "prefix" + branchPrefix
1642 changes
= p4ChangesForPaths([branchPrefix
], range)
1643 if len(changes
) <= 0:
1645 firstChange
= changes
[0]
1646 #print "first change in branch: %s" % firstChange
1647 sourceBranch
= self
.knownBranches
[branch
]
1648 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
1649 sourceRef
= self
.gitRefForBranch(sourceBranch
)
1650 #print "source " + sourceBranch
1652 branchParentChange
= int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath
, firstChange
))["change"])
1653 #print "branch parent: %s" % branchParentChange
1654 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
1655 if len(gitParent
) > 0:
1656 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
1657 #print "parent git commit: %s" % gitParent
1659 self
.importChanges(changes
)
1662 def importChanges(self
, changes
):
1664 for change
in changes
:
1665 description
= p4Cmd("describe %s" % change
)
1666 self
.updateOptionDict(description
)
1669 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
1674 if self
.detectBranches
:
1675 branches
= self
.splitFilesIntoBranches(description
)
1676 for branch
in branches
.keys():
1678 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
1682 filesForCommit
= branches
[branch
]
1685 print "branch is %s" % branch
1687 self
.updatedBranches
.add(branch
)
1689 if branch
not in self
.createdBranches
:
1690 self
.createdBranches
.add(branch
)
1691 parent
= self
.knownBranches
[branch
]
1692 if parent
== branch
:
1695 fullBranch
= self
.projectName
+ branch
1696 if fullBranch
not in self
.p4BranchesInGit
:
1698 print("\n Importing new branch %s" % fullBranch
);
1699 if self
.importNewBranch(branch
, change
- 1):
1701 self
.p4BranchesInGit
.append(fullBranch
)
1703 print("\n Resuming with change %s" % change
);
1706 print "parent determined through known branches: %s" % parent
1708 branch
= self
.gitRefForBranch(branch
)
1709 parent
= self
.gitRefForBranch(parent
)
1712 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
1714 if len(parent
) == 0 and branch
in self
.initialParents
:
1715 parent
= self
.initialParents
[branch
]
1716 del self
.initialParents
[branch
]
1718 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
1720 files
= self
.extractFilesFromCommit(description
)
1721 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
1723 self
.initialParent
= ""
1725 print self
.gitError
.read()
1728 def importHeadRevision(self
, revision
):
1729 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
1732 details
["user"] = "git perforce import user"
1733 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
1734 % (' '.join(self
.depotPaths
), revision
))
1735 details
["change"] = revision
1739 for info
in p4CmdList("files "
1740 + ' '.join(["%s...%s"
1742 for p
in self
.depotPaths
])):
1744 if 'code' in info
and info
['code'] == 'error':
1745 sys
.stderr
.write("p4 returned an error: %s\n"
1747 if info
['data'].find("must refer to client") >= 0:
1748 sys
.stderr
.write("This particular p4 error is misleading.\n")
1749 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
1750 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
1752 if 'p4ExitCode' in info
:
1753 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
1757 change
= int(info
["change"])
1758 if change
> newestRevision
:
1759 newestRevision
= change
1761 if info
["action"] in self
.delete_actions
:
1762 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1763 #fileCnt = fileCnt + 1
1766 for prop
in ["depotFile", "rev", "action", "type" ]:
1767 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
1769 fileCnt
= fileCnt
+ 1
1771 details
["change"] = newestRevision
1773 # Use time from top-most change so that all git-p4 clones of
1774 # the same p4 repo have the same commit SHA1s.
1775 res
= p4CmdList("describe -s %d" % newestRevision
)
1778 if r
.has_key('time'):
1779 newestTime
= int(r
['time'])
1780 if newestTime
is None:
1781 die("\"describe -s\" on newest change %d did not give a time")
1782 details
["time"] = newestTime
1784 self
.updateOptionDict(details
)
1786 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
1788 print "IO error with git fast-import. Is your git version recent enough?"
1789 print self
.gitError
.read()
1792 def getClientSpec(self
):
1793 specList
= p4CmdList( "client -o" )
1795 for entry
in specList
:
1796 for k
,v
in entry
.iteritems():
1797 if k
.startswith("View"):
1799 # p4 has these %%1 to %%9 arguments in specs to
1800 # reorder paths; which we can't handle (yet :)
1801 if re
.match('%%\d', v
) != None:
1802 print "Sorry, can't handle %%n arguments in client specs"
1805 if v
.startswith('"'):
1809 index
= v
.find("...")
1811 # save the "client view"; i.e the RHS of the view
1812 # line that tells the client where to put the
1813 # files for this view.
1814 cv
= v
[index
+3:].strip() # +3 to remove previous '...'
1816 # if the client view doesn't end with a
1817 # ... wildcard, then we're going to mess up the
1818 # output directory, so fail gracefully.
1819 if not cv
.endswith('...'):
1820 print 'Sorry, client view in "%s" needs to end with wildcard' % (k
)
1824 # now save the view; +index means included, -index
1825 # means it should be filtered out.
1827 if v
.startswith("-"):
1833 temp
[v
] = (include
, cv
)
1835 self
.clientSpecDirs
= temp
.items()
1836 self
.clientSpecDirs
.sort( lambda x
, y
: abs( y
[1][0] ) - abs( x
[1][0] ) )
1838 def run(self
, args
):
1839 self
.depotPaths
= []
1840 self
.changeRange
= ""
1841 self
.initialParent
= ""
1842 self
.previousDepotPaths
= []
1844 # map from branch depot path to parent branch
1845 self
.knownBranches
= {}
1846 self
.initialParents
= {}
1847 self
.hasOrigin
= originP4BranchesExist()
1848 if not self
.syncWithOrigin
:
1849 self
.hasOrigin
= False
1851 if self
.importIntoRemotes
:
1852 self
.refPrefix
= "refs/remotes/p4/"
1854 self
.refPrefix
= "refs/heads/p4/"
1856 if self
.syncWithOrigin
and self
.hasOrigin
:
1858 print "Syncing with origin first by calling git fetch origin"
1859 system("git fetch origin")
1861 if len(self
.branch
) == 0:
1862 self
.branch
= self
.refPrefix
+ "master"
1863 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
1864 system("git update-ref %s refs/heads/p4" % self
.branch
)
1865 system("git branch -D p4");
1866 # create it /after/ importing, when master exists
1867 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
1868 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
1870 if self
.useClientSpec
or gitConfig("git-p4.useclientspec") == "true":
1871 self
.getClientSpec()
1873 # TODO: should always look at previous commits,
1874 # merge with previous imports, if possible.
1877 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
1878 self
.listExistingP4GitBranches()
1880 if len(self
.p4BranchesInGit
) > 1:
1882 print "Importing from/into multiple branches"
1883 self
.detectBranches
= True
1886 print "branches: %s" % self
.p4BranchesInGit
1889 for branch
in self
.p4BranchesInGit
:
1890 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
1892 settings
= extractSettingsGitLog(logMsg
)
1894 self
.readOptions(settings
)
1895 if (settings
.has_key('depot-paths')
1896 and settings
.has_key ('change')):
1897 change
= int(settings
['change']) + 1
1898 p4Change
= max(p4Change
, change
)
1900 depotPaths
= sorted(settings
['depot-paths'])
1901 if self
.previousDepotPaths
== []:
1902 self
.previousDepotPaths
= depotPaths
1905 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
1906 prev_list
= prev
.split("/")
1907 cur_list
= cur
.split("/")
1908 for i
in range(0, min(len(cur_list
), len(prev_list
))):
1909 if cur_list
[i
] <> prev_list
[i
]:
1913 paths
.append ("/".join(cur_list
[:i
+ 1]))
1915 self
.previousDepotPaths
= paths
1918 self
.depotPaths
= sorted(self
.previousDepotPaths
)
1919 self
.changeRange
= "@%s,#head" % p4Change
1920 if not self
.detectBranches
:
1921 self
.initialParent
= parseRevision(self
.branch
)
1922 if not self
.silent
and not self
.detectBranches
:
1923 print "Performing incremental import into %s git branch" % self
.branch
1925 if not self
.branch
.startswith("refs/"):
1926 self
.branch
= "refs/heads/" + self
.branch
1928 if len(args
) == 0 and self
.depotPaths
:
1930 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
1932 if self
.depotPaths
and self
.depotPaths
!= args
:
1933 print ("previous import used depot path %s and now %s was specified. "
1934 "This doesn't work!" % (' '.join (self
.depotPaths
),
1938 self
.depotPaths
= sorted(args
)
1944 for p
in self
.depotPaths
:
1945 if p
.find("@") != -1:
1946 atIdx
= p
.index("@")
1947 self
.changeRange
= p
[atIdx
:]
1948 if self
.changeRange
== "@all":
1949 self
.changeRange
= ""
1950 elif ',' not in self
.changeRange
:
1951 revision
= self
.changeRange
1952 self
.changeRange
= ""
1954 elif p
.find("#") != -1:
1955 hashIdx
= p
.index("#")
1956 revision
= p
[hashIdx
:]
1958 elif self
.previousDepotPaths
== []:
1961 p
= re
.sub ("\.\.\.$", "", p
)
1962 if not p
.endswith("/"):
1967 self
.depotPaths
= newPaths
1970 self
.loadUserMapFromCache()
1972 if self
.detectLabels
:
1975 if self
.detectBranches
:
1976 ## FIXME - what's a P4 projectName ?
1977 self
.projectName
= self
.guessProjectName()
1980 self
.getBranchMappingFromGitBranches()
1982 self
.getBranchMapping()
1984 print "p4-git branches: %s" % self
.p4BranchesInGit
1985 print "initial parents: %s" % self
.initialParents
1986 for b
in self
.p4BranchesInGit
:
1990 b
= b
[len(self
.projectName
):]
1991 self
.createdBranches
.add(b
)
1993 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
1995 importProcess
= subprocess
.Popen(["git", "fast-import"],
1996 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
1997 stderr
=subprocess
.PIPE
);
1998 self
.gitOutput
= importProcess
.stdout
1999 self
.gitStream
= importProcess
.stdin
2000 self
.gitError
= importProcess
.stderr
2003 self
.importHeadRevision(revision
)
2007 if len(self
.changesFile
) > 0:
2008 output
= open(self
.changesFile
).readlines()
2011 changeSet
.add(int(line
))
2013 for change
in changeSet
:
2014 changes
.append(change
)
2018 # catch "git-p4 sync" with no new branches, in a repo that
2019 # does not have any existing git-p4 branches
2020 if len(args
) == 0 and not self
.p4BranchesInGit
:
2021 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2023 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2025 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2027 if len(self
.maxChanges
) > 0:
2028 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2030 if len(changes
) == 0:
2032 print "No changes to import!"
2035 if not self
.silent
and not self
.detectBranches
:
2036 print "Import destination: %s" % self
.branch
2038 self
.updatedBranches
= set()
2040 self
.importChanges(changes
)
2044 if len(self
.updatedBranches
) > 0:
2045 sys
.stdout
.write("Updated branches: ")
2046 for b
in self
.updatedBranches
:
2047 sys
.stdout
.write("%s " % b
)
2048 sys
.stdout
.write("\n")
2050 self
.gitStream
.close()
2051 if importProcess
.wait() != 0:
2052 die("fast-import failed: %s" % self
.gitError
.read())
2053 self
.gitOutput
.close()
2054 self
.gitError
.close()
2058 class P4Rebase(Command
):
2060 Command
.__init
__(self
)
2062 self
.description
= ("Fetches the latest revision from perforce and "
2063 + "rebases the current work (branch) against it")
2064 self
.verbose
= False
2066 def run(self
, args
):
2070 return self
.rebase()
2073 if os
.system("git update-index --refresh") != 0:
2074 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2075 if len(read_pipe("git diff-index HEAD --")) > 0:
2076 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2078 [upstream
, settings
] = findUpstreamBranchPoint()
2079 if len(upstream
) == 0:
2080 die("Cannot find upstream branchpoint for rebase")
2082 # the branchpoint may be p4/foo~3, so strip off the parent
2083 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2085 print "Rebasing the current branch onto %s" % upstream
2086 oldHead
= read_pipe("git rev-parse HEAD").strip()
2087 system("git rebase %s" % upstream
)
2088 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2091 class P4Clone(P4Sync
):
2093 P4Sync
.__init
__(self
)
2094 self
.description
= "Creates a new git repository and imports from Perforce into it"
2095 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2097 optparse
.make_option("--destination", dest
="cloneDestination",
2098 action
='store', default
=None,
2099 help="where to leave result of the clone"),
2100 optparse
.make_option("-/", dest
="cloneExclude",
2101 action
="append", type="string",
2102 help="exclude depot path"),
2103 optparse
.make_option("--bare", dest
="cloneBare",
2104 action
="store_true", default
=False),
2106 self
.cloneDestination
= None
2107 self
.needsGit
= False
2108 self
.cloneBare
= False
2110 # This is required for the "append" cloneExclude action
2111 def ensure_value(self
, attr
, value
):
2112 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2113 setattr(self
, attr
, value
)
2114 return getattr(self
, attr
)
2116 def defaultDestination(self
, args
):
2117 ## TODO: use common prefix of args?
2119 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2120 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2121 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2122 depotDir
= re
.sub(r
"/$", "", depotDir
)
2123 return os
.path
.split(depotDir
)[1]
2125 def run(self
, args
):
2129 if self
.keepRepoPath
and not self
.cloneDestination
:
2130 sys
.stderr
.write("Must specify destination for --keep-path\n")
2135 if not self
.cloneDestination
and len(depotPaths
) > 1:
2136 self
.cloneDestination
= depotPaths
[-1]
2137 depotPaths
= depotPaths
[:-1]
2139 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2140 for p
in depotPaths
:
2141 if not p
.startswith("//"):
2144 if not self
.cloneDestination
:
2145 self
.cloneDestination
= self
.defaultDestination(args
)
2147 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2149 if not os
.path
.exists(self
.cloneDestination
):
2150 os
.makedirs(self
.cloneDestination
)
2151 chdir(self
.cloneDestination
)
2153 init_cmd
= [ "git", "init" ]
2155 init_cmd
.append("--bare")
2156 subprocess
.check_call(init_cmd
)
2158 if not P4Sync
.run(self
, depotPaths
):
2160 if self
.branch
!= "master":
2161 if self
.importIntoRemotes
:
2162 masterbranch
= "refs/remotes/p4/master"
2164 masterbranch
= "refs/heads/p4/master"
2165 if gitBranchExists(masterbranch
):
2166 system("git branch master %s" % masterbranch
)
2167 if not self
.cloneBare
:
2168 system("git checkout -f")
2170 print "Could not detect main branch. No checkout/master branch created."
2174 class P4Branches(Command
):
2176 Command
.__init
__(self
)
2178 self
.description
= ("Shows the git branches that hold imports and their "
2179 + "corresponding perforce depot paths")
2180 self
.verbose
= False
2182 def run(self
, args
):
2183 if originP4BranchesExist():
2184 createOrUpdateBranchesFromOrigin()
2186 cmdline
= "git rev-parse --symbolic "
2187 cmdline
+= " --remotes"
2189 for line
in read_pipe_lines(cmdline
):
2192 if not line
.startswith('p4/') or line
== "p4/HEAD":
2196 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2197 settings
= extractSettingsGitLog(log
)
2199 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2202 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2204 optparse
.IndentedHelpFormatter
.__init
__(self
)
2206 def format_description(self
, description
):
2208 return description
+ "\n"
2212 def printUsage(commands
):
2213 print "usage: %s <command> [options]" % sys
.argv
[0]
2215 print "valid commands: %s" % ", ".join(commands
)
2217 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2222 "submit" : P4Submit
,
2223 "commit" : P4Submit
,
2225 "rebase" : P4Rebase
,
2227 "rollback" : P4RollBack
,
2228 "branches" : P4Branches
2233 if len(sys
.argv
[1:]) == 0:
2234 printUsage(commands
.keys())
2238 cmdName
= sys
.argv
[1]
2240 klass
= commands
[cmdName
]
2243 print "unknown command %s" % cmdName
2245 printUsage(commands
.keys())
2248 options
= cmd
.options
2249 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2253 if len(options
) > 0:
2254 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2256 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2258 description
= cmd
.description
,
2259 formatter
= HelpFormatter())
2261 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2263 verbose
= cmd
.verbose
2265 if cmd
.gitdir
== None:
2266 cmd
.gitdir
= os
.path
.abspath(".git")
2267 if not isValidGitDir(cmd
.gitdir
):
2268 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2269 if os
.path
.exists(cmd
.gitdir
):
2270 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2274 if not isValidGitDir(cmd
.gitdir
):
2275 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2276 cmd
.gitdir
+= "/.git"
2278 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2280 os
.environ
["GIT_DIR"] = cmd
.gitdir
2282 if not cmd
.run(args
):
2286 if __name__
== '__main__':