3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
18 def p4_build_cmd(cmd
):
19 """Build a suitable p4 command line.
21 This consolidates building and returning a p4 command line into one
22 location. It means that hooking into the environment, or other configuration
23 can be done more easily.
25 real_cmd
= "%s " % "p4"
27 user
= gitConfig("git-p4.user")
29 real_cmd
+= "-u %s " % user
31 password
= gitConfig("git-p4.password")
33 real_cmd
+= "-P %s " % password
35 port
= gitConfig("git-p4.port")
37 real_cmd
+= "-p %s " % port
39 host
= gitConfig("git-p4.host")
41 real_cmd
+= "-h %s " % host
43 client
= gitConfig("git-p4.client")
45 real_cmd
+= "-c %s " % client
47 real_cmd
+= "%s" % (cmd
)
61 sys
.stderr
.write(msg
+ "\n")
64 def write_pipe(c
, str):
66 sys
.stderr
.write('Writing pipe: %s\n' % c
)
68 pipe
= os
.popen(c
, 'w')
71 die('Command failed: %s' % c
)
75 def p4_write_pipe(c
, str):
76 real_cmd
= p4_build_cmd(c
)
77 return write_pipe(real_cmd
, str)
79 def read_pipe(c
, ignore_error
=False):
81 sys
.stderr
.write('Reading pipe: %s\n' % c
)
83 pipe
= os
.popen(c
, 'rb')
85 if pipe
.close() and not ignore_error
:
86 die('Command failed: %s' % c
)
90 def p4_read_pipe(c
, ignore_error
=False):
91 real_cmd
= p4_build_cmd(c
)
92 return read_pipe(real_cmd
, ignore_error
)
94 def read_pipe_lines(c
):
96 sys
.stderr
.write('Reading pipe: %s\n' % c
)
97 ## todo: check return status
98 pipe
= os
.popen(c
, 'rb')
99 val
= pipe
.readlines()
101 die('Command failed: %s' % c
)
105 def p4_read_pipe_lines(c
):
106 """Specifically invoke p4 on the command supplied. """
107 real_cmd
= p4_build_cmd(c
)
108 return read_pipe_lines(real_cmd
)
112 sys
.stderr
.write("executing %s\n" % cmd
)
113 if os
.system(cmd
) != 0:
114 die("command failed: %s" % cmd
)
117 """Specifically invoke p4 as the system command. """
118 real_cmd
= p4_build_cmd(cmd
)
119 return system(real_cmd
)
122 # Canonicalize the p4 type and return a tuple of the
123 # base type, plus any modifiers. See "p4 help filetypes"
124 # for a list and explanation.
126 def split_p4_type(p4type
):
128 p4_filetypes_historical
= {
129 "ctempobj": "binary+Sw",
135 "tempobj": "binary+FSw",
136 "ubinary": "binary+F",
137 "uresource": "resource+F",
138 "uxbinary": "binary+Fx",
139 "xbinary": "binary+x",
141 "xtempobj": "binary+Swx",
143 "xunicode": "unicode+x",
146 if p4type
in p4_filetypes_historical
:
147 p4type
= p4_filetypes_historical
[p4type
]
149 s
= p4type
.split("+")
157 def setP4ExecBit(file, mode
):
158 # Reopens an already open file and changes the execute bit to match
159 # the execute bit setting in the passed in mode.
163 if not isModeExec(mode
):
164 p4Type
= getP4OpenedType(file)
165 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
166 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
167 if p4Type
[-1] == "+":
168 p4Type
= p4Type
[0:-1]
170 p4_system("reopen -t %s %s" % (p4Type
, file))
172 def getP4OpenedType(file):
173 # Returns the perforce file type for the given file.
175 result
= p4_read_pipe("opened %s" % file)
176 match
= re
.match(".*\((.+)\)\r?$", result
)
178 return match
.group(1)
180 die("Could not determine file type for %s (result: '%s')" % (file, result
))
182 def diffTreePattern():
183 # This is a simple generator for the diff tree regex pattern. This could be
184 # a class variable if this and parseDiffTreeEntry were a part of a class.
185 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
189 def parseDiffTreeEntry(entry
):
190 """Parses a single diff tree entry into its component elements.
192 See git-diff-tree(1) manpage for details about the format of the diff
193 output. This method returns a dictionary with the following elements:
195 src_mode - The mode of the source file
196 dst_mode - The mode of the destination file
197 src_sha1 - The sha1 for the source file
198 dst_sha1 - The sha1 fr the destination file
199 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
200 status_score - The score for the status (applicable for 'C' and 'R'
201 statuses). This is None if there is no score.
202 src - The path for the source file.
203 dst - The path for the destination file. This is only present for
204 copy or renames. If it is not present, this is None.
206 If the pattern is not matched, None is returned."""
208 match
= diffTreePattern().next().match(entry
)
211 'src_mode': match
.group(1),
212 'dst_mode': match
.group(2),
213 'src_sha1': match
.group(3),
214 'dst_sha1': match
.group(4),
215 'status': match
.group(5),
216 'status_score': match
.group(6),
217 'src': match
.group(7),
218 'dst': match
.group(10)
222 def isModeExec(mode
):
223 # Returns True if the given git mode represents an executable file,
225 return mode
[-3:] == "755"
227 def isModeExecChanged(src_mode
, dst_mode
):
228 return isModeExec(src_mode
) != isModeExec(dst_mode
)
230 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
231 cmd
= p4_build_cmd("-G %s" % (cmd
))
233 sys
.stderr
.write("Opening pipe: %s\n" % cmd
)
235 # Use a temporary file to avoid deadlocks without
236 # subprocess.communicate(), which would put another copy
237 # of stdout into memory.
239 if stdin
is not None:
240 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
241 stdin_file
.write(stdin
)
245 p4
= subprocess
.Popen(cmd
, shell
=True,
247 stdout
=subprocess
.PIPE
)
252 entry
= marshal
.load(p4
.stdout
)
262 entry
["p4ExitCode"] = exitCode
268 list = p4CmdList(cmd
)
274 def p4Where(depotPath
):
275 if not depotPath
.endswith("/"):
277 depotPath
= depotPath
+ "..."
278 outputList
= p4CmdList("where %s" % depotPath
)
280 for entry
in outputList
:
281 if "depotFile" in entry
:
282 if entry
["depotFile"] == depotPath
:
285 elif "data" in entry
:
286 data
= entry
.get("data")
287 space
= data
.find(" ")
288 if data
[:space
] == depotPath
:
293 if output
["code"] == "error":
297 clientPath
= output
.get("path")
298 elif "data" in output
:
299 data
= output
.get("data")
300 lastSpace
= data
.rfind(" ")
301 clientPath
= data
[lastSpace
+ 1:]
303 if clientPath
.endswith("..."):
304 clientPath
= clientPath
[:-3]
307 def currentGitBranch():
308 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
310 def isValidGitDir(path
):
311 if (os
.path
.exists(path
+ "/HEAD")
312 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
316 def parseRevision(ref
):
317 return read_pipe("git rev-parse %s" % ref
).strip()
319 def extractLogMessageFromGitCommit(commit
):
322 ## fixme: title is first line of commit, not 1st paragraph.
324 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
333 def extractSettingsGitLog(log
):
335 for line
in log
.split("\n"):
337 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
341 assignments
= m
.group(1).split (':')
342 for a
in assignments
:
344 key
= vals
[0].strip()
345 val
= ('='.join (vals
[1:])).strip()
346 if val
.endswith ('\"') and val
.startswith('"'):
351 paths
= values
.get("depot-paths")
353 paths
= values
.get("depot-path")
355 values
['depot-paths'] = paths
.split(',')
358 def gitBranchExists(branch
):
359 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
360 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
361 return proc
.wait() == 0;
364 def gitConfig(key
, args
= None): # set args to "--bool", for instance
365 if not _gitConfig
.has_key(key
):
368 argsFilter
= "%s " % args
369 cmd
= "git config %s%s" % (argsFilter
, key
)
370 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
371 return _gitConfig
[key
]
373 def gitConfigList(key
):
374 if not _gitConfig
.has_key(key
):
375 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
376 return _gitConfig
[key
]
378 def p4BranchesInGit(branchesAreInRemotes
= True):
381 cmdline
= "git rev-parse --symbolic "
382 if branchesAreInRemotes
:
383 cmdline
+= " --remotes"
385 cmdline
+= " --branches"
387 for line
in read_pipe_lines(cmdline
):
390 ## only import to p4/
391 if not line
.startswith('p4/') or line
== "p4/HEAD":
396 branch
= re
.sub ("^p4/", "", line
)
398 branches
[branch
] = parseRevision(line
)
401 def findUpstreamBranchPoint(head
= "HEAD"):
402 branches
= p4BranchesInGit()
403 # map from depot-path to branch name
404 branchByDepotPath
= {}
405 for branch
in branches
.keys():
406 tip
= branches
[branch
]
407 log
= extractLogMessageFromGitCommit(tip
)
408 settings
= extractSettingsGitLog(log
)
409 if settings
.has_key("depot-paths"):
410 paths
= ",".join(settings
["depot-paths"])
411 branchByDepotPath
[paths
] = "remotes/p4/" + branch
415 while parent
< 65535:
416 commit
= head
+ "~%s" % parent
417 log
= extractLogMessageFromGitCommit(commit
)
418 settings
= extractSettingsGitLog(log
)
419 if settings
.has_key("depot-paths"):
420 paths
= ",".join(settings
["depot-paths"])
421 if branchByDepotPath
.has_key(paths
):
422 return [branchByDepotPath
[paths
], settings
]
426 return ["", settings
]
428 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
430 print ("Creating/updating branch(es) in %s based on origin branch(es)"
433 originPrefix
= "origin/p4/"
435 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
437 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
440 headName
= line
[len(originPrefix
):]
441 remoteHead
= localRefPrefix
+ headName
444 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
445 if (not original
.has_key('depot-paths')
446 or not original
.has_key('change')):
450 if not gitBranchExists(remoteHead
):
452 print "creating %s" % remoteHead
455 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
456 if settings
.has_key('change') > 0:
457 if settings
['depot-paths'] == original
['depot-paths']:
458 originP4Change
= int(original
['change'])
459 p4Change
= int(settings
['change'])
460 if originP4Change
> p4Change
:
461 print ("%s (%s) is newer than %s (%s). "
462 "Updating p4 branch from origin."
463 % (originHead
, originP4Change
,
464 remoteHead
, p4Change
))
467 print ("Ignoring: %s was imported from %s while "
468 "%s was imported from %s"
469 % (originHead
, ','.join(original
['depot-paths']),
470 remoteHead
, ','.join(settings
['depot-paths'])))
473 system("git update-ref %s %s" % (remoteHead
, originHead
))
475 def originP4BranchesExist():
476 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
478 def p4ChangesForPaths(depotPaths
, changeRange
):
480 output
= p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p
, changeRange
)
481 for p
in depotPaths
]))
485 changeNum
= int(line
.split(" ")[1])
486 changes
[changeNum
] = True
488 changelist
= changes
.keys()
492 def p4PathStartsWith(path
, prefix
):
493 # This method tries to remedy a potential mixed-case issue:
495 # If UserA adds //depot/DirA/file1
496 # and UserB adds //depot/dira/file2
498 # we may or may not have a problem. If you have core.ignorecase=true,
499 # we treat DirA and dira as the same directory
500 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
502 return path
.lower().startswith(prefix
.lower())
503 return path
.startswith(prefix
)
507 self
.usage
= "usage: %prog [options]"
512 self
.userMapFromPerforceServer
= False
514 def getUserCacheFilename(self
):
515 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
516 return home
+ "/.gitp4-usercache.txt"
518 def getUserMapFromPerforceServer(self
):
519 if self
.userMapFromPerforceServer
:
524 for output
in p4CmdList("users"):
525 if not output
.has_key("User"):
527 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
528 self
.emails
[output
["Email"]] = output
["User"]
532 for (key
, val
) in self
.users
.items():
533 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
535 open(self
.getUserCacheFilename(), "wb").write(s
)
536 self
.userMapFromPerforceServer
= True
538 def loadUserMapFromCache(self
):
540 self
.userMapFromPerforceServer
= False
542 cache
= open(self
.getUserCacheFilename(), "rb")
543 lines
= cache
.readlines()
546 entry
= line
.strip().split("\t")
547 self
.users
[entry
[0]] = entry
[1]
549 self
.getUserMapFromPerforceServer()
551 class P4Debug(Command
):
553 Command
.__init
__(self
)
555 optparse
.make_option("--verbose", dest
="verbose", action
="store_true",
558 self
.description
= "A tool to debug the output of p4 -G."
559 self
.needsGit
= False
564 for output
in p4CmdList(" ".join(args
)):
565 print 'Element: %d' % j
570 class P4RollBack(Command
):
572 Command
.__init
__(self
)
574 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
575 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
577 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
579 self
.rollbackLocalBranches
= False
584 maxChange
= int(args
[0])
586 if "p4ExitCode" in p4Cmd("changes -m 1"):
587 die("Problems executing p4");
589 if self
.rollbackLocalBranches
:
590 refPrefix
= "refs/heads/"
591 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
593 refPrefix
= "refs/remotes/"
594 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
597 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
599 ref
= refPrefix
+ line
600 log
= extractLogMessageFromGitCommit(ref
)
601 settings
= extractSettingsGitLog(log
)
603 depotPaths
= settings
['depot-paths']
604 change
= settings
['change']
608 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
609 for p
in depotPaths
]))) == 0:
610 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
611 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
614 while change
and int(change
) > maxChange
:
617 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
618 system("git update-ref %s \"%s^\"" % (ref
, ref
))
619 log
= extractLogMessageFromGitCommit(ref
)
620 settings
= extractSettingsGitLog(log
)
623 depotPaths
= settings
['depot-paths']
624 change
= settings
['change']
627 print "%s rewound to %s" % (ref
, change
)
631 class P4Submit(Command
, P4UserMap
):
633 Command
.__init
__(self
)
634 P4UserMap
.__init
__(self
)
636 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
637 optparse
.make_option("--origin", dest
="origin"),
638 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
639 # preserve the user, requires relevant p4 permissions
640 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
642 self
.description
= "Submit changes from git to the perforce depot."
643 self
.usage
+= " [name of git branch to submit into perforce depot]"
644 self
.interactive
= True
646 self
.detectRenames
= False
648 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
649 self
.isWindows
= (platform
.system() == "Windows")
650 self
.myP4UserId
= None
653 if len(p4CmdList("opened ...")) > 0:
654 die("You have files opened with perforce! Close them before starting the sync.")
656 # replaces everything between 'Description:' and the next P4 submit template field with the
658 def prepareLogMessage(self
, template
, message
):
661 inDescriptionSection
= False
663 for line
in template
.split("\n"):
664 if line
.startswith("#"):
665 result
+= line
+ "\n"
668 if inDescriptionSection
:
669 if line
.startswith("Files:") or line
.startswith("Jobs:"):
670 inDescriptionSection
= False
674 if line
.startswith("Description:"):
675 inDescriptionSection
= True
677 for messageLine
in message
.split("\n"):
678 line
+= "\t" + messageLine
+ "\n"
680 result
+= line
+ "\n"
684 def p4UserForCommit(self
,id):
685 # Return the tuple (perforce user,git email) for a given git commit id
686 self
.getUserMapFromPerforceServer()
687 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
688 gitEmail
= gitEmail
.strip()
689 if not self
.emails
.has_key(gitEmail
):
690 return (None,gitEmail
)
692 return (self
.emails
[gitEmail
],gitEmail
)
694 def checkValidP4Users(self
,commits
):
695 # check if any git authors cannot be mapped to p4 users
697 (user
,email
) = self
.p4UserForCommit(id)
699 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
700 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
703 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
705 def lastP4Changelist(self
):
706 # Get back the last changelist number submitted in this client spec. This
707 # then gets used to patch up the username in the change. If the same
708 # client spec is being used by multiple processes then this might go
710 results
= p4CmdList("client -o") # find the current client
713 if r
.has_key('Client'):
717 die("could not get client spec")
718 results
= p4CmdList("changes -c %s -m 1" % client
)
720 if r
.has_key('change'):
722 die("Could not get changelist number for last submit - cannot patch up user details")
724 def modifyChangelistUser(self
, changelist
, newUser
):
725 # fixup the user field of a changelist after it has been submitted.
726 changes
= p4CmdList("change -o %s" % changelist
)
727 if len(changes
) != 1:
728 die("Bad output from p4 change modifying %s to user %s" %
729 (changelist
, newUser
))
732 if c
['User'] == newUser
: return # nothing to do
734 input = marshal
.dumps(c
)
736 result
= p4CmdList("change -f -i", stdin
=input)
738 if r
.has_key('code'):
739 if r
['code'] == 'error':
740 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
741 if r
.has_key('data'):
742 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
744 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
746 def canChangeChangelists(self
):
747 # check to see if we have p4 admin or super-user permissions, either of
748 # which are required to modify changelists.
749 results
= p4CmdList("protects %s" % self
.depotPath
)
751 if r
.has_key('perm'):
752 if r
['perm'] == 'admin':
754 if r
['perm'] == 'super':
760 return self
.myP4UserId
762 results
= p4CmdList("user -o")
764 if r
.has_key('User'):
765 self
.myP4UserId
= r
['User']
767 die("Could not find your p4 user id")
769 def p4UserIsMe(self
, p4User
):
770 # return True if the given p4 user is actually me
772 if not p4User
or p4User
!= me
:
777 def prepareSubmitTemplate(self
):
778 # remove lines in the Files section that show changes to files outside the depot path we're committing into
780 inFilesSection
= False
781 for line
in p4_read_pipe_lines("change -o"):
782 if line
.endswith("\r\n"):
783 line
= line
[:-2] + "\n"
785 if line
.startswith("\t"):
786 # path starts and ends with a tab
788 lastTab
= path
.rfind("\t")
790 path
= path
[:lastTab
]
791 if not p4PathStartsWith(path
, self
.depotPath
):
794 inFilesSection
= False
796 if line
.startswith("Files:"):
797 inFilesSection
= True
803 def applyCommit(self
, id):
804 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
806 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
808 if not self
.detectRenames
:
809 # If not explicitly set check the config variable
810 self
.detectRenames
= gitConfig("git-p4.detectRenames")
812 if self
.detectRenames
.lower() == "false" or self
.detectRenames
== "":
814 elif self
.detectRenames
.lower() == "true":
817 diffOpts
= "-M%s" % self
.detectRenames
819 detectCopies
= gitConfig("git-p4.detectCopies")
820 if detectCopies
.lower() == "true":
822 elif detectCopies
!= "" and detectCopies
.lower() != "false":
823 diffOpts
+= " -C%s" % detectCopies
825 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
826 diffOpts
+= " --find-copies-harder"
828 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts
, id, id))
830 filesToDelete
= set()
832 filesToChangeExecBit
= {}
834 diff
= parseDiffTreeEntry(line
)
835 modifier
= diff
['status']
838 p4_system("edit \"%s\"" % path
)
839 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
840 filesToChangeExecBit
[path
] = diff
['dst_mode']
841 editedFiles
.add(path
)
842 elif modifier
== "A":
844 filesToChangeExecBit
[path
] = diff
['dst_mode']
845 if path
in filesToDelete
:
846 filesToDelete
.remove(path
)
847 elif modifier
== "D":
848 filesToDelete
.add(path
)
849 if path
in filesToAdd
:
850 filesToAdd
.remove(path
)
851 elif modifier
== "C":
852 src
, dest
= diff
['src'], diff
['dst']
853 p4_system("integrate -Dt \"%s\" \"%s\"" % (src
, dest
))
854 if diff
['src_sha1'] != diff
['dst_sha1']:
855 p4_system("edit \"%s\"" % (dest
))
856 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
857 p4_system("edit \"%s\"" % (dest
))
858 filesToChangeExecBit
[dest
] = diff
['dst_mode']
860 editedFiles
.add(dest
)
861 elif modifier
== "R":
862 src
, dest
= diff
['src'], diff
['dst']
863 p4_system("integrate -Dt \"%s\" \"%s\"" % (src
, dest
))
864 if diff
['src_sha1'] != diff
['dst_sha1']:
865 p4_system("edit \"%s\"" % (dest
))
866 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
867 p4_system("edit \"%s\"" % (dest
))
868 filesToChangeExecBit
[dest
] = diff
['dst_mode']
870 editedFiles
.add(dest
)
871 filesToDelete
.add(src
)
873 die("unknown modifier %s for %s" % (modifier
, path
))
875 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
876 patchcmd
= diffcmd
+ " | git apply "
877 tryPatchCmd
= patchcmd
+ "--check -"
878 applyPatchCmd
= patchcmd
+ "--check --apply -"
880 if os
.system(tryPatchCmd
) != 0:
881 print "Unfortunately applying the change failed!"
882 print "What do you want to do?"
884 while response
!= "s" and response
!= "a" and response
!= "w":
885 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
886 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
888 print "Skipping! Good luck with the next patches..."
889 for f
in editedFiles
:
890 p4_system("revert \"%s\"" % f
);
894 elif response
== "a":
895 os
.system(applyPatchCmd
)
896 if len(filesToAdd
) > 0:
897 print "You may also want to call p4 add on the following files:"
898 print " ".join(filesToAdd
)
899 if len(filesToDelete
):
900 print "The following files should be scheduled for deletion with p4 delete:"
901 print " ".join(filesToDelete
)
902 die("Please resolve and submit the conflict manually and "
903 + "continue afterwards with git-p4 submit --continue")
904 elif response
== "w":
905 system(diffcmd
+ " > patch.txt")
906 print "Patch saved to patch.txt in %s !" % self
.clientPath
907 die("Please resolve and submit the conflict manually and "
908 "continue afterwards with git-p4 submit --continue")
910 system(applyPatchCmd
)
913 p4_system("add \"%s\"" % f
)
914 for f
in filesToDelete
:
915 p4_system("revert \"%s\"" % f
)
916 p4_system("delete \"%s\"" % f
)
918 # Set/clear executable bits
919 for f
in filesToChangeExecBit
.keys():
920 mode
= filesToChangeExecBit
[f
]
921 setP4ExecBit(f
, mode
)
923 logMessage
= extractLogMessageFromGitCommit(id)
924 logMessage
= logMessage
.strip()
926 template
= self
.prepareSubmitTemplate()
929 submitTemplate
= self
.prepareLogMessage(template
, logMessage
)
931 if self
.preserveUser
:
932 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
934 if os
.environ
.has_key("P4DIFF"):
935 del(os
.environ
["P4DIFF"])
937 for editedFile
in editedFiles
:
938 diff
+= p4_read_pipe("diff -du %r" % editedFile
)
941 for newFile
in filesToAdd
:
942 newdiff
+= "==== new file ====\n"
943 newdiff
+= "--- /dev/null\n"
944 newdiff
+= "+++ %s\n" % newFile
945 f
= open(newFile
, "r")
946 for line
in f
.readlines():
947 newdiff
+= "+" + line
950 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
951 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
952 submitTemplate
+= "######## Use git-p4 option --preserve-user to modify authorship\n"
953 submitTemplate
+= "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
955 separatorLine
= "######## everything below this line is just the diff #######\n"
957 [handle
, fileName
] = tempfile
.mkstemp()
958 tmpFile
= os
.fdopen(handle
, "w+")
960 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
961 separatorLine
= separatorLine
.replace("\n", "\r\n")
962 newdiff
= newdiff
.replace("\n", "\r\n")
963 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
965 mtime
= os
.stat(fileName
).st_mtime
966 if os
.environ
.has_key("P4EDITOR"):
967 editor
= os
.environ
.get("P4EDITOR")
969 editor
= read_pipe("git var GIT_EDITOR").strip()
970 system(editor
+ " " + fileName
)
972 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
978 if checkModTime
and (os
.stat(fileName
).st_mtime
<= mtime
):
980 while response
!= "y" and response
!= "n":
981 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
984 tmpFile
= open(fileName
, "rb")
985 message
= tmpFile
.read()
987 submitTemplate
= message
[:message
.index(separatorLine
)]
989 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
990 p4_write_pipe("submit -i", submitTemplate
)
992 if self
.preserveUser
:
994 # Get last changelist number. Cannot easily get it from
995 # the submit command output as the output is unmarshalled.
996 changelist
= self
.lastP4Changelist()
997 self
.modifyChangelistUser(changelist
, p4User
)
1000 for f
in editedFiles
:
1001 p4_system("revert \"%s\"" % f
);
1002 for f
in filesToAdd
:
1003 p4_system("revert \"%s\"" % f
);
1008 fileName
= "submit.txt"
1009 file = open(fileName
, "w+")
1010 file.write(self
.prepareLogMessage(template
, logMessage
))
1012 print ("Perforce submit template written as %s. "
1013 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1014 % (fileName
, fileName
))
1016 def run(self
, args
):
1018 self
.master
= currentGitBranch()
1019 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1020 die("Detecting current git branch failed!")
1021 elif len(args
) == 1:
1022 self
.master
= args
[0]
1026 allowSubmit
= gitConfig("git-p4.allowSubmit")
1027 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1028 die("%s is not in git-p4.allowSubmit" % self
.master
)
1030 [upstream
, settings
] = findUpstreamBranchPoint()
1031 self
.depotPath
= settings
['depot-paths'][0]
1032 if len(self
.origin
) == 0:
1033 self
.origin
= upstream
1035 if self
.preserveUser
:
1036 if not self
.canChangeChangelists():
1037 die("Cannot preserve user names without p4 super-user or admin permissions")
1040 print "Origin branch is " + self
.origin
1042 if len(self
.depotPath
) == 0:
1043 print "Internal error: cannot locate perforce depot path from existing branches"
1046 self
.clientPath
= p4Where(self
.depotPath
)
1048 if len(self
.clientPath
) == 0:
1049 print "Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
1052 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1053 self
.oldWorkingDirectory
= os
.getcwd()
1055 chdir(self
.clientPath
)
1056 print "Synchronizing p4 checkout..."
1057 p4_system("sync ...")
1062 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1063 commits
.append(line
.strip())
1066 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1067 self
.checkAuthorship
= False
1069 self
.checkAuthorship
= True
1071 if self
.preserveUser
:
1072 self
.checkValidP4Users(commits
)
1074 while len(commits
) > 0:
1076 commits
= commits
[1:]
1077 self
.applyCommit(commit
)
1078 if not self
.interactive
:
1081 if len(commits
) == 0:
1082 print "All changes applied!"
1083 chdir(self
.oldWorkingDirectory
)
1093 class P4Sync(Command
, P4UserMap
):
1094 delete_actions
= ( "delete", "move/delete", "purge" )
1097 Command
.__init
__(self
)
1098 P4UserMap
.__init
__(self
)
1100 optparse
.make_option("--branch", dest
="branch"),
1101 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1102 optparse
.make_option("--changesfile", dest
="changesFile"),
1103 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1104 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1105 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
1106 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1107 help="Import into refs/heads/ , not refs/remotes"),
1108 optparse
.make_option("--max-changes", dest
="maxChanges"),
1109 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1110 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1111 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1112 help="Only sync files that are included in the Perforce Client Spec")
1114 self
.description
= """Imports from Perforce into a git repository.\n
1116 //depot/my/project/ -- to import the current head
1117 //depot/my/project/@all -- to import everything
1118 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1120 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1122 self
.usage
+= " //depot/path[@revRange]"
1124 self
.createdBranches
= set()
1125 self
.committedChanges
= set()
1127 self
.detectBranches
= False
1128 self
.detectLabels
= False
1129 self
.changesFile
= ""
1130 self
.syncWithOrigin
= True
1131 self
.verbose
= False
1132 self
.importIntoRemotes
= True
1133 self
.maxChanges
= ""
1134 self
.isWindows
= (platform
.system() == "Windows")
1135 self
.keepRepoPath
= False
1136 self
.depotPaths
= None
1137 self
.p4BranchesInGit
= []
1138 self
.cloneExclude
= []
1139 self
.useClientSpec
= False
1140 self
.clientSpecDirs
= []
1142 if gitConfig("git-p4.syncFromOrigin") == "false":
1143 self
.syncWithOrigin
= False
1146 # P4 wildcards are not allowed in filenames. P4 complains
1147 # if you simply add them, but you can force it with "-f", in
1148 # which case it translates them into %xx encoding internally.
1149 # Search for and fix just these four characters. Do % last so
1150 # that fixing it does not inadvertently create new %-escapes.
1152 def wildcard_decode(self
, path
):
1153 # Cannot have * in a filename in windows; untested as to
1154 # what p4 would do in such a case.
1155 if not self
.isWindows
:
1156 path
= path
.replace("%2A", "*")
1157 path
= path
.replace("%23", "#") \
1158 .replace("%40", "@") \
1159 .replace("%25", "%")
1162 def extractFilesFromCommit(self
, commit
):
1163 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1164 for path
in self
.cloneExclude
]
1167 while commit
.has_key("depotFile%s" % fnum
):
1168 path
= commit
["depotFile%s" % fnum
]
1170 if [p
for p
in self
.cloneExclude
1171 if p4PathStartsWith(path
, p
)]:
1174 found
= [p
for p
in self
.depotPaths
1175 if p4PathStartsWith(path
, p
)]
1182 file["rev"] = commit
["rev%s" % fnum
]
1183 file["action"] = commit
["action%s" % fnum
]
1184 file["type"] = commit
["type%s" % fnum
]
1189 def stripRepoPath(self
, path
, prefixes
):
1190 if self
.useClientSpec
:
1192 # if using the client spec, we use the output directory
1193 # specified in the client. For example, a view
1194 # //depot/foo/branch/... //client/branch/foo/...
1195 # will end up putting all foo/branch files into
1197 for val
in self
.clientSpecDirs
:
1198 if path
.startswith(val
[0]):
1199 # replace the depot path with the client path
1200 path
= path
.replace(val
[0], val
[1][1])
1201 # now strip out the client (//client/...)
1202 path
= re
.sub("^(//[^/]+/)", '', path
)
1203 # the rest is all path
1206 if self
.keepRepoPath
:
1207 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1210 if p4PathStartsWith(path
, p
):
1211 path
= path
[len(p
):]
1215 def splitFilesIntoBranches(self
, commit
):
1218 while commit
.has_key("depotFile%s" % fnum
):
1219 path
= commit
["depotFile%s" % fnum
]
1220 found
= [p
for p
in self
.depotPaths
1221 if p4PathStartsWith(path
, p
)]
1228 file["rev"] = commit
["rev%s" % fnum
]
1229 file["action"] = commit
["action%s" % fnum
]
1230 file["type"] = commit
["type%s" % fnum
]
1233 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1235 for branch
in self
.knownBranches
.keys():
1237 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1238 if relPath
.startswith(branch
+ "/"):
1239 if branch
not in branches
:
1240 branches
[branch
] = []
1241 branches
[branch
].append(file)
1246 # output one file from the P4 stream
1247 # - helper for streamP4Files
1249 def streamOneP4File(self
, file, contents
):
1250 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1251 relPath
= self
.wildcard_decode(relPath
)
1253 sys
.stderr
.write("%s\n" % relPath
)
1255 (type_base
, type_mods
) = split_p4_type(file["type"])
1258 if "x" in type_mods
:
1260 if type_base
== "symlink":
1262 # p4 print on a symlink contains "target\n"; remove the newline
1263 data
= ''.join(contents
)
1264 contents
= [data
[:-1]]
1266 if type_base
== "utf16":
1267 # p4 delivers different text in the python output to -G
1268 # than it does when using "print -o", or normal p4 client
1269 # operations. utf16 is converted to ascii or utf8, perhaps.
1270 # But ascii text saved as -t utf16 is completely mangled.
1271 # Invoke print -o to get the real contents.
1272 text
= p4_read_pipe('print -q -o - "%s"' % file['depotFile'])
1275 # Perhaps windows wants unicode, utf16 newlines translated too;
1276 # but this is not doing it.
1277 if self
.isWindows
and type_base
== "text":
1279 for data
in contents
:
1280 data
= data
.replace("\r\n", "\n")
1281 mangled
.append(data
)
1284 # Note that we do not try to de-mangle keywords on utf16 files,
1285 # even though in theory somebody may want that.
1286 if type_base
in ("text", "unicode", "binary"):
1287 if "ko" in type_mods
:
1288 contents
= map(lambda text
: re
.sub(r
'(?i)\$(Id|Header):[^$]*\$', r
'$\1$', text
), contents
)
1289 elif "k" in type_mods
:
1290 contents
= map(lambda text
: re
.sub(r
'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$', r
'$\1$', text
), contents
)
1292 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1297 length
= length
+ len(d
)
1299 self
.gitStream
.write("data %d\n" % length
)
1301 self
.gitStream
.write(d
)
1302 self
.gitStream
.write("\n")
1304 def streamOneP4Deletion(self
, file):
1305 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1307 sys
.stderr
.write("delete %s\n" % relPath
)
1308 self
.gitStream
.write("D %s\n" % relPath
)
1310 # handle another chunk of streaming data
1311 def streamP4FilesCb(self
, marshalled
):
1313 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1314 # start of a new file - output the old one first
1315 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1316 self
.stream_file
= {}
1317 self
.stream_contents
= []
1318 self
.stream_have_file_info
= False
1320 # pick up the new file information... for the
1321 # 'data' field we need to append to our array
1322 for k
in marshalled
.keys():
1324 self
.stream_contents
.append(marshalled
['data'])
1326 self
.stream_file
[k
] = marshalled
[k
]
1328 self
.stream_have_file_info
= True
1330 # Stream directly from "p4 files" into "git fast-import"
1331 def streamP4Files(self
, files
):
1338 for val
in self
.clientSpecDirs
:
1339 if f
['path'].startswith(val
[0]):
1345 filesForCommit
.append(f
)
1346 if f
['action'] in self
.delete_actions
:
1347 filesToDelete
.append(f
)
1349 filesToRead
.append(f
)
1352 for f
in filesToDelete
:
1353 self
.streamOneP4Deletion(f
)
1355 if len(filesToRead
) > 0:
1356 self
.stream_file
= {}
1357 self
.stream_contents
= []
1358 self
.stream_have_file_info
= False
1360 # curry self argument
1361 def streamP4FilesCbSelf(entry
):
1362 self
.streamP4FilesCb(entry
)
1364 p4CmdList("-x - print",
1365 '\n'.join(['%s#%s' % (f
['path'], f
['rev'])
1366 for f
in filesToRead
]),
1367 cb
=streamP4FilesCbSelf
)
1370 if self
.stream_file
.has_key('depotFile'):
1371 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1373 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
1374 epoch
= details
["time"]
1375 author
= details
["user"]
1376 self
.branchPrefixes
= branchPrefixes
1379 print "commit into %s" % branch
1381 # start with reading files; if that fails, we should not
1385 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
1386 new_files
.append (f
)
1388 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
1390 self
.gitStream
.write("commit %s\n" % branch
)
1391 # gitStream.write("mark :%s\n" % details["change"])
1392 self
.committedChanges
.add(int(details
["change"]))
1394 if author
not in self
.users
:
1395 self
.getUserMapFromPerforceServer()
1396 if author
in self
.users
:
1397 committer
= "%s %s %s" % (self
.users
[author
], epoch
, self
.tz
)
1399 committer
= "%s <a@b> %s %s" % (author
, epoch
, self
.tz
)
1401 self
.gitStream
.write("committer %s\n" % committer
)
1403 self
.gitStream
.write("data <<EOT\n")
1404 self
.gitStream
.write(details
["desc"])
1405 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1406 % (','.join (branchPrefixes
), details
["change"]))
1407 if len(details
['options']) > 0:
1408 self
.gitStream
.write(": options = %s" % details
['options'])
1409 self
.gitStream
.write("]\nEOT\n\n")
1413 print "parent %s" % parent
1414 self
.gitStream
.write("from %s\n" % parent
)
1416 self
.streamP4Files(new_files
)
1417 self
.gitStream
.write("\n")
1419 change
= int(details
["change"])
1421 if self
.labels
.has_key(change
):
1422 label
= self
.labels
[change
]
1423 labelDetails
= label
[0]
1424 labelRevisions
= label
[1]
1426 print "Change %s is labelled %s" % (change
, labelDetails
)
1428 files
= p4CmdList("files " + ' '.join (["%s...@%s" % (p
, change
)
1429 for p
in branchPrefixes
]))
1431 if len(files
) == len(labelRevisions
):
1435 if info
["action"] in self
.delete_actions
:
1437 cleanedFiles
[info
["depotFile"]] = info
["rev"]
1439 if cleanedFiles
== labelRevisions
:
1440 self
.gitStream
.write("tag tag_%s\n" % labelDetails
["label"])
1441 self
.gitStream
.write("from %s\n" % branch
)
1443 owner
= labelDetails
["Owner"]
1445 if author
in self
.users
:
1446 tagger
= "%s %s %s" % (self
.users
[owner
], epoch
, self
.tz
)
1448 tagger
= "%s <a@b> %s %s" % (owner
, epoch
, self
.tz
)
1449 self
.gitStream
.write("tagger %s\n" % tagger
)
1450 self
.gitStream
.write("data <<EOT\n")
1451 self
.gitStream
.write(labelDetails
["Description"])
1452 self
.gitStream
.write("EOT\n\n")
1456 print ("Tag %s does not match with change %s: files do not match."
1457 % (labelDetails
["label"], change
))
1461 print ("Tag %s does not match with change %s: file count is different."
1462 % (labelDetails
["label"], change
))
1464 def getLabels(self
):
1467 l
= p4CmdList("labels %s..." % ' '.join (self
.depotPaths
))
1468 if len(l
) > 0 and not self
.silent
:
1469 print "Finding files belonging to labels in %s" % `self
.depotPaths`
1472 label
= output
["label"]
1476 print "Querying files for label %s" % label
1477 for file in p4CmdList("files "
1478 + ' '.join (["%s...@%s" % (p
, label
)
1479 for p
in self
.depotPaths
])):
1480 revisions
[file["depotFile"]] = file["rev"]
1481 change
= int(file["change"])
1482 if change
> newestChange
:
1483 newestChange
= change
1485 self
.labels
[newestChange
] = [output
, revisions
]
1488 print "Label changes: %s" % self
.labels
.keys()
1490 def guessProjectName(self
):
1491 for p
in self
.depotPaths
:
1494 p
= p
[p
.strip().rfind("/") + 1:]
1495 if not p
.endswith("/"):
1499 def getBranchMapping(self
):
1500 lostAndFoundBranches
= set()
1502 user
= gitConfig("git-p4.branchUser")
1504 command
= "branches -u %s" % user
1506 command
= "branches"
1508 for info
in p4CmdList(command
):
1509 details
= p4Cmd("branch -o %s" % info
["branch"])
1511 while details
.has_key("View%s" % viewIdx
):
1512 paths
= details
["View%s" % viewIdx
].split(" ")
1513 viewIdx
= viewIdx
+ 1
1514 # require standard //depot/foo/... //depot/bar/... mapping
1515 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
1518 destination
= paths
[1]
1520 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
1521 source
= source
[len(self
.depotPaths
[0]):-4]
1522 destination
= destination
[len(self
.depotPaths
[0]):-4]
1524 if destination
in self
.knownBranches
:
1526 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
1527 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
1530 self
.knownBranches
[destination
] = source
1532 lostAndFoundBranches
.discard(destination
)
1534 if source
not in self
.knownBranches
:
1535 lostAndFoundBranches
.add(source
)
1537 # Perforce does not strictly require branches to be defined, so we also
1538 # check git config for a branch list.
1540 # Example of branch definition in git config file:
1542 # branchList=main:branchA
1543 # branchList=main:branchB
1544 # branchList=branchA:branchC
1545 configBranches
= gitConfigList("git-p4.branchList")
1546 for branch
in configBranches
:
1548 (source
, destination
) = branch
.split(":")
1549 self
.knownBranches
[destination
] = source
1551 lostAndFoundBranches
.discard(destination
)
1553 if source
not in self
.knownBranches
:
1554 lostAndFoundBranches
.add(source
)
1557 for branch
in lostAndFoundBranches
:
1558 self
.knownBranches
[branch
] = branch
1560 def getBranchMappingFromGitBranches(self
):
1561 branches
= p4BranchesInGit(self
.importIntoRemotes
)
1562 for branch
in branches
.keys():
1563 if branch
== "master":
1566 branch
= branch
[len(self
.projectName
):]
1567 self
.knownBranches
[branch
] = branch
1569 def listExistingP4GitBranches(self
):
1570 # branches holds mapping from name to commit
1571 branches
= p4BranchesInGit(self
.importIntoRemotes
)
1572 self
.p4BranchesInGit
= branches
.keys()
1573 for branch
in branches
.keys():
1574 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
1576 def updateOptionDict(self
, d
):
1578 if self
.keepRepoPath
:
1579 option_keys
['keepRepoPath'] = 1
1581 d
["options"] = ' '.join(sorted(option_keys
.keys()))
1583 def readOptions(self
, d
):
1584 self
.keepRepoPath
= (d
.has_key('options')
1585 and ('keepRepoPath' in d
['options']))
1587 def gitRefForBranch(self
, branch
):
1588 if branch
== "main":
1589 return self
.refPrefix
+ "master"
1591 if len(branch
) <= 0:
1594 return self
.refPrefix
+ self
.projectName
+ branch
1596 def gitCommitByP4Change(self
, ref
, change
):
1598 print "looking in ref " + ref
+ " for change %s using bisect..." % change
1601 latestCommit
= parseRevision(ref
)
1605 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
1606 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
1611 log
= extractLogMessageFromGitCommit(next
)
1612 settings
= extractSettingsGitLog(log
)
1613 currentChange
= int(settings
['change'])
1615 print "current change %s" % currentChange
1617 if currentChange
== change
:
1619 print "found %s" % next
1622 if currentChange
< change
:
1623 earliestCommit
= "^%s" % next
1625 latestCommit
= "%s" % next
1629 def importNewBranch(self
, branch
, maxChange
):
1630 # make fast-import flush all changes to disk and update the refs using the checkpoint
1631 # command so that we can try to find the branch parent in the git history
1632 self
.gitStream
.write("checkpoint\n\n");
1633 self
.gitStream
.flush();
1634 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
1635 range = "@1,%s" % maxChange
1636 #print "prefix" + branchPrefix
1637 changes
= p4ChangesForPaths([branchPrefix
], range)
1638 if len(changes
) <= 0:
1640 firstChange
= changes
[0]
1641 #print "first change in branch: %s" % firstChange
1642 sourceBranch
= self
.knownBranches
[branch
]
1643 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
1644 sourceRef
= self
.gitRefForBranch(sourceBranch
)
1645 #print "source " + sourceBranch
1647 branchParentChange
= int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath
, firstChange
))["change"])
1648 #print "branch parent: %s" % branchParentChange
1649 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
1650 if len(gitParent
) > 0:
1651 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
1652 #print "parent git commit: %s" % gitParent
1654 self
.importChanges(changes
)
1657 def importChanges(self
, changes
):
1659 for change
in changes
:
1660 description
= p4Cmd("describe %s" % change
)
1661 self
.updateOptionDict(description
)
1664 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
1669 if self
.detectBranches
:
1670 branches
= self
.splitFilesIntoBranches(description
)
1671 for branch
in branches
.keys():
1673 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
1677 filesForCommit
= branches
[branch
]
1680 print "branch is %s" % branch
1682 self
.updatedBranches
.add(branch
)
1684 if branch
not in self
.createdBranches
:
1685 self
.createdBranches
.add(branch
)
1686 parent
= self
.knownBranches
[branch
]
1687 if parent
== branch
:
1690 fullBranch
= self
.projectName
+ branch
1691 if fullBranch
not in self
.p4BranchesInGit
:
1693 print("\n Importing new branch %s" % fullBranch
);
1694 if self
.importNewBranch(branch
, change
- 1):
1696 self
.p4BranchesInGit
.append(fullBranch
)
1698 print("\n Resuming with change %s" % change
);
1701 print "parent determined through known branches: %s" % parent
1703 branch
= self
.gitRefForBranch(branch
)
1704 parent
= self
.gitRefForBranch(parent
)
1707 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
1709 if len(parent
) == 0 and branch
in self
.initialParents
:
1710 parent
= self
.initialParents
[branch
]
1711 del self
.initialParents
[branch
]
1713 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
1715 files
= self
.extractFilesFromCommit(description
)
1716 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
1718 self
.initialParent
= ""
1720 print self
.gitError
.read()
1723 def importHeadRevision(self
, revision
):
1724 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
1727 details
["user"] = "git perforce import user"
1728 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
1729 % (' '.join(self
.depotPaths
), revision
))
1730 details
["change"] = revision
1734 for info
in p4CmdList("files "
1735 + ' '.join(["%s...%s"
1737 for p
in self
.depotPaths
])):
1739 if 'code' in info
and info
['code'] == 'error':
1740 sys
.stderr
.write("p4 returned an error: %s\n"
1742 if info
['data'].find("must refer to client") >= 0:
1743 sys
.stderr
.write("This particular p4 error is misleading.\n")
1744 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
1745 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
1747 if 'p4ExitCode' in info
:
1748 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
1752 change
= int(info
["change"])
1753 if change
> newestRevision
:
1754 newestRevision
= change
1756 if info
["action"] in self
.delete_actions
:
1757 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1758 #fileCnt = fileCnt + 1
1761 for prop
in ["depotFile", "rev", "action", "type" ]:
1762 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
1764 fileCnt
= fileCnt
+ 1
1766 details
["change"] = newestRevision
1768 # Use time from top-most change so that all git-p4 clones of
1769 # the same p4 repo have the same commit SHA1s.
1770 res
= p4CmdList("describe -s %d" % newestRevision
)
1773 if r
.has_key('time'):
1774 newestTime
= int(r
['time'])
1775 if newestTime
is None:
1776 die("\"describe -s\" on newest change %d did not give a time")
1777 details
["time"] = newestTime
1779 self
.updateOptionDict(details
)
1781 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
1783 print "IO error with git fast-import. Is your git version recent enough?"
1784 print self
.gitError
.read()
1787 def getClientSpec(self
):
1788 specList
= p4CmdList( "client -o" )
1790 for entry
in specList
:
1791 for k
,v
in entry
.iteritems():
1792 if k
.startswith("View"):
1794 # p4 has these %%1 to %%9 arguments in specs to
1795 # reorder paths; which we can't handle (yet :)
1796 if re
.match('%%\d', v
) != None:
1797 print "Sorry, can't handle %%n arguments in client specs"
1800 if v
.startswith('"'):
1804 index
= v
.find("...")
1806 # save the "client view"; i.e the RHS of the view
1807 # line that tells the client where to put the
1808 # files for this view.
1809 cv
= v
[index
+3:].strip() # +3 to remove previous '...'
1811 # if the client view doesn't end with a
1812 # ... wildcard, then we're going to mess up the
1813 # output directory, so fail gracefully.
1814 if not cv
.endswith('...'):
1815 print 'Sorry, client view in "%s" needs to end with wildcard' % (k
)
1819 # now save the view; +index means included, -index
1820 # means it should be filtered out.
1822 if v
.startswith("-"):
1828 temp
[v
] = (include
, cv
)
1830 self
.clientSpecDirs
= temp
.items()
1831 self
.clientSpecDirs
.sort( lambda x
, y
: abs( y
[1][0] ) - abs( x
[1][0] ) )
1833 def run(self
, args
):
1834 self
.depotPaths
= []
1835 self
.changeRange
= ""
1836 self
.initialParent
= ""
1837 self
.previousDepotPaths
= []
1839 # map from branch depot path to parent branch
1840 self
.knownBranches
= {}
1841 self
.initialParents
= {}
1842 self
.hasOrigin
= originP4BranchesExist()
1843 if not self
.syncWithOrigin
:
1844 self
.hasOrigin
= False
1846 if self
.importIntoRemotes
:
1847 self
.refPrefix
= "refs/remotes/p4/"
1849 self
.refPrefix
= "refs/heads/p4/"
1851 if self
.syncWithOrigin
and self
.hasOrigin
:
1853 print "Syncing with origin first by calling git fetch origin"
1854 system("git fetch origin")
1856 if len(self
.branch
) == 0:
1857 self
.branch
= self
.refPrefix
+ "master"
1858 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
1859 system("git update-ref %s refs/heads/p4" % self
.branch
)
1860 system("git branch -D p4");
1861 # create it /after/ importing, when master exists
1862 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
1863 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
1865 if self
.useClientSpec
or gitConfig("git-p4.useclientspec") == "true":
1866 self
.getClientSpec()
1868 # TODO: should always look at previous commits,
1869 # merge with previous imports, if possible.
1872 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
1873 self
.listExistingP4GitBranches()
1875 if len(self
.p4BranchesInGit
) > 1:
1877 print "Importing from/into multiple branches"
1878 self
.detectBranches
= True
1881 print "branches: %s" % self
.p4BranchesInGit
1884 for branch
in self
.p4BranchesInGit
:
1885 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
1887 settings
= extractSettingsGitLog(logMsg
)
1889 self
.readOptions(settings
)
1890 if (settings
.has_key('depot-paths')
1891 and settings
.has_key ('change')):
1892 change
= int(settings
['change']) + 1
1893 p4Change
= max(p4Change
, change
)
1895 depotPaths
= sorted(settings
['depot-paths'])
1896 if self
.previousDepotPaths
== []:
1897 self
.previousDepotPaths
= depotPaths
1900 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
1901 prev_list
= prev
.split("/")
1902 cur_list
= cur
.split("/")
1903 for i
in range(0, min(len(cur_list
), len(prev_list
))):
1904 if cur_list
[i
] <> prev_list
[i
]:
1908 paths
.append ("/".join(cur_list
[:i
+ 1]))
1910 self
.previousDepotPaths
= paths
1913 self
.depotPaths
= sorted(self
.previousDepotPaths
)
1914 self
.changeRange
= "@%s,#head" % p4Change
1915 if not self
.detectBranches
:
1916 self
.initialParent
= parseRevision(self
.branch
)
1917 if not self
.silent
and not self
.detectBranches
:
1918 print "Performing incremental import into %s git branch" % self
.branch
1920 if not self
.branch
.startswith("refs/"):
1921 self
.branch
= "refs/heads/" + self
.branch
1923 if len(args
) == 0 and self
.depotPaths
:
1925 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
1927 if self
.depotPaths
and self
.depotPaths
!= args
:
1928 print ("previous import used depot path %s and now %s was specified. "
1929 "This doesn't work!" % (' '.join (self
.depotPaths
),
1933 self
.depotPaths
= sorted(args
)
1939 for p
in self
.depotPaths
:
1940 if p
.find("@") != -1:
1941 atIdx
= p
.index("@")
1942 self
.changeRange
= p
[atIdx
:]
1943 if self
.changeRange
== "@all":
1944 self
.changeRange
= ""
1945 elif ',' not in self
.changeRange
:
1946 revision
= self
.changeRange
1947 self
.changeRange
= ""
1949 elif p
.find("#") != -1:
1950 hashIdx
= p
.index("#")
1951 revision
= p
[hashIdx
:]
1953 elif self
.previousDepotPaths
== []:
1956 p
= re
.sub ("\.\.\.$", "", p
)
1957 if not p
.endswith("/"):
1962 self
.depotPaths
= newPaths
1965 self
.loadUserMapFromCache()
1967 if self
.detectLabels
:
1970 if self
.detectBranches
:
1971 ## FIXME - what's a P4 projectName ?
1972 self
.projectName
= self
.guessProjectName()
1975 self
.getBranchMappingFromGitBranches()
1977 self
.getBranchMapping()
1979 print "p4-git branches: %s" % self
.p4BranchesInGit
1980 print "initial parents: %s" % self
.initialParents
1981 for b
in self
.p4BranchesInGit
:
1985 b
= b
[len(self
.projectName
):]
1986 self
.createdBranches
.add(b
)
1988 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
1990 importProcess
= subprocess
.Popen(["git", "fast-import"],
1991 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
1992 stderr
=subprocess
.PIPE
);
1993 self
.gitOutput
= importProcess
.stdout
1994 self
.gitStream
= importProcess
.stdin
1995 self
.gitError
= importProcess
.stderr
1998 self
.importHeadRevision(revision
)
2002 if len(self
.changesFile
) > 0:
2003 output
= open(self
.changesFile
).readlines()
2006 changeSet
.add(int(line
))
2008 for change
in changeSet
:
2009 changes
.append(change
)
2013 # catch "git-p4 sync" with no new branches, in a repo that
2014 # does not have any existing git-p4 branches
2015 if len(args
) == 0 and not self
.p4BranchesInGit
:
2016 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2018 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2020 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2022 if len(self
.maxChanges
) > 0:
2023 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2025 if len(changes
) == 0:
2027 print "No changes to import!"
2030 if not self
.silent
and not self
.detectBranches
:
2031 print "Import destination: %s" % self
.branch
2033 self
.updatedBranches
= set()
2035 self
.importChanges(changes
)
2039 if len(self
.updatedBranches
) > 0:
2040 sys
.stdout
.write("Updated branches: ")
2041 for b
in self
.updatedBranches
:
2042 sys
.stdout
.write("%s " % b
)
2043 sys
.stdout
.write("\n")
2045 self
.gitStream
.close()
2046 if importProcess
.wait() != 0:
2047 die("fast-import failed: %s" % self
.gitError
.read())
2048 self
.gitOutput
.close()
2049 self
.gitError
.close()
2053 class P4Rebase(Command
):
2055 Command
.__init
__(self
)
2057 self
.description
= ("Fetches the latest revision from perforce and "
2058 + "rebases the current work (branch) against it")
2059 self
.verbose
= False
2061 def run(self
, args
):
2065 return self
.rebase()
2068 if os
.system("git update-index --refresh") != 0:
2069 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2070 if len(read_pipe("git diff-index HEAD --")) > 0:
2071 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2073 [upstream
, settings
] = findUpstreamBranchPoint()
2074 if len(upstream
) == 0:
2075 die("Cannot find upstream branchpoint for rebase")
2077 # the branchpoint may be p4/foo~3, so strip off the parent
2078 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2080 print "Rebasing the current branch onto %s" % upstream
2081 oldHead
= read_pipe("git rev-parse HEAD").strip()
2082 system("git rebase %s" % upstream
)
2083 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2086 class P4Clone(P4Sync
):
2088 P4Sync
.__init
__(self
)
2089 self
.description
= "Creates a new git repository and imports from Perforce into it"
2090 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2092 optparse
.make_option("--destination", dest
="cloneDestination",
2093 action
='store', default
=None,
2094 help="where to leave result of the clone"),
2095 optparse
.make_option("-/", dest
="cloneExclude",
2096 action
="append", type="string",
2097 help="exclude depot path"),
2098 optparse
.make_option("--bare", dest
="cloneBare",
2099 action
="store_true", default
=False),
2101 self
.cloneDestination
= None
2102 self
.needsGit
= False
2103 self
.cloneBare
= False
2105 # This is required for the "append" cloneExclude action
2106 def ensure_value(self
, attr
, value
):
2107 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2108 setattr(self
, attr
, value
)
2109 return getattr(self
, attr
)
2111 def defaultDestination(self
, args
):
2112 ## TODO: use common prefix of args?
2114 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2115 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2116 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2117 depotDir
= re
.sub(r
"/$", "", depotDir
)
2118 return os
.path
.split(depotDir
)[1]
2120 def run(self
, args
):
2124 if self
.keepRepoPath
and not self
.cloneDestination
:
2125 sys
.stderr
.write("Must specify destination for --keep-path\n")
2130 if not self
.cloneDestination
and len(depotPaths
) > 1:
2131 self
.cloneDestination
= depotPaths
[-1]
2132 depotPaths
= depotPaths
[:-1]
2134 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2135 for p
in depotPaths
:
2136 if not p
.startswith("//"):
2139 if not self
.cloneDestination
:
2140 self
.cloneDestination
= self
.defaultDestination(args
)
2142 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2144 if not os
.path
.exists(self
.cloneDestination
):
2145 os
.makedirs(self
.cloneDestination
)
2146 chdir(self
.cloneDestination
)
2148 init_cmd
= [ "git", "init" ]
2150 init_cmd
.append("--bare")
2151 subprocess
.check_call(init_cmd
)
2153 if not P4Sync
.run(self
, depotPaths
):
2155 if self
.branch
!= "master":
2156 if self
.importIntoRemotes
:
2157 masterbranch
= "refs/remotes/p4/master"
2159 masterbranch
= "refs/heads/p4/master"
2160 if gitBranchExists(masterbranch
):
2161 system("git branch master %s" % masterbranch
)
2162 if not self
.cloneBare
:
2163 system("git checkout -f")
2165 print "Could not detect main branch. No checkout/master branch created."
2169 class P4Branches(Command
):
2171 Command
.__init
__(self
)
2173 self
.description
= ("Shows the git branches that hold imports and their "
2174 + "corresponding perforce depot paths")
2175 self
.verbose
= False
2177 def run(self
, args
):
2178 if originP4BranchesExist():
2179 createOrUpdateBranchesFromOrigin()
2181 cmdline
= "git rev-parse --symbolic "
2182 cmdline
+= " --remotes"
2184 for line
in read_pipe_lines(cmdline
):
2187 if not line
.startswith('p4/') or line
== "p4/HEAD":
2191 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2192 settings
= extractSettingsGitLog(log
)
2194 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2197 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2199 optparse
.IndentedHelpFormatter
.__init
__(self
)
2201 def format_description(self
, description
):
2203 return description
+ "\n"
2207 def printUsage(commands
):
2208 print "usage: %s <command> [options]" % sys
.argv
[0]
2210 print "valid commands: %s" % ", ".join(commands
)
2212 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2217 "submit" : P4Submit
,
2218 "commit" : P4Submit
,
2220 "rebase" : P4Rebase
,
2222 "rollback" : P4RollBack
,
2223 "branches" : P4Branches
2228 if len(sys
.argv
[1:]) == 0:
2229 printUsage(commands
.keys())
2233 cmdName
= sys
.argv
[1]
2235 klass
= commands
[cmdName
]
2238 print "unknown command %s" % cmdName
2240 printUsage(commands
.keys())
2243 options
= cmd
.options
2244 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2248 if len(options
) > 0:
2249 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2251 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2253 description
= cmd
.description
,
2254 formatter
= HelpFormatter())
2256 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2258 verbose
= cmd
.verbose
2260 if cmd
.gitdir
== None:
2261 cmd
.gitdir
= os
.path
.abspath(".git")
2262 if not isValidGitDir(cmd
.gitdir
):
2263 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2264 if os
.path
.exists(cmd
.gitdir
):
2265 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2269 if not isValidGitDir(cmd
.gitdir
):
2270 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2271 cmd
.gitdir
+= "/.git"
2273 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2275 os
.environ
["GIT_DIR"] = cmd
.gitdir
2277 if not cmd
.run(args
):
2281 if __name__
== '__main__':