3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
124 expand
= isinstance(cmd
,basestring
)
126 sys
.stderr
.write("executing %s\n" % str(cmd
))
127 subprocess
.check_call(cmd
, shell
=expand
)
130 """Specifically invoke p4 as the system command. """
131 real_cmd
= p4_build_cmd(cmd
)
132 expand
= isinstance(real_cmd
, basestring
)
133 subprocess
.check_call(real_cmd
, shell
=expand
)
135 def p4_integrate(src
, dest
):
136 p4_system(["integrate", "-Dt", src
, dest
])
139 p4_system(["sync", path
])
142 p4_system(["add", f
])
145 p4_system(["delete", f
])
148 p4_system(["edit", f
])
151 p4_system(["revert", f
])
153 def p4_reopen(type, file):
154 p4_system(["reopen", "-t", type, file])
157 # Canonicalize the p4 type and return a tuple of the
158 # base type, plus any modifiers. See "p4 help filetypes"
159 # for a list and explanation.
161 def split_p4_type(p4type
):
163 p4_filetypes_historical
= {
164 "ctempobj": "binary+Sw",
170 "tempobj": "binary+FSw",
171 "ubinary": "binary+F",
172 "uresource": "resource+F",
173 "uxbinary": "binary+Fx",
174 "xbinary": "binary+x",
176 "xtempobj": "binary+Swx",
178 "xunicode": "unicode+x",
181 if p4type
in p4_filetypes_historical
:
182 p4type
= p4_filetypes_historical
[p4type
]
184 s
= p4type
.split("+")
192 # return the raw p4 type of a file (text, text+ko, etc)
195 results
= p4CmdList(["fstat", "-T", "headType", file])
196 return results
[0]['headType']
199 # Given a type base and modifier, return a regexp matching
200 # the keywords that can be expanded in the file
202 def p4_keywords_regexp_for_type(base
, type_mods
):
203 if base
in ("text", "unicode", "binary"):
205 if "ko" in type_mods
:
207 elif "k" in type_mods
:
208 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
212 \$ # Starts with a dollar, followed by...
213 (%s) # one of the keywords, followed by...
214 (:[^$]+)? # possibly an old expansion, followed by...
222 # Given a file, return a regexp matching the possible
223 # RCS keywords that will be expanded, or None for files
224 # with kw expansion turned off.
226 def p4_keywords_regexp_for_file(file):
227 if not os
.path
.exists(file):
230 (type_base
, type_mods
) = split_p4_type(p4_type(file))
231 return p4_keywords_regexp_for_type(type_base
, type_mods
)
233 def setP4ExecBit(file, mode
):
234 # Reopens an already open file and changes the execute bit to match
235 # the execute bit setting in the passed in mode.
239 if not isModeExec(mode
):
240 p4Type
= getP4OpenedType(file)
241 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
242 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
243 if p4Type
[-1] == "+":
244 p4Type
= p4Type
[0:-1]
246 p4_reopen(p4Type
, file)
248 def getP4OpenedType(file):
249 # Returns the perforce file type for the given file.
251 result
= p4_read_pipe(["opened", file])
252 match
= re
.match(".*\((.+)\)\r?$", result
)
254 return match
.group(1)
256 die("Could not determine file type for %s (result: '%s')" % (file, result
))
258 # Return the set of all p4 labels
259 def getP4Labels(depotPaths
):
261 if isinstance(depotPaths
,basestring
):
262 depotPaths
= [depotPaths
]
264 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
270 # Return the set of all git tags
273 for line
in read_pipe_lines(["git", "tag"]):
278 def diffTreePattern():
279 # This is a simple generator for the diff tree regex pattern. This could be
280 # a class variable if this and parseDiffTreeEntry were a part of a class.
281 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
285 def parseDiffTreeEntry(entry
):
286 """Parses a single diff tree entry into its component elements.
288 See git-diff-tree(1) manpage for details about the format of the diff
289 output. This method returns a dictionary with the following elements:
291 src_mode - The mode of the source file
292 dst_mode - The mode of the destination file
293 src_sha1 - The sha1 for the source file
294 dst_sha1 - The sha1 fr the destination file
295 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
296 status_score - The score for the status (applicable for 'C' and 'R'
297 statuses). This is None if there is no score.
298 src - The path for the source file.
299 dst - The path for the destination file. This is only present for
300 copy or renames. If it is not present, this is None.
302 If the pattern is not matched, None is returned."""
304 match
= diffTreePattern().next().match(entry
)
307 'src_mode': match
.group(1),
308 'dst_mode': match
.group(2),
309 'src_sha1': match
.group(3),
310 'dst_sha1': match
.group(4),
311 'status': match
.group(5),
312 'status_score': match
.group(6),
313 'src': match
.group(7),
314 'dst': match
.group(10)
318 def isModeExec(mode
):
319 # Returns True if the given git mode represents an executable file,
321 return mode
[-3:] == "755"
323 def isModeExecChanged(src_mode
, dst_mode
):
324 return isModeExec(src_mode
) != isModeExec(dst_mode
)
326 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
328 if isinstance(cmd
,basestring
):
335 cmd
= p4_build_cmd(cmd
)
337 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
339 # Use a temporary file to avoid deadlocks without
340 # subprocess.communicate(), which would put another copy
341 # of stdout into memory.
343 if stdin
is not None:
344 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
345 if isinstance(stdin
,basestring
):
346 stdin_file
.write(stdin
)
349 stdin_file
.write(i
+ '\n')
353 p4
= subprocess
.Popen(cmd
,
356 stdout
=subprocess
.PIPE
)
361 entry
= marshal
.load(p4
.stdout
)
371 entry
["p4ExitCode"] = exitCode
377 list = p4CmdList(cmd
)
383 def p4Where(depotPath
):
384 if not depotPath
.endswith("/"):
386 depotPath
= depotPath
+ "..."
387 outputList
= p4CmdList(["where", depotPath
])
389 for entry
in outputList
:
390 if "depotFile" in entry
:
391 if entry
["depotFile"] == depotPath
:
394 elif "data" in entry
:
395 data
= entry
.get("data")
396 space
= data
.find(" ")
397 if data
[:space
] == depotPath
:
402 if output
["code"] == "error":
406 clientPath
= output
.get("path")
407 elif "data" in output
:
408 data
= output
.get("data")
409 lastSpace
= data
.rfind(" ")
410 clientPath
= data
[lastSpace
+ 1:]
412 if clientPath
.endswith("..."):
413 clientPath
= clientPath
[:-3]
416 def currentGitBranch():
417 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
419 def isValidGitDir(path
):
420 if (os
.path
.exists(path
+ "/HEAD")
421 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
425 def parseRevision(ref
):
426 return read_pipe("git rev-parse %s" % ref
).strip()
428 def branchExists(ref
):
429 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
433 def extractLogMessageFromGitCommit(commit
):
436 ## fixme: title is first line of commit, not 1st paragraph.
438 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
447 def extractSettingsGitLog(log
):
449 for line
in log
.split("\n"):
451 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
455 assignments
= m
.group(1).split (':')
456 for a
in assignments
:
458 key
= vals
[0].strip()
459 val
= ('='.join (vals
[1:])).strip()
460 if val
.endswith ('\"') and val
.startswith('"'):
465 paths
= values
.get("depot-paths")
467 paths
= values
.get("depot-path")
469 values
['depot-paths'] = paths
.split(',')
472 def gitBranchExists(branch
):
473 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
474 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
475 return proc
.wait() == 0;
478 def gitConfig(key
, args
= None): # set args to "--bool", for instance
479 if not _gitConfig
.has_key(key
):
482 argsFilter
= "%s " % args
483 cmd
= "git config %s%s" % (argsFilter
, key
)
484 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
485 return _gitConfig
[key
]
487 def gitConfigList(key
):
488 if not _gitConfig
.has_key(key
):
489 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
490 return _gitConfig
[key
]
492 def p4BranchesInGit(branchesAreInRemotes
= True):
495 cmdline
= "git rev-parse --symbolic "
496 if branchesAreInRemotes
:
497 cmdline
+= " --remotes"
499 cmdline
+= " --branches"
501 for line
in read_pipe_lines(cmdline
):
504 ## only import to p4/
505 if not line
.startswith('p4/') or line
== "p4/HEAD":
510 branch
= re
.sub ("^p4/", "", line
)
512 branches
[branch
] = parseRevision(line
)
515 def findUpstreamBranchPoint(head
= "HEAD"):
516 branches
= p4BranchesInGit()
517 # map from depot-path to branch name
518 branchByDepotPath
= {}
519 for branch
in branches
.keys():
520 tip
= branches
[branch
]
521 log
= extractLogMessageFromGitCommit(tip
)
522 settings
= extractSettingsGitLog(log
)
523 if settings
.has_key("depot-paths"):
524 paths
= ",".join(settings
["depot-paths"])
525 branchByDepotPath
[paths
] = "remotes/p4/" + branch
529 while parent
< 65535:
530 commit
= head
+ "~%s" % parent
531 log
= extractLogMessageFromGitCommit(commit
)
532 settings
= extractSettingsGitLog(log
)
533 if settings
.has_key("depot-paths"):
534 paths
= ",".join(settings
["depot-paths"])
535 if branchByDepotPath
.has_key(paths
):
536 return [branchByDepotPath
[paths
], settings
]
540 return ["", settings
]
542 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
544 print ("Creating/updating branch(es) in %s based on origin branch(es)"
547 originPrefix
= "origin/p4/"
549 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
551 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
554 headName
= line
[len(originPrefix
):]
555 remoteHead
= localRefPrefix
+ headName
558 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
559 if (not original
.has_key('depot-paths')
560 or not original
.has_key('change')):
564 if not gitBranchExists(remoteHead
):
566 print "creating %s" % remoteHead
569 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
570 if settings
.has_key('change') > 0:
571 if settings
['depot-paths'] == original
['depot-paths']:
572 originP4Change
= int(original
['change'])
573 p4Change
= int(settings
['change'])
574 if originP4Change
> p4Change
:
575 print ("%s (%s) is newer than %s (%s). "
576 "Updating p4 branch from origin."
577 % (originHead
, originP4Change
,
578 remoteHead
, p4Change
))
581 print ("Ignoring: %s was imported from %s while "
582 "%s was imported from %s"
583 % (originHead
, ','.join(original
['depot-paths']),
584 remoteHead
, ','.join(settings
['depot-paths'])))
587 system("git update-ref %s %s" % (remoteHead
, originHead
))
589 def originP4BranchesExist():
590 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
592 def p4ChangesForPaths(depotPaths
, changeRange
):
596 cmd
+= ["%s...%s" % (p
, changeRange
)]
597 output
= p4_read_pipe_lines(cmd
)
601 changeNum
= int(line
.split(" ")[1])
602 changes
[changeNum
] = True
604 changelist
= changes
.keys()
608 def p4PathStartsWith(path
, prefix
):
609 # This method tries to remedy a potential mixed-case issue:
611 # If UserA adds //depot/DirA/file1
612 # and UserB adds //depot/dira/file2
614 # we may or may not have a problem. If you have core.ignorecase=true,
615 # we treat DirA and dira as the same directory
616 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
618 return path
.lower().startswith(prefix
.lower())
619 return path
.startswith(prefix
)
622 """Look at the p4 client spec, create a View() object that contains
623 all the mappings, and return it."""
625 specList
= p4CmdList("client -o")
626 if len(specList
) != 1:
627 die('Output from "client -o" is %d lines, expecting 1' %
630 # dictionary of all client parameters
633 # just the keys that start with "View"
634 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
639 # append the lines, in order, to the view
640 for view_num
in range(len(view_keys
)):
641 k
= "View%d" % view_num
642 if k
not in view_keys
:
643 die("Expected view key %s missing" % k
)
644 view
.append(entry
[k
])
649 """Grab the client directory."""
651 output
= p4CmdList("client -o")
653 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
656 if "Root" not in entry
:
657 die('Client has no "Root"')
663 self
.usage
= "usage: %prog [options]"
668 self
.userMapFromPerforceServer
= False
669 self
.myP4UserId
= None
673 return self
.myP4UserId
675 results
= p4CmdList("user -o")
677 if r
.has_key('User'):
678 self
.myP4UserId
= r
['User']
680 die("Could not find your p4 user id")
682 def p4UserIsMe(self
, p4User
):
683 # return True if the given p4 user is actually me
685 if not p4User
or p4User
!= me
:
690 def getUserCacheFilename(self
):
691 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
692 return home
+ "/.gitp4-usercache.txt"
694 def getUserMapFromPerforceServer(self
):
695 if self
.userMapFromPerforceServer
:
700 for output
in p4CmdList("users"):
701 if not output
.has_key("User"):
703 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
704 self
.emails
[output
["Email"]] = output
["User"]
708 for (key
, val
) in self
.users
.items():
709 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
711 open(self
.getUserCacheFilename(), "wb").write(s
)
712 self
.userMapFromPerforceServer
= True
714 def loadUserMapFromCache(self
):
716 self
.userMapFromPerforceServer
= False
718 cache
= open(self
.getUserCacheFilename(), "rb")
719 lines
= cache
.readlines()
722 entry
= line
.strip().split("\t")
723 self
.users
[entry
[0]] = entry
[1]
725 self
.getUserMapFromPerforceServer()
727 class P4Debug(Command
):
729 Command
.__init
__(self
)
731 optparse
.make_option("--verbose", dest
="verbose", action
="store_true",
734 self
.description
= "A tool to debug the output of p4 -G."
735 self
.needsGit
= False
740 for output
in p4CmdList(args
):
741 print 'Element: %d' % j
746 class P4RollBack(Command
):
748 Command
.__init
__(self
)
750 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
751 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
753 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
755 self
.rollbackLocalBranches
= False
760 maxChange
= int(args
[0])
762 if "p4ExitCode" in p4Cmd("changes -m 1"):
763 die("Problems executing p4");
765 if self
.rollbackLocalBranches
:
766 refPrefix
= "refs/heads/"
767 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
769 refPrefix
= "refs/remotes/"
770 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
773 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
775 ref
= refPrefix
+ line
776 log
= extractLogMessageFromGitCommit(ref
)
777 settings
= extractSettingsGitLog(log
)
779 depotPaths
= settings
['depot-paths']
780 change
= settings
['change']
784 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
785 for p
in depotPaths
]))) == 0:
786 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
787 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
790 while change
and int(change
) > maxChange
:
793 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
794 system("git update-ref %s \"%s^\"" % (ref
, ref
))
795 log
= extractLogMessageFromGitCommit(ref
)
796 settings
= extractSettingsGitLog(log
)
799 depotPaths
= settings
['depot-paths']
800 change
= settings
['change']
803 print "%s rewound to %s" % (ref
, change
)
807 class P4Submit(Command
, P4UserMap
):
809 Command
.__init
__(self
)
810 P4UserMap
.__init
__(self
)
812 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
813 optparse
.make_option("--origin", dest
="origin"),
814 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
815 # preserve the user, requires relevant p4 permissions
816 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
817 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
819 self
.description
= "Submit changes from git to the perforce depot."
820 self
.usage
+= " [name of git branch to submit into perforce depot]"
821 self
.interactive
= True
823 self
.detectRenames
= False
825 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
826 self
.isWindows
= (platform
.system() == "Windows")
827 self
.exportLabels
= False
830 if len(p4CmdList("opened ...")) > 0:
831 die("You have files opened with perforce! Close them before starting the sync.")
833 # replaces everything between 'Description:' and the next P4 submit template field with the
835 def prepareLogMessage(self
, template
, message
):
838 inDescriptionSection
= False
840 for line
in template
.split("\n"):
841 if line
.startswith("#"):
842 result
+= line
+ "\n"
845 if inDescriptionSection
:
846 if line
.startswith("Files:") or line
.startswith("Jobs:"):
847 inDescriptionSection
= False
851 if line
.startswith("Description:"):
852 inDescriptionSection
= True
854 for messageLine
in message
.split("\n"):
855 line
+= "\t" + messageLine
+ "\n"
857 result
+= line
+ "\n"
861 def patchRCSKeywords(self
, file, pattern
):
862 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
863 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
865 outFile
= os
.fdopen(handle
, "w+")
866 inFile
= open(file, "r")
867 regexp
= re
.compile(pattern
, re
.VERBOSE
)
868 for line
in inFile
.readlines():
869 line
= regexp
.sub(r
'$\1$', line
)
873 # Forcibly overwrite the original file
875 shutil
.move(outFileName
, file)
877 # cleanup our temporary file
878 os
.unlink(outFileName
)
879 print "Failed to strip RCS keywords in %s" % file
882 print "Patched up RCS keywords in %s" % file
884 def p4UserForCommit(self
,id):
885 # Return the tuple (perforce user,git email) for a given git commit id
886 self
.getUserMapFromPerforceServer()
887 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
888 gitEmail
= gitEmail
.strip()
889 if not self
.emails
.has_key(gitEmail
):
890 return (None,gitEmail
)
892 return (self
.emails
[gitEmail
],gitEmail
)
894 def checkValidP4Users(self
,commits
):
895 # check if any git authors cannot be mapped to p4 users
897 (user
,email
) = self
.p4UserForCommit(id)
899 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
900 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
903 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
905 def lastP4Changelist(self
):
906 # Get back the last changelist number submitted in this client spec. This
907 # then gets used to patch up the username in the change. If the same
908 # client spec is being used by multiple processes then this might go
910 results
= p4CmdList("client -o") # find the current client
913 if r
.has_key('Client'):
917 die("could not get client spec")
918 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
920 if r
.has_key('change'):
922 die("Could not get changelist number for last submit - cannot patch up user details")
924 def modifyChangelistUser(self
, changelist
, newUser
):
925 # fixup the user field of a changelist after it has been submitted.
926 changes
= p4CmdList("change -o %s" % changelist
)
927 if len(changes
) != 1:
928 die("Bad output from p4 change modifying %s to user %s" %
929 (changelist
, newUser
))
932 if c
['User'] == newUser
: return # nothing to do
934 input = marshal
.dumps(c
)
936 result
= p4CmdList("change -f -i", stdin
=input)
938 if r
.has_key('code'):
939 if r
['code'] == 'error':
940 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
941 if r
.has_key('data'):
942 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
944 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
946 def canChangeChangelists(self
):
947 # check to see if we have p4 admin or super-user permissions, either of
948 # which are required to modify changelists.
949 results
= p4CmdList(["protects", self
.depotPath
])
951 if r
.has_key('perm'):
952 if r
['perm'] == 'admin':
954 if r
['perm'] == 'super':
958 def prepareSubmitTemplate(self
):
959 # remove lines in the Files section that show changes to files outside the depot path we're committing into
961 inFilesSection
= False
962 for line
in p4_read_pipe_lines(['change', '-o']):
963 if line
.endswith("\r\n"):
964 line
= line
[:-2] + "\n"
966 if line
.startswith("\t"):
967 # path starts and ends with a tab
969 lastTab
= path
.rfind("\t")
971 path
= path
[:lastTab
]
972 if not p4PathStartsWith(path
, self
.depotPath
):
975 inFilesSection
= False
977 if line
.startswith("Files:"):
978 inFilesSection
= True
984 def edit_template(self
, template_file
):
985 """Invoke the editor to let the user change the submission
986 message. Return true if okay to continue with the submit."""
988 # if configured to skip the editing part, just submit
989 if gitConfig("git-p4.skipSubmitEdit") == "true":
992 # look at the modification time, to check later if the user saved
994 mtime
= os
.stat(template_file
).st_mtime
997 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
998 editor
= os
.environ
.get("P4EDITOR")
1000 editor
= read_pipe("git var GIT_EDITOR").strip()
1001 system(editor
+ " " + template_file
)
1003 # If the file was not saved, prompt to see if this patch should
1004 # be skipped. But skip this verification step if configured so.
1005 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1008 # modification time updated means user saved the file
1009 if os
.stat(template_file
).st_mtime
> mtime
:
1013 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1019 def applyCommit(self
, id):
1020 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1022 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1024 if not self
.detectRenames
:
1025 # If not explicitly set check the config variable
1026 self
.detectRenames
= gitConfig("git-p4.detectRenames")
1028 if self
.detectRenames
.lower() == "false" or self
.detectRenames
== "":
1030 elif self
.detectRenames
.lower() == "true":
1033 diffOpts
= "-M%s" % self
.detectRenames
1035 detectCopies
= gitConfig("git-p4.detectCopies")
1036 if detectCopies
.lower() == "true":
1038 elif detectCopies
!= "" and detectCopies
.lower() != "false":
1039 diffOpts
+= " -C%s" % detectCopies
1041 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1042 diffOpts
+= " --find-copies-harder"
1044 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts
, id, id))
1046 filesToDelete
= set()
1048 filesToChangeExecBit
= {}
1051 diff
= parseDiffTreeEntry(line
)
1052 modifier
= diff
['status']
1056 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1057 filesToChangeExecBit
[path
] = diff
['dst_mode']
1058 editedFiles
.add(path
)
1059 elif modifier
== "A":
1060 filesToAdd
.add(path
)
1061 filesToChangeExecBit
[path
] = diff
['dst_mode']
1062 if path
in filesToDelete
:
1063 filesToDelete
.remove(path
)
1064 elif modifier
== "D":
1065 filesToDelete
.add(path
)
1066 if path
in filesToAdd
:
1067 filesToAdd
.remove(path
)
1068 elif modifier
== "C":
1069 src
, dest
= diff
['src'], diff
['dst']
1070 p4_integrate(src
, dest
)
1071 if diff
['src_sha1'] != diff
['dst_sha1']:
1073 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1075 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1077 editedFiles
.add(dest
)
1078 elif modifier
== "R":
1079 src
, dest
= diff
['src'], diff
['dst']
1080 p4_integrate(src
, dest
)
1081 if diff
['src_sha1'] != diff
['dst_sha1']:
1083 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1085 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1087 editedFiles
.add(dest
)
1088 filesToDelete
.add(src
)
1090 die("unknown modifier %s for %s" % (modifier
, path
))
1092 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1093 patchcmd
= diffcmd
+ " | git apply "
1094 tryPatchCmd
= patchcmd
+ "--check -"
1095 applyPatchCmd
= patchcmd
+ "--check --apply -"
1096 patch_succeeded
= True
1098 if os
.system(tryPatchCmd
) != 0:
1099 fixed_rcs_keywords
= False
1100 patch_succeeded
= False
1101 print "Unfortunately applying the change failed!"
1103 # Patch failed, maybe it's just RCS keyword woes. Look through
1104 # the patch to see if that's possible.
1105 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1109 for file in editedFiles | filesToDelete
:
1110 # did this file's delta contain RCS keywords?
1111 pattern
= p4_keywords_regexp_for_file(file)
1114 # this file is a possibility...look for RCS keywords.
1115 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1116 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1117 if regexp
.search(line
):
1119 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1120 kwfiles
[file] = pattern
1123 for file in kwfiles
:
1125 print "zapping %s with %s" % (line
,pattern
)
1126 self
.patchRCSKeywords(file, kwfiles
[file])
1127 fixed_rcs_keywords
= True
1129 if fixed_rcs_keywords
:
1130 print "Retrying the patch with RCS keywords cleaned up"
1131 if os
.system(tryPatchCmd
) == 0:
1132 patch_succeeded
= True
1134 if not patch_succeeded
:
1135 print "What do you want to do?"
1137 while response
!= "s" and response
!= "a" and response
!= "w":
1138 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1139 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1141 print "Skipping! Good luck with the next patches..."
1142 for f
in editedFiles
:
1144 for f
in filesToAdd
:
1147 elif response
== "a":
1148 os
.system(applyPatchCmd
)
1149 if len(filesToAdd
) > 0:
1150 print "You may also want to call p4 add on the following files:"
1151 print " ".join(filesToAdd
)
1152 if len(filesToDelete
):
1153 print "The following files should be scheduled for deletion with p4 delete:"
1154 print " ".join(filesToDelete
)
1155 die("Please resolve and submit the conflict manually and "
1156 + "continue afterwards with git p4 submit --continue")
1157 elif response
== "w":
1158 system(diffcmd
+ " > patch.txt")
1159 print "Patch saved to patch.txt in %s !" % self
.clientPath
1160 die("Please resolve and submit the conflict manually and "
1161 "continue afterwards with git p4 submit --continue")
1163 system(applyPatchCmd
)
1165 for f
in filesToAdd
:
1167 for f
in filesToDelete
:
1171 # Set/clear executable bits
1172 for f
in filesToChangeExecBit
.keys():
1173 mode
= filesToChangeExecBit
[f
]
1174 setP4ExecBit(f
, mode
)
1176 logMessage
= extractLogMessageFromGitCommit(id)
1177 logMessage
= logMessage
.strip()
1179 template
= self
.prepareSubmitTemplate()
1181 if self
.interactive
:
1182 submitTemplate
= self
.prepareLogMessage(template
, logMessage
)
1184 if self
.preserveUser
:
1185 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1187 if os
.environ
.has_key("P4DIFF"):
1188 del(os
.environ
["P4DIFF"])
1190 for editedFile
in editedFiles
:
1191 diff
+= p4_read_pipe(['diff', '-du', editedFile
])
1194 for newFile
in filesToAdd
:
1195 newdiff
+= "==== new file ====\n"
1196 newdiff
+= "--- /dev/null\n"
1197 newdiff
+= "+++ %s\n" % newFile
1198 f
= open(newFile
, "r")
1199 for line
in f
.readlines():
1200 newdiff
+= "+" + line
1203 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1204 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1205 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1206 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1208 separatorLine
= "######## everything below this line is just the diff #######\n"
1210 (handle
, fileName
) = tempfile
.mkstemp()
1211 tmpFile
= os
.fdopen(handle
, "w+")
1213 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1214 separatorLine
= separatorLine
.replace("\n", "\r\n")
1215 newdiff
= newdiff
.replace("\n", "\r\n")
1216 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1219 if self
.edit_template(fileName
):
1220 # read the edited message and submit
1221 tmpFile
= open(fileName
, "rb")
1222 message
= tmpFile
.read()
1224 submitTemplate
= message
[:message
.index(separatorLine
)]
1226 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1227 p4_write_pipe(['submit', '-i'], submitTemplate
)
1229 if self
.preserveUser
:
1231 # Get last changelist number. Cannot easily get it from
1232 # the submit command output as the output is
1234 changelist
= self
.lastP4Changelist()
1235 self
.modifyChangelistUser(changelist
, p4User
)
1238 print "Submission cancelled, undoing p4 changes."
1239 for f
in editedFiles
:
1241 for f
in filesToAdd
:
1247 fileName
= "submit.txt"
1248 file = open(fileName
, "w+")
1249 file.write(self
.prepareLogMessage(template
, logMessage
))
1251 print ("Perforce submit template written as %s. "
1252 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1253 % (fileName
, fileName
))
1255 # Export git tags as p4 labels. Create a p4 label and then tag
1257 def exportGitTags(self
, gitTags
):
1258 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1259 if len(validLabelRegexp
) == 0:
1260 validLabelRegexp
= defaultLabelRegexp
1261 m
= re
.compile(validLabelRegexp
)
1263 for name
in gitTags
:
1265 if not m
.match(name
):
1267 print "tag %s does not match regexp %s" % (name
, validTagRegexp
)
1270 # Get the p4 commit this corresponds to
1271 logMessage
= extractLogMessageFromGitCommit(name
)
1272 values
= extractSettingsGitLog(logMessage
)
1274 if not values
.has_key('change'):
1275 # a tag pointing to something not sent to p4; ignore
1277 print "git tag %s does not give a p4 commit" % name
1280 changelist
= values
['change']
1282 # Get the tag details.
1286 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1289 if re
.match(r
'tag\s+', l
):
1291 elif re
.match(r
'\s*$', l
):
1298 body
= ["lightweight tag imported by git p4\n"]
1300 # Create the label - use the same view as the client spec we are using
1301 clientSpec
= getClientSpec()
1303 labelTemplate
= "Label: %s\n" % name
1304 labelTemplate
+= "Description:\n"
1306 labelTemplate
+= "\t" + b
+ "\n"
1307 labelTemplate
+= "View:\n"
1308 for mapping
in clientSpec
.mappings
:
1309 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1311 p4_write_pipe(["label", "-i"], labelTemplate
)
1314 p4_system(["tag", "-l", name
] +
1315 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1318 print "created p4 label for tag %s" % name
1320 def run(self
, args
):
1322 self
.master
= currentGitBranch()
1323 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1324 die("Detecting current git branch failed!")
1325 elif len(args
) == 1:
1326 self
.master
= args
[0]
1327 if not branchExists(self
.master
):
1328 die("Branch %s does not exist" % self
.master
)
1332 allowSubmit
= gitConfig("git-p4.allowSubmit")
1333 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1334 die("%s is not in git-p4.allowSubmit" % self
.master
)
1336 [upstream
, settings
] = findUpstreamBranchPoint()
1337 self
.depotPath
= settings
['depot-paths'][0]
1338 if len(self
.origin
) == 0:
1339 self
.origin
= upstream
1341 if self
.preserveUser
:
1342 if not self
.canChangeChangelists():
1343 die("Cannot preserve user names without p4 super-user or admin permissions")
1346 print "Origin branch is " + self
.origin
1348 if len(self
.depotPath
) == 0:
1349 print "Internal error: cannot locate perforce depot path from existing branches"
1352 self
.useClientSpec
= False
1353 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1354 self
.useClientSpec
= True
1355 if self
.useClientSpec
:
1356 self
.clientSpecDirs
= getClientSpec()
1358 if self
.useClientSpec
:
1359 # all files are relative to the client spec
1360 self
.clientPath
= getClientRoot()
1362 self
.clientPath
= p4Where(self
.depotPath
)
1364 if self
.clientPath
== "":
1365 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1367 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1368 self
.oldWorkingDirectory
= os
.getcwd()
1370 # ensure the clientPath exists
1371 if not os
.path
.exists(self
.clientPath
):
1372 os
.makedirs(self
.clientPath
)
1374 chdir(self
.clientPath
)
1375 print "Synchronizing p4 checkout..."
1380 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1381 commits
.append(line
.strip())
1384 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1385 self
.checkAuthorship
= False
1387 self
.checkAuthorship
= True
1389 if self
.preserveUser
:
1390 self
.checkValidP4Users(commits
)
1392 while len(commits
) > 0:
1394 commits
= commits
[1:]
1395 self
.applyCommit(commit
)
1396 if not self
.interactive
:
1399 if len(commits
) == 0:
1400 print "All changes applied!"
1401 chdir(self
.oldWorkingDirectory
)
1409 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1410 self
.exportLabels
= true
1412 if self
.exportLabels
:
1413 p4Labels
= getP4Labels(self
.depotPath
)
1414 gitTags
= getGitTags()
1416 missingGitTags
= gitTags
- p4Labels
1417 self
.exportGitTags(missingGitTags
)
1422 """Represent a p4 view ("p4 help views"), and map files in a
1423 repo according to the view."""
1426 """A depot or client path, possibly containing wildcards.
1427 The only one supported is ... at the end, currently.
1428 Initialize with the full path, with //depot or //client."""
1430 def __init__(self
, path
, is_depot
):
1432 self
.is_depot
= is_depot
1433 self
.find_wildcards()
1434 # remember the prefix bit, useful for relative mappings
1435 m
= re
.match("(//[^/]+/)", self
.path
)
1437 die("Path %s does not start with //prefix/" % self
.path
)
1439 if not self
.is_depot
:
1440 # strip //client/ on client paths
1441 self
.path
= self
.path
[len(prefix
):]
1443 def find_wildcards(self
):
1444 """Make sure wildcards are valid, and set up internal
1447 self
.ends_triple_dot
= False
1448 # There are three wildcards allowed in p4 views
1449 # (see "p4 help views"). This code knows how to
1450 # handle "..." (only at the end), but cannot deal with
1451 # "%%n" or "*". Only check the depot_side, as p4 should
1452 # validate that the client_side matches too.
1453 if re
.search(r
'%%[1-9]', self
.path
):
1454 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1455 if self
.path
.find("*") >= 0:
1456 die("Can't handle * wildcards in view: %s" % self
.path
)
1457 triple_dot_index
= self
.path
.find("...")
1458 if triple_dot_index
>= 0:
1459 if triple_dot_index
!= len(self
.path
) - 3:
1460 die("Can handle only single ... wildcard, at end: %s" %
1462 self
.ends_triple_dot
= True
1464 def ensure_compatible(self
, other_path
):
1465 """Make sure the wildcards agree."""
1466 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1467 die("Both paths must end with ... if either does;\n" +
1468 "paths: %s %s" % (self
.path
, other_path
.path
))
1470 def match_wildcards(self
, test_path
):
1471 """See if this test_path matches us, and fill in the value
1472 of the wildcards if so. Returns a tuple of
1473 (True|False, wildcards[]). For now, only the ... at end
1474 is supported, so at most one wildcard."""
1475 if self
.ends_triple_dot
:
1476 dotless
= self
.path
[:-3]
1477 if test_path
.startswith(dotless
):
1478 wildcard
= test_path
[len(dotless
):]
1479 return (True, [ wildcard
])
1481 if test_path
== self
.path
:
1485 def match(self
, test_path
):
1486 """Just return if it matches; don't bother with the wildcards."""
1487 b
, _
= self
.match_wildcards(test_path
)
1490 def fill_in_wildcards(self
, wildcards
):
1491 """Return the relative path, with the wildcards filled in
1492 if there are any."""
1493 if self
.ends_triple_dot
:
1494 return self
.path
[:-3] + wildcards
[0]
1498 class Mapping(object):
1499 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1500 # depot_side is without the trailing /... if it had one
1501 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1502 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1503 self
.overlay
= overlay
# started with "+"
1504 self
.exclude
= exclude
# started with "-"
1505 assert not (self
.overlay
and self
.exclude
)
1506 self
.depot_side
.ensure_compatible(self
.client_side
)
1514 return "View.Mapping: %s%s -> %s" % \
1515 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1517 def map_depot_to_client(self
, depot_path
):
1518 """Calculate the client path if using this mapping on the
1519 given depot path; does not consider the effect of other
1520 mappings in a view. Even excluded mappings are returned."""
1521 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1524 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1533 def append(self
, view_line
):
1534 """Parse a view line, splitting it into depot and client
1535 sides. Append to self.mappings, preserving order."""
1537 # Split the view line into exactly two words. P4 enforces
1538 # structure on these lines that simplifies this quite a bit.
1540 # Either or both words may be double-quoted.
1541 # Single quotes do not matter.
1542 # Double-quote marks cannot occur inside the words.
1543 # A + or - prefix is also inside the quotes.
1544 # There are no quotes unless they contain a space.
1545 # The line is already white-space stripped.
1546 # The two words are separated by a single space.
1548 if view_line
[0] == '"':
1549 # First word is double quoted. Find its end.
1550 close_quote_index
= view_line
.find('"', 1)
1551 if close_quote_index
<= 0:
1552 die("No first-word closing quote found: %s" % view_line
)
1553 depot_side
= view_line
[1:close_quote_index
]
1554 # skip closing quote and space
1555 rhs_index
= close_quote_index
+ 1 + 1
1557 space_index
= view_line
.find(" ")
1558 if space_index
<= 0:
1559 die("No word-splitting space found: %s" % view_line
)
1560 depot_side
= view_line
[0:space_index
]
1561 rhs_index
= space_index
+ 1
1563 if view_line
[rhs_index
] == '"':
1564 # Second word is double quoted. Make sure there is a
1565 # double quote at the end too.
1566 if not view_line
.endswith('"'):
1567 die("View line with rhs quote should end with one: %s" %
1570 client_side
= view_line
[rhs_index
+1:-1]
1572 client_side
= view_line
[rhs_index
:]
1574 # prefix + means overlay on previous mapping
1576 if depot_side
.startswith("+"):
1578 depot_side
= depot_side
[1:]
1580 # prefix - means exclude this path
1582 if depot_side
.startswith("-"):
1584 depot_side
= depot_side
[1:]
1586 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1587 self
.mappings
.append(m
)
1589 def map_in_client(self
, depot_path
):
1590 """Return the relative location in the client where this
1591 depot file should live. Returns "" if the file should
1592 not be mapped in the client."""
1597 # look at later entries first
1598 for m
in self
.mappings
[::-1]:
1600 # see where will this path end up in the client
1601 p
= m
.map_depot_to_client(depot_path
)
1604 # Depot path does not belong in client. Must remember
1605 # this, as previous items should not cause files to
1606 # exist in this path either. Remember that the list is
1607 # being walked from the end, which has higher precedence.
1608 # Overlap mappings do not exclude previous mappings.
1610 paths_filled
.append(m
.client_side
)
1613 # This mapping matched; no need to search any further.
1614 # But, the mapping could be rejected if the client path
1615 # has already been claimed by an earlier mapping (i.e.
1616 # one later in the list, which we are walking backwards).
1617 already_mapped_in_client
= False
1618 for f
in paths_filled
:
1619 # this is View.Path.match
1621 already_mapped_in_client
= True
1623 if not already_mapped_in_client
:
1624 # Include this file, unless it is from a line that
1625 # explicitly said to exclude it.
1629 # a match, even if rejected, always stops the search
1634 class P4Sync(Command
, P4UserMap
):
1635 delete_actions
= ( "delete", "move/delete", "purge" )
1638 Command
.__init
__(self
)
1639 P4UserMap
.__init
__(self
)
1641 optparse
.make_option("--branch", dest
="branch"),
1642 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1643 optparse
.make_option("--changesfile", dest
="changesFile"),
1644 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1645 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1646 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1647 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
1648 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1649 help="Import into refs/heads/ , not refs/remotes"),
1650 optparse
.make_option("--max-changes", dest
="maxChanges"),
1651 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1652 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1653 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1654 help="Only sync files that are included in the Perforce Client Spec")
1656 self
.description
= """Imports from Perforce into a git repository.\n
1658 //depot/my/project/ -- to import the current head
1659 //depot/my/project/@all -- to import everything
1660 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1662 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1664 self
.usage
+= " //depot/path[@revRange]"
1666 self
.createdBranches
= set()
1667 self
.committedChanges
= set()
1669 self
.detectBranches
= False
1670 self
.detectLabels
= False
1671 self
.importLabels
= False
1672 self
.changesFile
= ""
1673 self
.syncWithOrigin
= True
1674 self
.verbose
= False
1675 self
.importIntoRemotes
= True
1676 self
.maxChanges
= ""
1677 self
.isWindows
= (platform
.system() == "Windows")
1678 self
.keepRepoPath
= False
1679 self
.depotPaths
= None
1680 self
.p4BranchesInGit
= []
1681 self
.cloneExclude
= []
1682 self
.useClientSpec
= False
1683 self
.useClientSpec_from_options
= False
1684 self
.clientSpecDirs
= None
1685 self
.tempBranches
= []
1686 self
.tempBranchLocation
= "git-p4-tmp"
1688 if gitConfig("git-p4.syncFromOrigin") == "false":
1689 self
.syncWithOrigin
= False
1692 # P4 wildcards are not allowed in filenames. P4 complains
1693 # if you simply add them, but you can force it with "-f", in
1694 # which case it translates them into %xx encoding internally.
1695 # Search for and fix just these four characters. Do % last so
1696 # that fixing it does not inadvertently create new %-escapes.
1698 def wildcard_decode(self
, path
):
1699 # Cannot have * in a filename in windows; untested as to
1700 # what p4 would do in such a case.
1701 if not self
.isWindows
:
1702 path
= path
.replace("%2A", "*")
1703 path
= path
.replace("%23", "#") \
1704 .replace("%40", "@") \
1705 .replace("%25", "%")
1708 # Force a checkpoint in fast-import and wait for it to finish
1709 def checkpoint(self
):
1710 self
.gitStream
.write("checkpoint\n\n")
1711 self
.gitStream
.write("progress checkpoint\n\n")
1712 out
= self
.gitOutput
.readline()
1714 print "checkpoint finished: " + out
1716 def extractFilesFromCommit(self
, commit
):
1717 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1718 for path
in self
.cloneExclude
]
1721 while commit
.has_key("depotFile%s" % fnum
):
1722 path
= commit
["depotFile%s" % fnum
]
1724 if [p
for p
in self
.cloneExclude
1725 if p4PathStartsWith(path
, p
)]:
1728 found
= [p
for p
in self
.depotPaths
1729 if p4PathStartsWith(path
, p
)]
1736 file["rev"] = commit
["rev%s" % fnum
]
1737 file["action"] = commit
["action%s" % fnum
]
1738 file["type"] = commit
["type%s" % fnum
]
1743 def stripRepoPath(self
, path
, prefixes
):
1744 if self
.useClientSpec
:
1745 return self
.clientSpecDirs
.map_in_client(path
)
1747 if self
.keepRepoPath
:
1748 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1751 if p4PathStartsWith(path
, p
):
1752 path
= path
[len(p
):]
1756 def splitFilesIntoBranches(self
, commit
):
1759 while commit
.has_key("depotFile%s" % fnum
):
1760 path
= commit
["depotFile%s" % fnum
]
1761 found
= [p
for p
in self
.depotPaths
1762 if p4PathStartsWith(path
, p
)]
1769 file["rev"] = commit
["rev%s" % fnum
]
1770 file["action"] = commit
["action%s" % fnum
]
1771 file["type"] = commit
["type%s" % fnum
]
1774 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1776 for branch
in self
.knownBranches
.keys():
1778 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1779 if relPath
.startswith(branch
+ "/"):
1780 if branch
not in branches
:
1781 branches
[branch
] = []
1782 branches
[branch
].append(file)
1787 # output one file from the P4 stream
1788 # - helper for streamP4Files
1790 def streamOneP4File(self
, file, contents
):
1791 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1792 relPath
= self
.wildcard_decode(relPath
)
1794 sys
.stderr
.write("%s\n" % relPath
)
1796 (type_base
, type_mods
) = split_p4_type(file["type"])
1799 if "x" in type_mods
:
1801 if type_base
== "symlink":
1803 # p4 print on a symlink contains "target\n"; remove the newline
1804 data
= ''.join(contents
)
1805 contents
= [data
[:-1]]
1807 if type_base
== "utf16":
1808 # p4 delivers different text in the python output to -G
1809 # than it does when using "print -o", or normal p4 client
1810 # operations. utf16 is converted to ascii or utf8, perhaps.
1811 # But ascii text saved as -t utf16 is completely mangled.
1812 # Invoke print -o to get the real contents.
1813 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1816 if type_base
== "apple":
1817 # Apple filetype files will be streamed as a concatenation of
1818 # its appledouble header and the contents. This is useless
1819 # on both macs and non-macs. If using "print -q -o xx", it
1820 # will create "xx" with the data, and "%xx" with the header.
1821 # This is also not very useful.
1823 # Ideally, someday, this script can learn how to generate
1824 # appledouble files directly and import those to git, but
1825 # non-mac machines can never find a use for apple filetype.
1826 print "\nIgnoring apple filetype file %s" % file['depotFile']
1829 # Perhaps windows wants unicode, utf16 newlines translated too;
1830 # but this is not doing it.
1831 if self
.isWindows
and type_base
== "text":
1833 for data
in contents
:
1834 data
= data
.replace("\r\n", "\n")
1835 mangled
.append(data
)
1838 # Note that we do not try to de-mangle keywords on utf16 files,
1839 # even though in theory somebody may want that.
1840 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1842 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1843 text
= ''.join(contents
)
1844 text
= regexp
.sub(r
'$\1$', text
)
1847 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1852 length
= length
+ len(d
)
1854 self
.gitStream
.write("data %d\n" % length
)
1856 self
.gitStream
.write(d
)
1857 self
.gitStream
.write("\n")
1859 def streamOneP4Deletion(self
, file):
1860 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1862 sys
.stderr
.write("delete %s\n" % relPath
)
1863 self
.gitStream
.write("D %s\n" % relPath
)
1865 # handle another chunk of streaming data
1866 def streamP4FilesCb(self
, marshalled
):
1868 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1869 # start of a new file - output the old one first
1870 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1871 self
.stream_file
= {}
1872 self
.stream_contents
= []
1873 self
.stream_have_file_info
= False
1875 # pick up the new file information... for the
1876 # 'data' field we need to append to our array
1877 for k
in marshalled
.keys():
1879 self
.stream_contents
.append(marshalled
['data'])
1881 self
.stream_file
[k
] = marshalled
[k
]
1883 self
.stream_have_file_info
= True
1885 # Stream directly from "p4 files" into "git fast-import"
1886 def streamP4Files(self
, files
):
1892 # if using a client spec, only add the files that have
1893 # a path in the client
1894 if self
.clientSpecDirs
:
1895 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
1898 filesForCommit
.append(f
)
1899 if f
['action'] in self
.delete_actions
:
1900 filesToDelete
.append(f
)
1902 filesToRead
.append(f
)
1905 for f
in filesToDelete
:
1906 self
.streamOneP4Deletion(f
)
1908 if len(filesToRead
) > 0:
1909 self
.stream_file
= {}
1910 self
.stream_contents
= []
1911 self
.stream_have_file_info
= False
1913 # curry self argument
1914 def streamP4FilesCbSelf(entry
):
1915 self
.streamP4FilesCb(entry
)
1917 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
1919 p4CmdList(["-x", "-", "print"],
1921 cb
=streamP4FilesCbSelf
)
1924 if self
.stream_file
.has_key('depotFile'):
1925 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1927 def make_email(self
, userid
):
1928 if userid
in self
.users
:
1929 return self
.users
[userid
]
1931 return "%s <a@b>" % userid
1934 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
1936 print "writing tag %s for commit %s" % (labelName
, commit
)
1937 gitStream
.write("tag %s\n" % labelName
)
1938 gitStream
.write("from %s\n" % commit
)
1940 if labelDetails
.has_key('Owner'):
1941 owner
= labelDetails
["Owner"]
1945 # Try to use the owner of the p4 label, or failing that,
1946 # the current p4 user id.
1948 email
= self
.make_email(owner
)
1950 email
= self
.make_email(self
.p4UserId())
1951 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
1953 gitStream
.write("tagger %s\n" % tagger
)
1955 print "labelDetails=",labelDetails
1956 if labelDetails
.has_key('Description'):
1957 description
= labelDetails
['Description']
1959 description
= 'Label from git p4'
1961 gitStream
.write("data %d\n" % len(description
))
1962 gitStream
.write(description
)
1963 gitStream
.write("\n")
1965 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
1966 epoch
= details
["time"]
1967 author
= details
["user"]
1968 self
.branchPrefixes
= branchPrefixes
1971 print "commit into %s" % branch
1973 # start with reading files; if that fails, we should not
1977 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
1978 new_files
.append (f
)
1980 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
1982 self
.gitStream
.write("commit %s\n" % branch
)
1983 # gitStream.write("mark :%s\n" % details["change"])
1984 self
.committedChanges
.add(int(details
["change"]))
1986 if author
not in self
.users
:
1987 self
.getUserMapFromPerforceServer()
1988 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
1990 self
.gitStream
.write("committer %s\n" % committer
)
1992 self
.gitStream
.write("data <<EOT\n")
1993 self
.gitStream
.write(details
["desc"])
1994 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1995 % (','.join (branchPrefixes
), details
["change"]))
1996 if len(details
['options']) > 0:
1997 self
.gitStream
.write(": options = %s" % details
['options'])
1998 self
.gitStream
.write("]\nEOT\n\n")
2002 print "parent %s" % parent
2003 self
.gitStream
.write("from %s\n" % parent
)
2005 self
.streamP4Files(new_files
)
2006 self
.gitStream
.write("\n")
2008 change
= int(details
["change"])
2010 if self
.labels
.has_key(change
):
2011 label
= self
.labels
[change
]
2012 labelDetails
= label
[0]
2013 labelRevisions
= label
[1]
2015 print "Change %s is labelled %s" % (change
, labelDetails
)
2017 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2018 for p
in branchPrefixes
])
2020 if len(files
) == len(labelRevisions
):
2024 if info
["action"] in self
.delete_actions
:
2026 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2028 if cleanedFiles
== labelRevisions
:
2029 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2033 print ("Tag %s does not match with change %s: files do not match."
2034 % (labelDetails
["label"], change
))
2038 print ("Tag %s does not match with change %s: file count is different."
2039 % (labelDetails
["label"], change
))
2041 # Build a dictionary of changelists and labels, for "detect-labels" option.
2042 def getLabels(self
):
2045 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2046 if len(l
) > 0 and not self
.silent
:
2047 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2050 label
= output
["label"]
2054 print "Querying files for label %s" % label
2055 for file in p4CmdList(["files"] +
2056 ["%s...@%s" % (p
, label
)
2057 for p
in self
.depotPaths
]):
2058 revisions
[file["depotFile"]] = file["rev"]
2059 change
= int(file["change"])
2060 if change
> newestChange
:
2061 newestChange
= change
2063 self
.labels
[newestChange
] = [output
, revisions
]
2066 print "Label changes: %s" % self
.labels
.keys()
2068 # Import p4 labels as git tags. A direct mapping does not
2069 # exist, so assume that if all the files are at the same revision
2070 # then we can use that, or it's something more complicated we should
2072 def importP4Labels(self
, stream
, p4Labels
):
2074 print "import p4 labels: " + ' '.join(p4Labels
)
2076 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2077 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2078 if len(validLabelRegexp
) == 0:
2079 validLabelRegexp
= defaultLabelRegexp
2080 m
= re
.compile(validLabelRegexp
)
2082 for name
in p4Labels
:
2085 if not m
.match(name
):
2087 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2090 if name
in ignoredP4Labels
:
2093 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2095 # get the most recent changelist for each file in this label
2096 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2097 for p
in self
.depotPaths
])
2099 if change
.has_key('change'):
2100 # find the corresponding git commit; take the oldest commit
2101 changelist
= int(change
['change'])
2102 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2103 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2104 if len(gitCommit
) == 0:
2105 print "could not find git commit for changelist %d" % changelist
2107 gitCommit
= gitCommit
.strip()
2109 # Convert from p4 time format
2111 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2113 print "Could not convert label time %s" % labelDetail
['Update']
2116 when
= int(time
.mktime(tmwhen
))
2117 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2119 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2122 print "Label %s has no changelists - possibly deleted?" % name
2125 # We can't import this label; don't try again as it will get very
2126 # expensive repeatedly fetching all the files for labels that will
2127 # never be imported. If the label is moved in the future, the
2128 # ignore will need to be removed manually.
2129 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2131 def guessProjectName(self
):
2132 for p
in self
.depotPaths
:
2135 p
= p
[p
.strip().rfind("/") + 1:]
2136 if not p
.endswith("/"):
2140 def getBranchMapping(self
):
2141 lostAndFoundBranches
= set()
2143 user
= gitConfig("git-p4.branchUser")
2145 command
= "branches -u %s" % user
2147 command
= "branches"
2149 for info
in p4CmdList(command
):
2150 details
= p4Cmd(["branch", "-o", info
["branch"]])
2152 while details
.has_key("View%s" % viewIdx
):
2153 paths
= details
["View%s" % viewIdx
].split(" ")
2154 viewIdx
= viewIdx
+ 1
2155 # require standard //depot/foo/... //depot/bar/... mapping
2156 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2159 destination
= paths
[1]
2161 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2162 source
= source
[len(self
.depotPaths
[0]):-4]
2163 destination
= destination
[len(self
.depotPaths
[0]):-4]
2165 if destination
in self
.knownBranches
:
2167 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2168 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2171 self
.knownBranches
[destination
] = source
2173 lostAndFoundBranches
.discard(destination
)
2175 if source
not in self
.knownBranches
:
2176 lostAndFoundBranches
.add(source
)
2178 # Perforce does not strictly require branches to be defined, so we also
2179 # check git config for a branch list.
2181 # Example of branch definition in git config file:
2183 # branchList=main:branchA
2184 # branchList=main:branchB
2185 # branchList=branchA:branchC
2186 configBranches
= gitConfigList("git-p4.branchList")
2187 for branch
in configBranches
:
2189 (source
, destination
) = branch
.split(":")
2190 self
.knownBranches
[destination
] = source
2192 lostAndFoundBranches
.discard(destination
)
2194 if source
not in self
.knownBranches
:
2195 lostAndFoundBranches
.add(source
)
2198 for branch
in lostAndFoundBranches
:
2199 self
.knownBranches
[branch
] = branch
2201 def getBranchMappingFromGitBranches(self
):
2202 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2203 for branch
in branches
.keys():
2204 if branch
== "master":
2207 branch
= branch
[len(self
.projectName
):]
2208 self
.knownBranches
[branch
] = branch
2210 def listExistingP4GitBranches(self
):
2211 # branches holds mapping from name to commit
2212 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2213 self
.p4BranchesInGit
= branches
.keys()
2214 for branch
in branches
.keys():
2215 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2217 def updateOptionDict(self
, d
):
2219 if self
.keepRepoPath
:
2220 option_keys
['keepRepoPath'] = 1
2222 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2224 def readOptions(self
, d
):
2225 self
.keepRepoPath
= (d
.has_key('options')
2226 and ('keepRepoPath' in d
['options']))
2228 def gitRefForBranch(self
, branch
):
2229 if branch
== "main":
2230 return self
.refPrefix
+ "master"
2232 if len(branch
) <= 0:
2235 return self
.refPrefix
+ self
.projectName
+ branch
2237 def gitCommitByP4Change(self
, ref
, change
):
2239 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2242 latestCommit
= parseRevision(ref
)
2246 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2247 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2252 log
= extractLogMessageFromGitCommit(next
)
2253 settings
= extractSettingsGitLog(log
)
2254 currentChange
= int(settings
['change'])
2256 print "current change %s" % currentChange
2258 if currentChange
== change
:
2260 print "found %s" % next
2263 if currentChange
< change
:
2264 earliestCommit
= "^%s" % next
2266 latestCommit
= "%s" % next
2270 def importNewBranch(self
, branch
, maxChange
):
2271 # make fast-import flush all changes to disk and update the refs using the checkpoint
2272 # command so that we can try to find the branch parent in the git history
2273 self
.gitStream
.write("checkpoint\n\n");
2274 self
.gitStream
.flush();
2275 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2276 range = "@1,%s" % maxChange
2277 #print "prefix" + branchPrefix
2278 changes
= p4ChangesForPaths([branchPrefix
], range)
2279 if len(changes
) <= 0:
2281 firstChange
= changes
[0]
2282 #print "first change in branch: %s" % firstChange
2283 sourceBranch
= self
.knownBranches
[branch
]
2284 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2285 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2286 #print "source " + sourceBranch
2288 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2289 #print "branch parent: %s" % branchParentChange
2290 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2291 if len(gitParent
) > 0:
2292 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2293 #print "parent git commit: %s" % gitParent
2295 self
.importChanges(changes
)
2298 def searchParent(self
, parent
, branch
, target
):
2300 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2302 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2305 print "Found parent of %s in commit %s" % (branch
, blob
)
2312 def importChanges(self
, changes
):
2314 for change
in changes
:
2315 description
= p4Cmd(["describe", str(change
)])
2316 self
.updateOptionDict(description
)
2319 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2324 if self
.detectBranches
:
2325 branches
= self
.splitFilesIntoBranches(description
)
2326 for branch
in branches
.keys():
2328 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2332 filesForCommit
= branches
[branch
]
2335 print "branch is %s" % branch
2337 self
.updatedBranches
.add(branch
)
2339 if branch
not in self
.createdBranches
:
2340 self
.createdBranches
.add(branch
)
2341 parent
= self
.knownBranches
[branch
]
2342 if parent
== branch
:
2345 fullBranch
= self
.projectName
+ branch
2346 if fullBranch
not in self
.p4BranchesInGit
:
2348 print("\n Importing new branch %s" % fullBranch
);
2349 if self
.importNewBranch(branch
, change
- 1):
2351 self
.p4BranchesInGit
.append(fullBranch
)
2353 print("\n Resuming with change %s" % change
);
2356 print "parent determined through known branches: %s" % parent
2358 branch
= self
.gitRefForBranch(branch
)
2359 parent
= self
.gitRefForBranch(parent
)
2362 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2364 if len(parent
) == 0 and branch
in self
.initialParents
:
2365 parent
= self
.initialParents
[branch
]
2366 del self
.initialParents
[branch
]
2370 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2372 print "Creating temporary branch: " + tempBranch
2373 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2374 self
.tempBranches
.append(tempBranch
)
2376 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2378 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2381 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2382 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2384 files
= self
.extractFilesFromCommit(description
)
2385 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2387 self
.initialParent
= ""
2389 print self
.gitError
.read()
2392 def importHeadRevision(self
, revision
):
2393 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2396 details
["user"] = "git perforce import user"
2397 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2398 % (' '.join(self
.depotPaths
), revision
))
2399 details
["change"] = revision
2403 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2405 for info
in p4CmdList(["files"] + fileArgs
):
2407 if 'code' in info
and info
['code'] == 'error':
2408 sys
.stderr
.write("p4 returned an error: %s\n"
2410 if info
['data'].find("must refer to client") >= 0:
2411 sys
.stderr
.write("This particular p4 error is misleading.\n")
2412 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2413 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2415 if 'p4ExitCode' in info
:
2416 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2420 change
= int(info
["change"])
2421 if change
> newestRevision
:
2422 newestRevision
= change
2424 if info
["action"] in self
.delete_actions
:
2425 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2426 #fileCnt = fileCnt + 1
2429 for prop
in ["depotFile", "rev", "action", "type" ]:
2430 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2432 fileCnt
= fileCnt
+ 1
2434 details
["change"] = newestRevision
2436 # Use time from top-most change so that all git p4 clones of
2437 # the same p4 repo have the same commit SHA1s.
2438 res
= p4CmdList("describe -s %d" % newestRevision
)
2441 if r
.has_key('time'):
2442 newestTime
= int(r
['time'])
2443 if newestTime
is None:
2444 die("\"describe -s\" on newest change %d did not give a time")
2445 details
["time"] = newestTime
2447 self
.updateOptionDict(details
)
2449 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2451 print "IO error with git fast-import. Is your git version recent enough?"
2452 print self
.gitError
.read()
2455 def run(self
, args
):
2456 self
.depotPaths
= []
2457 self
.changeRange
= ""
2458 self
.initialParent
= ""
2459 self
.previousDepotPaths
= []
2461 # map from branch depot path to parent branch
2462 self
.knownBranches
= {}
2463 self
.initialParents
= {}
2464 self
.hasOrigin
= originP4BranchesExist()
2465 if not self
.syncWithOrigin
:
2466 self
.hasOrigin
= False
2468 if self
.importIntoRemotes
:
2469 self
.refPrefix
= "refs/remotes/p4/"
2471 self
.refPrefix
= "refs/heads/p4/"
2473 if self
.syncWithOrigin
and self
.hasOrigin
:
2475 print "Syncing with origin first by calling git fetch origin"
2476 system("git fetch origin")
2478 if len(self
.branch
) == 0:
2479 self
.branch
= self
.refPrefix
+ "master"
2480 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2481 system("git update-ref %s refs/heads/p4" % self
.branch
)
2482 system("git branch -D p4");
2483 # create it /after/ importing, when master exists
2484 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2485 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2487 # accept either the command-line option, or the configuration variable
2488 if self
.useClientSpec
:
2489 # will use this after clone to set the variable
2490 self
.useClientSpec_from_options
= True
2492 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2493 self
.useClientSpec
= True
2494 if self
.useClientSpec
:
2495 self
.clientSpecDirs
= getClientSpec()
2497 # TODO: should always look at previous commits,
2498 # merge with previous imports, if possible.
2501 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2502 self
.listExistingP4GitBranches()
2504 if len(self
.p4BranchesInGit
) > 1:
2506 print "Importing from/into multiple branches"
2507 self
.detectBranches
= True
2510 print "branches: %s" % self
.p4BranchesInGit
2513 for branch
in self
.p4BranchesInGit
:
2514 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2516 settings
= extractSettingsGitLog(logMsg
)
2518 self
.readOptions(settings
)
2519 if (settings
.has_key('depot-paths')
2520 and settings
.has_key ('change')):
2521 change
= int(settings
['change']) + 1
2522 p4Change
= max(p4Change
, change
)
2524 depotPaths
= sorted(settings
['depot-paths'])
2525 if self
.previousDepotPaths
== []:
2526 self
.previousDepotPaths
= depotPaths
2529 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2530 prev_list
= prev
.split("/")
2531 cur_list
= cur
.split("/")
2532 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2533 if cur_list
[i
] <> prev_list
[i
]:
2537 paths
.append ("/".join(cur_list
[:i
+ 1]))
2539 self
.previousDepotPaths
= paths
2542 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2543 self
.changeRange
= "@%s,#head" % p4Change
2544 if not self
.detectBranches
:
2545 self
.initialParent
= parseRevision(self
.branch
)
2546 if not self
.silent
and not self
.detectBranches
:
2547 print "Performing incremental import into %s git branch" % self
.branch
2549 if not self
.branch
.startswith("refs/"):
2550 self
.branch
= "refs/heads/" + self
.branch
2552 if len(args
) == 0 and self
.depotPaths
:
2554 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2556 if self
.depotPaths
and self
.depotPaths
!= args
:
2557 print ("previous import used depot path %s and now %s was specified. "
2558 "This doesn't work!" % (' '.join (self
.depotPaths
),
2562 self
.depotPaths
= sorted(args
)
2567 # Make sure no revision specifiers are used when --changesfile
2569 bad_changesfile
= False
2570 if len(self
.changesFile
) > 0:
2571 for p
in self
.depotPaths
:
2572 if p
.find("@") >= 0 or p
.find("#") >= 0:
2573 bad_changesfile
= True
2576 die("Option --changesfile is incompatible with revision specifiers")
2579 for p
in self
.depotPaths
:
2580 if p
.find("@") != -1:
2581 atIdx
= p
.index("@")
2582 self
.changeRange
= p
[atIdx
:]
2583 if self
.changeRange
== "@all":
2584 self
.changeRange
= ""
2585 elif ',' not in self
.changeRange
:
2586 revision
= self
.changeRange
2587 self
.changeRange
= ""
2589 elif p
.find("#") != -1:
2590 hashIdx
= p
.index("#")
2591 revision
= p
[hashIdx
:]
2593 elif self
.previousDepotPaths
== []:
2594 # pay attention to changesfile, if given, else import
2595 # the entire p4 tree at the head revision
2596 if len(self
.changesFile
) == 0:
2599 p
= re
.sub ("\.\.\.$", "", p
)
2600 if not p
.endswith("/"):
2605 self
.depotPaths
= newPaths
2607 self
.loadUserMapFromCache()
2609 if self
.detectLabels
:
2612 if self
.detectBranches
:
2613 ## FIXME - what's a P4 projectName ?
2614 self
.projectName
= self
.guessProjectName()
2617 self
.getBranchMappingFromGitBranches()
2619 self
.getBranchMapping()
2621 print "p4-git branches: %s" % self
.p4BranchesInGit
2622 print "initial parents: %s" % self
.initialParents
2623 for b
in self
.p4BranchesInGit
:
2627 b
= b
[len(self
.projectName
):]
2628 self
.createdBranches
.add(b
)
2630 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2632 importProcess
= subprocess
.Popen(["git", "fast-import"],
2633 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2634 stderr
=subprocess
.PIPE
);
2635 self
.gitOutput
= importProcess
.stdout
2636 self
.gitStream
= importProcess
.stdin
2637 self
.gitError
= importProcess
.stderr
2640 self
.importHeadRevision(revision
)
2644 if len(self
.changesFile
) > 0:
2645 output
= open(self
.changesFile
).readlines()
2648 changeSet
.add(int(line
))
2650 for change
in changeSet
:
2651 changes
.append(change
)
2655 # catch "git p4 sync" with no new branches, in a repo that
2656 # does not have any existing p4 branches
2657 if len(args
) == 0 and not self
.p4BranchesInGit
:
2658 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2660 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2662 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2664 if len(self
.maxChanges
) > 0:
2665 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2667 if len(changes
) == 0:
2669 print "No changes to import!"
2671 if not self
.silent
and not self
.detectBranches
:
2672 print "Import destination: %s" % self
.branch
2674 self
.updatedBranches
= set()
2676 self
.importChanges(changes
)
2680 if len(self
.updatedBranches
) > 0:
2681 sys
.stdout
.write("Updated branches: ")
2682 for b
in self
.updatedBranches
:
2683 sys
.stdout
.write("%s " % b
)
2684 sys
.stdout
.write("\n")
2686 if gitConfig("git-p4.importLabels", "--bool") == "true":
2687 self
.importLabels
= true
2689 if self
.importLabels
:
2690 p4Labels
= getP4Labels(self
.depotPaths
)
2691 gitTags
= getGitTags()
2693 missingP4Labels
= p4Labels
- gitTags
2694 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2696 self
.gitStream
.close()
2697 if importProcess
.wait() != 0:
2698 die("fast-import failed: %s" % self
.gitError
.read())
2699 self
.gitOutput
.close()
2700 self
.gitError
.close()
2702 # Cleanup temporary branches created during import
2703 if self
.tempBranches
!= []:
2704 for branch
in self
.tempBranches
:
2705 read_pipe("git update-ref -d %s" % branch
)
2706 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2710 class P4Rebase(Command
):
2712 Command
.__init
__(self
)
2714 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2715 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
2717 self
.verbose
= False
2718 self
.importLabels
= False
2719 self
.description
= ("Fetches the latest revision from perforce and "
2720 + "rebases the current work (branch) against it")
2722 def run(self
, args
):
2724 sync
.importLabels
= self
.importLabels
2727 return self
.rebase()
2730 if os
.system("git update-index --refresh") != 0:
2731 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2732 if len(read_pipe("git diff-index HEAD --")) > 0:
2733 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2735 [upstream
, settings
] = findUpstreamBranchPoint()
2736 if len(upstream
) == 0:
2737 die("Cannot find upstream branchpoint for rebase")
2739 # the branchpoint may be p4/foo~3, so strip off the parent
2740 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2742 print "Rebasing the current branch onto %s" % upstream
2743 oldHead
= read_pipe("git rev-parse HEAD").strip()
2744 system("git rebase %s" % upstream
)
2745 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2748 class P4Clone(P4Sync
):
2750 P4Sync
.__init
__(self
)
2751 self
.description
= "Creates a new git repository and imports from Perforce into it"
2752 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2754 optparse
.make_option("--destination", dest
="cloneDestination",
2755 action
='store', default
=None,
2756 help="where to leave result of the clone"),
2757 optparse
.make_option("-/", dest
="cloneExclude",
2758 action
="append", type="string",
2759 help="exclude depot path"),
2760 optparse
.make_option("--bare", dest
="cloneBare",
2761 action
="store_true", default
=False),
2763 self
.cloneDestination
= None
2764 self
.needsGit
= False
2765 self
.cloneBare
= False
2767 # This is required for the "append" cloneExclude action
2768 def ensure_value(self
, attr
, value
):
2769 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2770 setattr(self
, attr
, value
)
2771 return getattr(self
, attr
)
2773 def defaultDestination(self
, args
):
2774 ## TODO: use common prefix of args?
2776 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2777 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2778 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2779 depotDir
= re
.sub(r
"/$", "", depotDir
)
2780 return os
.path
.split(depotDir
)[1]
2782 def run(self
, args
):
2786 if self
.keepRepoPath
and not self
.cloneDestination
:
2787 sys
.stderr
.write("Must specify destination for --keep-path\n")
2792 if not self
.cloneDestination
and len(depotPaths
) > 1:
2793 self
.cloneDestination
= depotPaths
[-1]
2794 depotPaths
= depotPaths
[:-1]
2796 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2797 for p
in depotPaths
:
2798 if not p
.startswith("//"):
2801 if not self
.cloneDestination
:
2802 self
.cloneDestination
= self
.defaultDestination(args
)
2804 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2806 if not os
.path
.exists(self
.cloneDestination
):
2807 os
.makedirs(self
.cloneDestination
)
2808 chdir(self
.cloneDestination
)
2810 init_cmd
= [ "git", "init" ]
2812 init_cmd
.append("--bare")
2813 subprocess
.check_call(init_cmd
)
2815 if not P4Sync
.run(self
, depotPaths
):
2817 if self
.branch
!= "master":
2818 if self
.importIntoRemotes
:
2819 masterbranch
= "refs/remotes/p4/master"
2821 masterbranch
= "refs/heads/p4/master"
2822 if gitBranchExists(masterbranch
):
2823 system("git branch master %s" % masterbranch
)
2824 if not self
.cloneBare
:
2825 system("git checkout -f")
2827 print "Could not detect main branch. No checkout/master branch created."
2829 # auto-set this variable if invoked with --use-client-spec
2830 if self
.useClientSpec_from_options
:
2831 system("git config --bool git-p4.useclientspec true")
2835 class P4Branches(Command
):
2837 Command
.__init
__(self
)
2839 self
.description
= ("Shows the git branches that hold imports and their "
2840 + "corresponding perforce depot paths")
2841 self
.verbose
= False
2843 def run(self
, args
):
2844 if originP4BranchesExist():
2845 createOrUpdateBranchesFromOrigin()
2847 cmdline
= "git rev-parse --symbolic "
2848 cmdline
+= " --remotes"
2850 for line
in read_pipe_lines(cmdline
):
2853 if not line
.startswith('p4/') or line
== "p4/HEAD":
2857 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2858 settings
= extractSettingsGitLog(log
)
2860 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2863 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2865 optparse
.IndentedHelpFormatter
.__init
__(self
)
2867 def format_description(self
, description
):
2869 return description
+ "\n"
2873 def printUsage(commands
):
2874 print "usage: %s <command> [options]" % sys
.argv
[0]
2876 print "valid commands: %s" % ", ".join(commands
)
2878 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2883 "submit" : P4Submit
,
2884 "commit" : P4Submit
,
2886 "rebase" : P4Rebase
,
2888 "rollback" : P4RollBack
,
2889 "branches" : P4Branches
2894 if len(sys
.argv
[1:]) == 0:
2895 printUsage(commands
.keys())
2899 cmdName
= sys
.argv
[1]
2901 klass
= commands
[cmdName
]
2904 print "unknown command %s" % cmdName
2906 printUsage(commands
.keys())
2909 options
= cmd
.options
2910 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2914 if len(options
) > 0:
2916 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2918 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2920 description
= cmd
.description
,
2921 formatter
= HelpFormatter())
2923 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2925 verbose
= cmd
.verbose
2927 if cmd
.gitdir
== None:
2928 cmd
.gitdir
= os
.path
.abspath(".git")
2929 if not isValidGitDir(cmd
.gitdir
):
2930 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2931 if os
.path
.exists(cmd
.gitdir
):
2932 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2936 if not isValidGitDir(cmd
.gitdir
):
2937 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2938 cmd
.gitdir
+= "/.git"
2940 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2942 os
.environ
["GIT_DIR"] = cmd
.gitdir
2944 if not cmd
.run(args
):
2949 if __name__
== '__main__':