3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[A-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
124 expand
= isinstance(cmd
,basestring
)
126 sys
.stderr
.write("executing %s\n" % str(cmd
))
127 subprocess
.check_call(cmd
, shell
=expand
)
130 """Specifically invoke p4 as the system command. """
131 real_cmd
= p4_build_cmd(cmd
)
132 expand
= isinstance(real_cmd
, basestring
)
133 subprocess
.check_call(real_cmd
, shell
=expand
)
135 def p4_integrate(src
, dest
):
136 p4_system(["integrate", "-Dt", src
, dest
])
139 p4_system(["sync", path
])
142 p4_system(["add", f
])
145 p4_system(["delete", f
])
148 p4_system(["edit", f
])
151 p4_system(["revert", f
])
153 def p4_reopen(type, file):
154 p4_system(["reopen", "-t", type, file])
157 # Canonicalize the p4 type and return a tuple of the
158 # base type, plus any modifiers. See "p4 help filetypes"
159 # for a list and explanation.
161 def split_p4_type(p4type
):
163 p4_filetypes_historical
= {
164 "ctempobj": "binary+Sw",
170 "tempobj": "binary+FSw",
171 "ubinary": "binary+F",
172 "uresource": "resource+F",
173 "uxbinary": "binary+Fx",
174 "xbinary": "binary+x",
176 "xtempobj": "binary+Swx",
178 "xunicode": "unicode+x",
181 if p4type
in p4_filetypes_historical
:
182 p4type
= p4_filetypes_historical
[p4type
]
184 s
= p4type
.split("+")
192 # return the raw p4 type of a file (text, text+ko, etc)
195 results
= p4CmdList(["fstat", "-T", "headType", file])
196 return results
[0]['headType']
199 # Given a type base and modifier, return a regexp matching
200 # the keywords that can be expanded in the file
202 def p4_keywords_regexp_for_type(base
, type_mods
):
203 if base
in ("text", "unicode", "binary"):
205 if "ko" in type_mods
:
207 elif "k" in type_mods
:
208 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
212 \$ # Starts with a dollar, followed by...
213 (%s) # one of the keywords, followed by...
214 (:[^$]+)? # possibly an old expansion, followed by...
222 # Given a file, return a regexp matching the possible
223 # RCS keywords that will be expanded, or None for files
224 # with kw expansion turned off.
226 def p4_keywords_regexp_for_file(file):
227 if not os
.path
.exists(file):
230 (type_base
, type_mods
) = split_p4_type(p4_type(file))
231 return p4_keywords_regexp_for_type(type_base
, type_mods
)
233 def setP4ExecBit(file, mode
):
234 # Reopens an already open file and changes the execute bit to match
235 # the execute bit setting in the passed in mode.
239 if not isModeExec(mode
):
240 p4Type
= getP4OpenedType(file)
241 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
242 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
243 if p4Type
[-1] == "+":
244 p4Type
= p4Type
[0:-1]
246 p4_reopen(p4Type
, file)
248 def getP4OpenedType(file):
249 # Returns the perforce file type for the given file.
251 result
= p4_read_pipe(["opened", file])
252 match
= re
.match(".*\((.+)\)\r?$", result
)
254 return match
.group(1)
256 die("Could not determine file type for %s (result: '%s')" % (file, result
))
258 # Return the set of all p4 labels
259 def getP4Labels(depotPaths
):
261 if isinstance(depotPaths
,basestring
):
262 depotPaths
= [depotPaths
]
264 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
270 # Return the set of all git tags
273 for line
in read_pipe_lines(["git", "tag"]):
278 def diffTreePattern():
279 # This is a simple generator for the diff tree regex pattern. This could be
280 # a class variable if this and parseDiffTreeEntry were a part of a class.
281 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
285 def parseDiffTreeEntry(entry
):
286 """Parses a single diff tree entry into its component elements.
288 See git-diff-tree(1) manpage for details about the format of the diff
289 output. This method returns a dictionary with the following elements:
291 src_mode - The mode of the source file
292 dst_mode - The mode of the destination file
293 src_sha1 - The sha1 for the source file
294 dst_sha1 - The sha1 fr the destination file
295 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
296 status_score - The score for the status (applicable for 'C' and 'R'
297 statuses). This is None if there is no score.
298 src - The path for the source file.
299 dst - The path for the destination file. This is only present for
300 copy or renames. If it is not present, this is None.
302 If the pattern is not matched, None is returned."""
304 match
= diffTreePattern().next().match(entry
)
307 'src_mode': match
.group(1),
308 'dst_mode': match
.group(2),
309 'src_sha1': match
.group(3),
310 'dst_sha1': match
.group(4),
311 'status': match
.group(5),
312 'status_score': match
.group(6),
313 'src': match
.group(7),
314 'dst': match
.group(10)
318 def isModeExec(mode
):
319 # Returns True if the given git mode represents an executable file,
321 return mode
[-3:] == "755"
323 def isModeExecChanged(src_mode
, dst_mode
):
324 return isModeExec(src_mode
) != isModeExec(dst_mode
)
326 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
328 if isinstance(cmd
,basestring
):
335 cmd
= p4_build_cmd(cmd
)
337 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
339 # Use a temporary file to avoid deadlocks without
340 # subprocess.communicate(), which would put another copy
341 # of stdout into memory.
343 if stdin
is not None:
344 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
345 if isinstance(stdin
,basestring
):
346 stdin_file
.write(stdin
)
349 stdin_file
.write(i
+ '\n')
353 p4
= subprocess
.Popen(cmd
,
356 stdout
=subprocess
.PIPE
)
361 entry
= marshal
.load(p4
.stdout
)
371 entry
["p4ExitCode"] = exitCode
377 list = p4CmdList(cmd
)
383 def p4Where(depotPath
):
384 if not depotPath
.endswith("/"):
386 depotPath
= depotPath
+ "..."
387 outputList
= p4CmdList(["where", depotPath
])
389 for entry
in outputList
:
390 if "depotFile" in entry
:
391 if entry
["depotFile"] == depotPath
:
394 elif "data" in entry
:
395 data
= entry
.get("data")
396 space
= data
.find(" ")
397 if data
[:space
] == depotPath
:
402 if output
["code"] == "error":
406 clientPath
= output
.get("path")
407 elif "data" in output
:
408 data
= output
.get("data")
409 lastSpace
= data
.rfind(" ")
410 clientPath
= data
[lastSpace
+ 1:]
412 if clientPath
.endswith("..."):
413 clientPath
= clientPath
[:-3]
416 def currentGitBranch():
417 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
419 def isValidGitDir(path
):
420 if (os
.path
.exists(path
+ "/HEAD")
421 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
425 def parseRevision(ref
):
426 return read_pipe("git rev-parse %s" % ref
).strip()
428 def branchExists(ref
):
429 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
433 def extractLogMessageFromGitCommit(commit
):
436 ## fixme: title is first line of commit, not 1st paragraph.
438 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
447 def extractSettingsGitLog(log
):
449 for line
in log
.split("\n"):
451 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
455 assignments
= m
.group(1).split (':')
456 for a
in assignments
:
458 key
= vals
[0].strip()
459 val
= ('='.join (vals
[1:])).strip()
460 if val
.endswith ('\"') and val
.startswith('"'):
465 paths
= values
.get("depot-paths")
467 paths
= values
.get("depot-path")
469 values
['depot-paths'] = paths
.split(',')
472 def gitBranchExists(branch
):
473 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
474 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
475 return proc
.wait() == 0;
478 def gitConfig(key
, args
= None): # set args to "--bool", for instance
479 if not _gitConfig
.has_key(key
):
482 argsFilter
= "%s " % args
483 cmd
= "git config %s%s" % (argsFilter
, key
)
484 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
485 return _gitConfig
[key
]
487 def gitConfigList(key
):
488 if not _gitConfig
.has_key(key
):
489 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
490 return _gitConfig
[key
]
492 def p4BranchesInGit(branchesAreInRemotes
= True):
495 cmdline
= "git rev-parse --symbolic "
496 if branchesAreInRemotes
:
497 cmdline
+= " --remotes"
499 cmdline
+= " --branches"
501 for line
in read_pipe_lines(cmdline
):
504 ## only import to p4/
505 if not line
.startswith('p4/') or line
== "p4/HEAD":
510 branch
= re
.sub ("^p4/", "", line
)
512 branches
[branch
] = parseRevision(line
)
515 def findUpstreamBranchPoint(head
= "HEAD"):
516 branches
= p4BranchesInGit()
517 # map from depot-path to branch name
518 branchByDepotPath
= {}
519 for branch
in branches
.keys():
520 tip
= branches
[branch
]
521 log
= extractLogMessageFromGitCommit(tip
)
522 settings
= extractSettingsGitLog(log
)
523 if settings
.has_key("depot-paths"):
524 paths
= ",".join(settings
["depot-paths"])
525 branchByDepotPath
[paths
] = "remotes/p4/" + branch
529 while parent
< 65535:
530 commit
= head
+ "~%s" % parent
531 log
= extractLogMessageFromGitCommit(commit
)
532 settings
= extractSettingsGitLog(log
)
533 if settings
.has_key("depot-paths"):
534 paths
= ",".join(settings
["depot-paths"])
535 if branchByDepotPath
.has_key(paths
):
536 return [branchByDepotPath
[paths
], settings
]
540 return ["", settings
]
542 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
544 print ("Creating/updating branch(es) in %s based on origin branch(es)"
547 originPrefix
= "origin/p4/"
549 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
551 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
554 headName
= line
[len(originPrefix
):]
555 remoteHead
= localRefPrefix
+ headName
558 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
559 if (not original
.has_key('depot-paths')
560 or not original
.has_key('change')):
564 if not gitBranchExists(remoteHead
):
566 print "creating %s" % remoteHead
569 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
570 if settings
.has_key('change') > 0:
571 if settings
['depot-paths'] == original
['depot-paths']:
572 originP4Change
= int(original
['change'])
573 p4Change
= int(settings
['change'])
574 if originP4Change
> p4Change
:
575 print ("%s (%s) is newer than %s (%s). "
576 "Updating p4 branch from origin."
577 % (originHead
, originP4Change
,
578 remoteHead
, p4Change
))
581 print ("Ignoring: %s was imported from %s while "
582 "%s was imported from %s"
583 % (originHead
, ','.join(original
['depot-paths']),
584 remoteHead
, ','.join(settings
['depot-paths'])))
587 system("git update-ref %s %s" % (remoteHead
, originHead
))
589 def originP4BranchesExist():
590 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
592 def p4ChangesForPaths(depotPaths
, changeRange
):
596 cmd
+= ["%s...%s" % (p
, changeRange
)]
597 output
= p4_read_pipe_lines(cmd
)
601 changeNum
= int(line
.split(" ")[1])
602 changes
[changeNum
] = True
604 changelist
= changes
.keys()
608 def p4PathStartsWith(path
, prefix
):
609 # This method tries to remedy a potential mixed-case issue:
611 # If UserA adds //depot/DirA/file1
612 # and UserB adds //depot/dira/file2
614 # we may or may not have a problem. If you have core.ignorecase=true,
615 # we treat DirA and dira as the same directory
616 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
618 return path
.lower().startswith(prefix
.lower())
619 return path
.startswith(prefix
)
622 """Look at the p4 client spec, create a View() object that contains
623 all the mappings, and return it."""
625 specList
= p4CmdList("client -o")
626 if len(specList
) != 1:
627 die('Output from "client -o" is %d lines, expecting 1' %
630 # dictionary of all client parameters
633 # just the keys that start with "View"
634 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
639 # append the lines, in order, to the view
640 for view_num
in range(len(view_keys
)):
641 k
= "View%d" % view_num
642 if k
not in view_keys
:
643 die("Expected view key %s missing" % k
)
644 view
.append(entry
[k
])
649 """Grab the client directory."""
651 output
= p4CmdList("client -o")
653 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
656 if "Root" not in entry
:
657 die('Client has no "Root"')
663 self
.usage
= "usage: %prog [options]"
668 self
.userMapFromPerforceServer
= False
669 self
.myP4UserId
= None
673 return self
.myP4UserId
675 results
= p4CmdList("user -o")
677 if r
.has_key('User'):
678 self
.myP4UserId
= r
['User']
680 die("Could not find your p4 user id")
682 def p4UserIsMe(self
, p4User
):
683 # return True if the given p4 user is actually me
685 if not p4User
or p4User
!= me
:
690 def getUserCacheFilename(self
):
691 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
692 return home
+ "/.gitp4-usercache.txt"
694 def getUserMapFromPerforceServer(self
):
695 if self
.userMapFromPerforceServer
:
700 for output
in p4CmdList("users"):
701 if not output
.has_key("User"):
703 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
704 self
.emails
[output
["Email"]] = output
["User"]
708 for (key
, val
) in self
.users
.items():
709 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
711 open(self
.getUserCacheFilename(), "wb").write(s
)
712 self
.userMapFromPerforceServer
= True
714 def loadUserMapFromCache(self
):
716 self
.userMapFromPerforceServer
= False
718 cache
= open(self
.getUserCacheFilename(), "rb")
719 lines
= cache
.readlines()
722 entry
= line
.strip().split("\t")
723 self
.users
[entry
[0]] = entry
[1]
725 self
.getUserMapFromPerforceServer()
727 class P4Debug(Command
):
729 Command
.__init
__(self
)
731 optparse
.make_option("--verbose", dest
="verbose", action
="store_true",
734 self
.description
= "A tool to debug the output of p4 -G."
735 self
.needsGit
= False
740 for output
in p4CmdList(args
):
741 print 'Element: %d' % j
746 class P4RollBack(Command
):
748 Command
.__init
__(self
)
750 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
751 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
753 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
755 self
.rollbackLocalBranches
= False
760 maxChange
= int(args
[0])
762 if "p4ExitCode" in p4Cmd("changes -m 1"):
763 die("Problems executing p4");
765 if self
.rollbackLocalBranches
:
766 refPrefix
= "refs/heads/"
767 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
769 refPrefix
= "refs/remotes/"
770 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
773 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
775 ref
= refPrefix
+ line
776 log
= extractLogMessageFromGitCommit(ref
)
777 settings
= extractSettingsGitLog(log
)
779 depotPaths
= settings
['depot-paths']
780 change
= settings
['change']
784 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
785 for p
in depotPaths
]))) == 0:
786 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
787 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
790 while change
and int(change
) > maxChange
:
793 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
794 system("git update-ref %s \"%s^\"" % (ref
, ref
))
795 log
= extractLogMessageFromGitCommit(ref
)
796 settings
= extractSettingsGitLog(log
)
799 depotPaths
= settings
['depot-paths']
800 change
= settings
['change']
803 print "%s rewound to %s" % (ref
, change
)
807 class P4Submit(Command
, P4UserMap
):
809 Command
.__init
__(self
)
810 P4UserMap
.__init
__(self
)
812 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
813 optparse
.make_option("--origin", dest
="origin"),
814 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
815 # preserve the user, requires relevant p4 permissions
816 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
817 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
819 self
.description
= "Submit changes from git to the perforce depot."
820 self
.usage
+= " [name of git branch to submit into perforce depot]"
821 self
.interactive
= True
823 self
.detectRenames
= False
825 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
826 self
.isWindows
= (platform
.system() == "Windows")
827 self
.exportLabels
= False
830 if len(p4CmdList("opened ...")) > 0:
831 die("You have files opened with perforce! Close them before starting the sync.")
833 # replaces everything between 'Description:' and the next P4 submit template field with the
835 def prepareLogMessage(self
, template
, message
):
838 inDescriptionSection
= False
840 for line
in template
.split("\n"):
841 if line
.startswith("#"):
842 result
+= line
+ "\n"
845 if inDescriptionSection
:
846 if line
.startswith("Files:") or line
.startswith("Jobs:"):
847 inDescriptionSection
= False
851 if line
.startswith("Description:"):
852 inDescriptionSection
= True
854 for messageLine
in message
.split("\n"):
855 line
+= "\t" + messageLine
+ "\n"
857 result
+= line
+ "\n"
861 def patchRCSKeywords(self
, file, pattern
):
862 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
863 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
865 outFile
= os
.fdopen(handle
, "w+")
866 inFile
= open(file, "r")
867 regexp
= re
.compile(pattern
, re
.VERBOSE
)
868 for line
in inFile
.readlines():
869 line
= regexp
.sub(r
'$\1$', line
)
873 # Forcibly overwrite the original file
875 shutil
.move(outFileName
, file)
877 # cleanup our temporary file
878 os
.unlink(outFileName
)
879 print "Failed to strip RCS keywords in %s" % file
882 print "Patched up RCS keywords in %s" % file
884 def p4UserForCommit(self
,id):
885 # Return the tuple (perforce user,git email) for a given git commit id
886 self
.getUserMapFromPerforceServer()
887 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
888 gitEmail
= gitEmail
.strip()
889 if not self
.emails
.has_key(gitEmail
):
890 return (None,gitEmail
)
892 return (self
.emails
[gitEmail
],gitEmail
)
894 def checkValidP4Users(self
,commits
):
895 # check if any git authors cannot be mapped to p4 users
897 (user
,email
) = self
.p4UserForCommit(id)
899 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
900 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
903 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
905 def lastP4Changelist(self
):
906 # Get back the last changelist number submitted in this client spec. This
907 # then gets used to patch up the username in the change. If the same
908 # client spec is being used by multiple processes then this might go
910 results
= p4CmdList("client -o") # find the current client
913 if r
.has_key('Client'):
917 die("could not get client spec")
918 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
920 if r
.has_key('change'):
922 die("Could not get changelist number for last submit - cannot patch up user details")
924 def modifyChangelistUser(self
, changelist
, newUser
):
925 # fixup the user field of a changelist after it has been submitted.
926 changes
= p4CmdList("change -o %s" % changelist
)
927 if len(changes
) != 1:
928 die("Bad output from p4 change modifying %s to user %s" %
929 (changelist
, newUser
))
932 if c
['User'] == newUser
: return # nothing to do
934 input = marshal
.dumps(c
)
936 result
= p4CmdList("change -f -i", stdin
=input)
938 if r
.has_key('code'):
939 if r
['code'] == 'error':
940 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
941 if r
.has_key('data'):
942 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
944 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
946 def canChangeChangelists(self
):
947 # check to see if we have p4 admin or super-user permissions, either of
948 # which are required to modify changelists.
949 results
= p4CmdList(["protects", self
.depotPath
])
951 if r
.has_key('perm'):
952 if r
['perm'] == 'admin':
954 if r
['perm'] == 'super':
958 def prepareSubmitTemplate(self
):
959 # remove lines in the Files section that show changes to files outside the depot path we're committing into
961 inFilesSection
= False
962 for line
in p4_read_pipe_lines(['change', '-o']):
963 if line
.endswith("\r\n"):
964 line
= line
[:-2] + "\n"
966 if line
.startswith("\t"):
967 # path starts and ends with a tab
969 lastTab
= path
.rfind("\t")
971 path
= path
[:lastTab
]
972 if not p4PathStartsWith(path
, self
.depotPath
):
975 inFilesSection
= False
977 if line
.startswith("Files:"):
978 inFilesSection
= True
984 def edit_template(self
, template_file
):
985 """Invoke the editor to let the user change the submission
986 message. Return true if okay to continue with the submit."""
988 # if configured to skip the editing part, just submit
989 if gitConfig("git-p4.skipSubmitEdit") == "true":
992 # look at the modification time, to check later if the user saved
994 mtime
= os
.stat(template_file
).st_mtime
997 if os
.environ
.has_key("P4EDITOR"):
998 editor
= os
.environ
.get("P4EDITOR")
1000 editor
= read_pipe("git var GIT_EDITOR").strip()
1001 system(editor
+ " " + template_file
)
1003 # If the file was not saved, prompt to see if this patch should
1004 # be skipped. But skip this verification step if configured so.
1005 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1008 # modification time updated means user saved the file
1009 if os
.stat(template_file
).st_mtime
> mtime
:
1013 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1019 def applyCommit(self
, id):
1020 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1022 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1024 if not self
.detectRenames
:
1025 # If not explicitly set check the config variable
1026 self
.detectRenames
= gitConfig("git-p4.detectRenames")
1028 if self
.detectRenames
.lower() == "false" or self
.detectRenames
== "":
1030 elif self
.detectRenames
.lower() == "true":
1033 diffOpts
= "-M%s" % self
.detectRenames
1035 detectCopies
= gitConfig("git-p4.detectCopies")
1036 if detectCopies
.lower() == "true":
1038 elif detectCopies
!= "" and detectCopies
.lower() != "false":
1039 diffOpts
+= " -C%s" % detectCopies
1041 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1042 diffOpts
+= " --find-copies-harder"
1044 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts
, id, id))
1046 filesToDelete
= set()
1048 filesToChangeExecBit
= {}
1051 diff
= parseDiffTreeEntry(line
)
1052 modifier
= diff
['status']
1056 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1057 filesToChangeExecBit
[path
] = diff
['dst_mode']
1058 editedFiles
.add(path
)
1059 elif modifier
== "A":
1060 filesToAdd
.add(path
)
1061 filesToChangeExecBit
[path
] = diff
['dst_mode']
1062 if path
in filesToDelete
:
1063 filesToDelete
.remove(path
)
1064 elif modifier
== "D":
1065 filesToDelete
.add(path
)
1066 if path
in filesToAdd
:
1067 filesToAdd
.remove(path
)
1068 elif modifier
== "C":
1069 src
, dest
= diff
['src'], diff
['dst']
1070 p4_integrate(src
, dest
)
1071 if diff
['src_sha1'] != diff
['dst_sha1']:
1073 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1075 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1077 editedFiles
.add(dest
)
1078 elif modifier
== "R":
1079 src
, dest
= diff
['src'], diff
['dst']
1080 p4_integrate(src
, dest
)
1081 if diff
['src_sha1'] != diff
['dst_sha1']:
1083 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1085 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1087 editedFiles
.add(dest
)
1088 filesToDelete
.add(src
)
1090 die("unknown modifier %s for %s" % (modifier
, path
))
1092 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1093 patchcmd
= diffcmd
+ " | git apply "
1094 tryPatchCmd
= patchcmd
+ "--check -"
1095 applyPatchCmd
= patchcmd
+ "--check --apply -"
1096 patch_succeeded
= True
1098 if os
.system(tryPatchCmd
) != 0:
1099 fixed_rcs_keywords
= False
1100 patch_succeeded
= False
1101 print "Unfortunately applying the change failed!"
1103 # Patch failed, maybe it's just RCS keyword woes. Look through
1104 # the patch to see if that's possible.
1105 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1109 for file in editedFiles | filesToDelete
:
1110 # did this file's delta contain RCS keywords?
1111 pattern
= p4_keywords_regexp_for_file(file)
1114 # this file is a possibility...look for RCS keywords.
1115 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1116 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1117 if regexp
.search(line
):
1119 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1120 kwfiles
[file] = pattern
1123 for file in kwfiles
:
1125 print "zapping %s with %s" % (line
,pattern
)
1126 self
.patchRCSKeywords(file, kwfiles
[file])
1127 fixed_rcs_keywords
= True
1129 if fixed_rcs_keywords
:
1130 print "Retrying the patch with RCS keywords cleaned up"
1131 if os
.system(tryPatchCmd
) == 0:
1132 patch_succeeded
= True
1134 if not patch_succeeded
:
1135 print "What do you want to do?"
1137 while response
!= "s" and response
!= "a" and response
!= "w":
1138 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1139 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1141 print "Skipping! Good luck with the next patches..."
1142 for f
in editedFiles
:
1144 for f
in filesToAdd
:
1147 elif response
== "a":
1148 os
.system(applyPatchCmd
)
1149 if len(filesToAdd
) > 0:
1150 print "You may also want to call p4 add on the following files:"
1151 print " ".join(filesToAdd
)
1152 if len(filesToDelete
):
1153 print "The following files should be scheduled for deletion with p4 delete:"
1154 print " ".join(filesToDelete
)
1155 die("Please resolve and submit the conflict manually and "
1156 + "continue afterwards with git p4 submit --continue")
1157 elif response
== "w":
1158 system(diffcmd
+ " > patch.txt")
1159 print "Patch saved to patch.txt in %s !" % self
.clientPath
1160 die("Please resolve and submit the conflict manually and "
1161 "continue afterwards with git p4 submit --continue")
1163 system(applyPatchCmd
)
1165 for f
in filesToAdd
:
1167 for f
in filesToDelete
:
1171 # Set/clear executable bits
1172 for f
in filesToChangeExecBit
.keys():
1173 mode
= filesToChangeExecBit
[f
]
1174 setP4ExecBit(f
, mode
)
1176 logMessage
= extractLogMessageFromGitCommit(id)
1177 logMessage
= logMessage
.strip()
1179 template
= self
.prepareSubmitTemplate()
1181 if self
.interactive
:
1182 submitTemplate
= self
.prepareLogMessage(template
, logMessage
)
1184 if self
.preserveUser
:
1185 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1187 if os
.environ
.has_key("P4DIFF"):
1188 del(os
.environ
["P4DIFF"])
1190 for editedFile
in editedFiles
:
1191 diff
+= p4_read_pipe(['diff', '-du', editedFile
])
1194 for newFile
in filesToAdd
:
1195 newdiff
+= "==== new file ====\n"
1196 newdiff
+= "--- /dev/null\n"
1197 newdiff
+= "+++ %s\n" % newFile
1198 f
= open(newFile
, "r")
1199 for line
in f
.readlines():
1200 newdiff
+= "+" + line
1203 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1204 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1205 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1206 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1208 separatorLine
= "######## everything below this line is just the diff #######\n"
1210 (handle
, fileName
) = tempfile
.mkstemp()
1211 tmpFile
= os
.fdopen(handle
, "w+")
1213 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1214 separatorLine
= separatorLine
.replace("\n", "\r\n")
1215 newdiff
= newdiff
.replace("\n", "\r\n")
1216 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1219 if self
.edit_template(fileName
):
1220 # read the edited message and submit
1221 tmpFile
= open(fileName
, "rb")
1222 message
= tmpFile
.read()
1224 submitTemplate
= message
[:message
.index(separatorLine
)]
1226 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1227 p4_write_pipe(['submit', '-i'], submitTemplate
)
1229 if self
.preserveUser
:
1231 # Get last changelist number. Cannot easily get it from
1232 # the submit command output as the output is
1234 changelist
= self
.lastP4Changelist()
1235 self
.modifyChangelistUser(changelist
, p4User
)
1238 print "Submission cancelled, undoing p4 changes."
1239 for f
in editedFiles
:
1241 for f
in filesToAdd
:
1247 fileName
= "submit.txt"
1248 file = open(fileName
, "w+")
1249 file.write(self
.prepareLogMessage(template
, logMessage
))
1251 print ("Perforce submit template written as %s. "
1252 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1253 % (fileName
, fileName
))
1255 # Export git tags as p4 labels. Create a p4 label and then tag
1257 def exportGitTags(self
, gitTags
):
1258 validTagRegexp
= gitConfig("git-p4.validTagRegexp")
1259 if len(validTagRegexp
) == 0:
1260 validTagRegexp
= defaultLabelRegexp
1261 m
= re
.compile(validTagRegexp
)
1262 commit_re
= re
.compile(r
'\s*\[git-p4:.*change = (\d+)\s*\]')
1264 for name
in gitTags
:
1266 if not m
.match(name
):
1268 print "tag %s does not match regexp %s" % (name
, validTagRegexp
)
1271 # Get the p4 commit this corresponds to
1273 for l
in read_pipe_lines(["git", "log", "--max-count=1", name
]):
1274 match
= commit_re
.match(l
)
1276 changelist
= match
.group(1)
1279 # a tag pointing to something not sent to p4; ignore
1281 print "git tag %s does not give a p4 commit" % name
1284 # Get the tag details.
1288 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1291 if re
.match(r
'tag\s+', l
):
1293 elif re
.match(r
'\s*$', l
):
1300 body
= ["lightweight tag imported by git p4\n"]
1302 # Create the label - use the same view as the client spec we are using
1303 clientSpec
= getClientSpec()
1305 labelTemplate
= "Label: %s\n" % name
1306 labelTemplate
+= "Description:\n"
1308 labelTemplate
+= "\t" + b
+ "\n"
1309 labelTemplate
+= "View:\n"
1310 for mapping
in clientSpec
.mappings
:
1311 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1313 p4_write_pipe(["label", "-i"], labelTemplate
)
1316 p4_system(["tag", "-l", name
] +
1317 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1320 print "created p4 label for tag %s" % name
1322 def run(self
, args
):
1324 self
.master
= currentGitBranch()
1325 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1326 die("Detecting current git branch failed!")
1327 elif len(args
) == 1:
1328 self
.master
= args
[0]
1329 if not branchExists(self
.master
):
1330 die("Branch %s does not exist" % self
.master
)
1334 allowSubmit
= gitConfig("git-p4.allowSubmit")
1335 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1336 die("%s is not in git-p4.allowSubmit" % self
.master
)
1338 [upstream
, settings
] = findUpstreamBranchPoint()
1339 self
.depotPath
= settings
['depot-paths'][0]
1340 if len(self
.origin
) == 0:
1341 self
.origin
= upstream
1343 if self
.preserveUser
:
1344 if not self
.canChangeChangelists():
1345 die("Cannot preserve user names without p4 super-user or admin permissions")
1348 print "Origin branch is " + self
.origin
1350 if len(self
.depotPath
) == 0:
1351 print "Internal error: cannot locate perforce depot path from existing branches"
1354 self
.useClientSpec
= False
1355 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1356 self
.useClientSpec
= True
1357 if self
.useClientSpec
:
1358 self
.clientSpecDirs
= getClientSpec()
1360 if self
.useClientSpec
:
1361 # all files are relative to the client spec
1362 self
.clientPath
= getClientRoot()
1364 self
.clientPath
= p4Where(self
.depotPath
)
1366 if self
.clientPath
== "":
1367 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1369 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1370 self
.oldWorkingDirectory
= os
.getcwd()
1372 # ensure the clientPath exists
1373 if not os
.path
.exists(self
.clientPath
):
1374 os
.makedirs(self
.clientPath
)
1376 chdir(self
.clientPath
)
1377 print "Synchronizing p4 checkout..."
1382 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1383 commits
.append(line
.strip())
1386 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1387 self
.checkAuthorship
= False
1389 self
.checkAuthorship
= True
1391 if self
.preserveUser
:
1392 self
.checkValidP4Users(commits
)
1394 while len(commits
) > 0:
1396 commits
= commits
[1:]
1397 self
.applyCommit(commit
)
1398 if not self
.interactive
:
1401 if len(commits
) == 0:
1402 print "All changes applied!"
1403 chdir(self
.oldWorkingDirectory
)
1411 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1412 self
.exportLabels
= true
1414 if self
.exportLabels
:
1415 p4Labels
= getP4Labels(self
.depotPath
)
1416 gitTags
= getGitTags()
1418 missingGitTags
= gitTags
- p4Labels
1419 self
.exportGitTags(missingGitTags
)
1424 """Represent a p4 view ("p4 help views"), and map files in a
1425 repo according to the view."""
1428 """A depot or client path, possibly containing wildcards.
1429 The only one supported is ... at the end, currently.
1430 Initialize with the full path, with //depot or //client."""
1432 def __init__(self
, path
, is_depot
):
1434 self
.is_depot
= is_depot
1435 self
.find_wildcards()
1436 # remember the prefix bit, useful for relative mappings
1437 m
= re
.match("(//[^/]+/)", self
.path
)
1439 die("Path %s does not start with //prefix/" % self
.path
)
1441 if not self
.is_depot
:
1442 # strip //client/ on client paths
1443 self
.path
= self
.path
[len(prefix
):]
1445 def find_wildcards(self
):
1446 """Make sure wildcards are valid, and set up internal
1449 self
.ends_triple_dot
= False
1450 # There are three wildcards allowed in p4 views
1451 # (see "p4 help views"). This code knows how to
1452 # handle "..." (only at the end), but cannot deal with
1453 # "%%n" or "*". Only check the depot_side, as p4 should
1454 # validate that the client_side matches too.
1455 if re
.search(r
'%%[1-9]', self
.path
):
1456 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1457 if self
.path
.find("*") >= 0:
1458 die("Can't handle * wildcards in view: %s" % self
.path
)
1459 triple_dot_index
= self
.path
.find("...")
1460 if triple_dot_index
>= 0:
1461 if triple_dot_index
!= len(self
.path
) - 3:
1462 die("Can handle only single ... wildcard, at end: %s" %
1464 self
.ends_triple_dot
= True
1466 def ensure_compatible(self
, other_path
):
1467 """Make sure the wildcards agree."""
1468 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1469 die("Both paths must end with ... if either does;\n" +
1470 "paths: %s %s" % (self
.path
, other_path
.path
))
1472 def match_wildcards(self
, test_path
):
1473 """See if this test_path matches us, and fill in the value
1474 of the wildcards if so. Returns a tuple of
1475 (True|False, wildcards[]). For now, only the ... at end
1476 is supported, so at most one wildcard."""
1477 if self
.ends_triple_dot
:
1478 dotless
= self
.path
[:-3]
1479 if test_path
.startswith(dotless
):
1480 wildcard
= test_path
[len(dotless
):]
1481 return (True, [ wildcard
])
1483 if test_path
== self
.path
:
1487 def match(self
, test_path
):
1488 """Just return if it matches; don't bother with the wildcards."""
1489 b
, _
= self
.match_wildcards(test_path
)
1492 def fill_in_wildcards(self
, wildcards
):
1493 """Return the relative path, with the wildcards filled in
1494 if there are any."""
1495 if self
.ends_triple_dot
:
1496 return self
.path
[:-3] + wildcards
[0]
1500 class Mapping(object):
1501 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1502 # depot_side is without the trailing /... if it had one
1503 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1504 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1505 self
.overlay
= overlay
# started with "+"
1506 self
.exclude
= exclude
# started with "-"
1507 assert not (self
.overlay
and self
.exclude
)
1508 self
.depot_side
.ensure_compatible(self
.client_side
)
1516 return "View.Mapping: %s%s -> %s" % \
1517 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1519 def map_depot_to_client(self
, depot_path
):
1520 """Calculate the client path if using this mapping on the
1521 given depot path; does not consider the effect of other
1522 mappings in a view. Even excluded mappings are returned."""
1523 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1526 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1535 def append(self
, view_line
):
1536 """Parse a view line, splitting it into depot and client
1537 sides. Append to self.mappings, preserving order."""
1539 # Split the view line into exactly two words. P4 enforces
1540 # structure on these lines that simplifies this quite a bit.
1542 # Either or both words may be double-quoted.
1543 # Single quotes do not matter.
1544 # Double-quote marks cannot occur inside the words.
1545 # A + or - prefix is also inside the quotes.
1546 # There are no quotes unless they contain a space.
1547 # The line is already white-space stripped.
1548 # The two words are separated by a single space.
1550 if view_line
[0] == '"':
1551 # First word is double quoted. Find its end.
1552 close_quote_index
= view_line
.find('"', 1)
1553 if close_quote_index
<= 0:
1554 die("No first-word closing quote found: %s" % view_line
)
1555 depot_side
= view_line
[1:close_quote_index
]
1556 # skip closing quote and space
1557 rhs_index
= close_quote_index
+ 1 + 1
1559 space_index
= view_line
.find(" ")
1560 if space_index
<= 0:
1561 die("No word-splitting space found: %s" % view_line
)
1562 depot_side
= view_line
[0:space_index
]
1563 rhs_index
= space_index
+ 1
1565 if view_line
[rhs_index
] == '"':
1566 # Second word is double quoted. Make sure there is a
1567 # double quote at the end too.
1568 if not view_line
.endswith('"'):
1569 die("View line with rhs quote should end with one: %s" %
1572 client_side
= view_line
[rhs_index
+1:-1]
1574 client_side
= view_line
[rhs_index
:]
1576 # prefix + means overlay on previous mapping
1578 if depot_side
.startswith("+"):
1580 depot_side
= depot_side
[1:]
1582 # prefix - means exclude this path
1584 if depot_side
.startswith("-"):
1586 depot_side
= depot_side
[1:]
1588 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1589 self
.mappings
.append(m
)
1591 def map_in_client(self
, depot_path
):
1592 """Return the relative location in the client where this
1593 depot file should live. Returns "" if the file should
1594 not be mapped in the client."""
1599 # look at later entries first
1600 for m
in self
.mappings
[::-1]:
1602 # see where will this path end up in the client
1603 p
= m
.map_depot_to_client(depot_path
)
1606 # Depot path does not belong in client. Must remember
1607 # this, as previous items should not cause files to
1608 # exist in this path either. Remember that the list is
1609 # being walked from the end, which has higher precedence.
1610 # Overlap mappings do not exclude previous mappings.
1612 paths_filled
.append(m
.client_side
)
1615 # This mapping matched; no need to search any further.
1616 # But, the mapping could be rejected if the client path
1617 # has already been claimed by an earlier mapping (i.e.
1618 # one later in the list, which we are walking backwards).
1619 already_mapped_in_client
= False
1620 for f
in paths_filled
:
1621 # this is View.Path.match
1623 already_mapped_in_client
= True
1625 if not already_mapped_in_client
:
1626 # Include this file, unless it is from a line that
1627 # explicitly said to exclude it.
1631 # a match, even if rejected, always stops the search
1636 class P4Sync(Command
, P4UserMap
):
1637 delete_actions
= ( "delete", "move/delete", "purge" )
1640 Command
.__init
__(self
)
1641 P4UserMap
.__init
__(self
)
1643 optparse
.make_option("--branch", dest
="branch"),
1644 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1645 optparse
.make_option("--changesfile", dest
="changesFile"),
1646 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1647 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1648 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1649 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
1650 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1651 help="Import into refs/heads/ , not refs/remotes"),
1652 optparse
.make_option("--max-changes", dest
="maxChanges"),
1653 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1654 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1655 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1656 help="Only sync files that are included in the Perforce Client Spec")
1658 self
.description
= """Imports from Perforce into a git repository.\n
1660 //depot/my/project/ -- to import the current head
1661 //depot/my/project/@all -- to import everything
1662 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1664 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1666 self
.usage
+= " //depot/path[@revRange]"
1668 self
.createdBranches
= set()
1669 self
.committedChanges
= set()
1671 self
.detectBranches
= False
1672 self
.detectLabels
= False
1673 self
.importLabels
= False
1674 self
.changesFile
= ""
1675 self
.syncWithOrigin
= True
1676 self
.verbose
= False
1677 self
.importIntoRemotes
= True
1678 self
.maxChanges
= ""
1679 self
.isWindows
= (platform
.system() == "Windows")
1680 self
.keepRepoPath
= False
1681 self
.depotPaths
= None
1682 self
.p4BranchesInGit
= []
1683 self
.cloneExclude
= []
1684 self
.useClientSpec
= False
1685 self
.useClientSpec_from_options
= False
1686 self
.clientSpecDirs
= None
1687 self
.tempBranches
= []
1688 self
.tempBranchLocation
= "git-p4-tmp"
1690 if gitConfig("git-p4.syncFromOrigin") == "false":
1691 self
.syncWithOrigin
= False
1694 # P4 wildcards are not allowed in filenames. P4 complains
1695 # if you simply add them, but you can force it with "-f", in
1696 # which case it translates them into %xx encoding internally.
1697 # Search for and fix just these four characters. Do % last so
1698 # that fixing it does not inadvertently create new %-escapes.
1700 def wildcard_decode(self
, path
):
1701 # Cannot have * in a filename in windows; untested as to
1702 # what p4 would do in such a case.
1703 if not self
.isWindows
:
1704 path
= path
.replace("%2A", "*")
1705 path
= path
.replace("%23", "#") \
1706 .replace("%40", "@") \
1707 .replace("%25", "%")
1710 # Force a checkpoint in fast-import and wait for it to finish
1711 def checkpoint(self
):
1712 self
.gitStream
.write("checkpoint\n\n")
1713 self
.gitStream
.write("progress checkpoint\n\n")
1714 out
= self
.gitOutput
.readline()
1716 print "checkpoint finished: " + out
1718 def extractFilesFromCommit(self
, commit
):
1719 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1720 for path
in self
.cloneExclude
]
1723 while commit
.has_key("depotFile%s" % fnum
):
1724 path
= commit
["depotFile%s" % fnum
]
1726 if [p
for p
in self
.cloneExclude
1727 if p4PathStartsWith(path
, p
)]:
1730 found
= [p
for p
in self
.depotPaths
1731 if p4PathStartsWith(path
, p
)]
1738 file["rev"] = commit
["rev%s" % fnum
]
1739 file["action"] = commit
["action%s" % fnum
]
1740 file["type"] = commit
["type%s" % fnum
]
1745 def stripRepoPath(self
, path
, prefixes
):
1746 if self
.useClientSpec
:
1747 return self
.clientSpecDirs
.map_in_client(path
)
1749 if self
.keepRepoPath
:
1750 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1753 if p4PathStartsWith(path
, p
):
1754 path
= path
[len(p
):]
1758 def splitFilesIntoBranches(self
, commit
):
1761 while commit
.has_key("depotFile%s" % fnum
):
1762 path
= commit
["depotFile%s" % fnum
]
1763 found
= [p
for p
in self
.depotPaths
1764 if p4PathStartsWith(path
, p
)]
1771 file["rev"] = commit
["rev%s" % fnum
]
1772 file["action"] = commit
["action%s" % fnum
]
1773 file["type"] = commit
["type%s" % fnum
]
1776 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1778 for branch
in self
.knownBranches
.keys():
1780 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1781 if relPath
.startswith(branch
+ "/"):
1782 if branch
not in branches
:
1783 branches
[branch
] = []
1784 branches
[branch
].append(file)
1789 # output one file from the P4 stream
1790 # - helper for streamP4Files
1792 def streamOneP4File(self
, file, contents
):
1793 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1794 relPath
= self
.wildcard_decode(relPath
)
1796 sys
.stderr
.write("%s\n" % relPath
)
1798 (type_base
, type_mods
) = split_p4_type(file["type"])
1801 if "x" in type_mods
:
1803 if type_base
== "symlink":
1805 # p4 print on a symlink contains "target\n"; remove the newline
1806 data
= ''.join(contents
)
1807 contents
= [data
[:-1]]
1809 if type_base
== "utf16":
1810 # p4 delivers different text in the python output to -G
1811 # than it does when using "print -o", or normal p4 client
1812 # operations. utf16 is converted to ascii or utf8, perhaps.
1813 # But ascii text saved as -t utf16 is completely mangled.
1814 # Invoke print -o to get the real contents.
1815 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1818 if type_base
== "apple":
1819 # Apple filetype files will be streamed as a concatenation of
1820 # its appledouble header and the contents. This is useless
1821 # on both macs and non-macs. If using "print -q -o xx", it
1822 # will create "xx" with the data, and "%xx" with the header.
1823 # This is also not very useful.
1825 # Ideally, someday, this script can learn how to generate
1826 # appledouble files directly and import those to git, but
1827 # non-mac machines can never find a use for apple filetype.
1828 print "\nIgnoring apple filetype file %s" % file['depotFile']
1831 # Perhaps windows wants unicode, utf16 newlines translated too;
1832 # but this is not doing it.
1833 if self
.isWindows
and type_base
== "text":
1835 for data
in contents
:
1836 data
= data
.replace("\r\n", "\n")
1837 mangled
.append(data
)
1840 # Note that we do not try to de-mangle keywords on utf16 files,
1841 # even though in theory somebody may want that.
1842 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1844 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1845 text
= ''.join(contents
)
1846 text
= regexp
.sub(r
'$\1$', text
)
1849 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1854 length
= length
+ len(d
)
1856 self
.gitStream
.write("data %d\n" % length
)
1858 self
.gitStream
.write(d
)
1859 self
.gitStream
.write("\n")
1861 def streamOneP4Deletion(self
, file):
1862 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1864 sys
.stderr
.write("delete %s\n" % relPath
)
1865 self
.gitStream
.write("D %s\n" % relPath
)
1867 # handle another chunk of streaming data
1868 def streamP4FilesCb(self
, marshalled
):
1870 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1871 # start of a new file - output the old one first
1872 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1873 self
.stream_file
= {}
1874 self
.stream_contents
= []
1875 self
.stream_have_file_info
= False
1877 # pick up the new file information... for the
1878 # 'data' field we need to append to our array
1879 for k
in marshalled
.keys():
1881 self
.stream_contents
.append(marshalled
['data'])
1883 self
.stream_file
[k
] = marshalled
[k
]
1885 self
.stream_have_file_info
= True
1887 # Stream directly from "p4 files" into "git fast-import"
1888 def streamP4Files(self
, files
):
1894 # if using a client spec, only add the files that have
1895 # a path in the client
1896 if self
.clientSpecDirs
:
1897 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
1900 filesForCommit
.append(f
)
1901 if f
['action'] in self
.delete_actions
:
1902 filesToDelete
.append(f
)
1904 filesToRead
.append(f
)
1907 for f
in filesToDelete
:
1908 self
.streamOneP4Deletion(f
)
1910 if len(filesToRead
) > 0:
1911 self
.stream_file
= {}
1912 self
.stream_contents
= []
1913 self
.stream_have_file_info
= False
1915 # curry self argument
1916 def streamP4FilesCbSelf(entry
):
1917 self
.streamP4FilesCb(entry
)
1919 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
1921 p4CmdList(["-x", "-", "print"],
1923 cb
=streamP4FilesCbSelf
)
1926 if self
.stream_file
.has_key('depotFile'):
1927 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1929 def make_email(self
, userid
):
1930 if userid
in self
.users
:
1931 return self
.users
[userid
]
1933 return "%s <a@b>" % userid
1936 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
1938 print "writing tag %s for commit %s" % (labelName
, commit
)
1939 gitStream
.write("tag %s\n" % labelName
)
1940 gitStream
.write("from %s\n" % commit
)
1942 if labelDetails
.has_key('Owner'):
1943 owner
= labelDetails
["Owner"]
1947 # Try to use the owner of the p4 label, or failing that,
1948 # the current p4 user id.
1950 email
= self
.make_email(owner
)
1952 email
= self
.make_email(self
.p4UserId())
1953 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
1955 gitStream
.write("tagger %s\n" % tagger
)
1957 print "labelDetails=",labelDetails
1958 if labelDetails
.has_key('Description'):
1959 description
= labelDetails
['Description']
1961 description
= 'Label from git p4'
1963 gitStream
.write("data %d\n" % len(description
))
1964 gitStream
.write(description
)
1965 gitStream
.write("\n")
1967 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
1968 epoch
= details
["time"]
1969 author
= details
["user"]
1970 self
.branchPrefixes
= branchPrefixes
1973 print "commit into %s" % branch
1975 # start with reading files; if that fails, we should not
1979 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
1980 new_files
.append (f
)
1982 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
1984 self
.gitStream
.write("commit %s\n" % branch
)
1985 # gitStream.write("mark :%s\n" % details["change"])
1986 self
.committedChanges
.add(int(details
["change"]))
1988 if author
not in self
.users
:
1989 self
.getUserMapFromPerforceServer()
1990 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
1992 self
.gitStream
.write("committer %s\n" % committer
)
1994 self
.gitStream
.write("data <<EOT\n")
1995 self
.gitStream
.write(details
["desc"])
1996 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1997 % (','.join (branchPrefixes
), details
["change"]))
1998 if len(details
['options']) > 0:
1999 self
.gitStream
.write(": options = %s" % details
['options'])
2000 self
.gitStream
.write("]\nEOT\n\n")
2004 print "parent %s" % parent
2005 self
.gitStream
.write("from %s\n" % parent
)
2007 self
.streamP4Files(new_files
)
2008 self
.gitStream
.write("\n")
2010 change
= int(details
["change"])
2012 if self
.labels
.has_key(change
):
2013 label
= self
.labels
[change
]
2014 labelDetails
= label
[0]
2015 labelRevisions
= label
[1]
2017 print "Change %s is labelled %s" % (change
, labelDetails
)
2019 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2020 for p
in branchPrefixes
])
2022 if len(files
) == len(labelRevisions
):
2026 if info
["action"] in self
.delete_actions
:
2028 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2030 if cleanedFiles
== labelRevisions
:
2031 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2035 print ("Tag %s does not match with change %s: files do not match."
2036 % (labelDetails
["label"], change
))
2040 print ("Tag %s does not match with change %s: file count is different."
2041 % (labelDetails
["label"], change
))
2043 # Build a dictionary of changelists and labels, for "detect-labels" option.
2044 def getLabels(self
):
2047 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2048 if len(l
) > 0 and not self
.silent
:
2049 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2052 label
= output
["label"]
2056 print "Querying files for label %s" % label
2057 for file in p4CmdList(["files"] +
2058 ["%s...@%s" % (p
, label
)
2059 for p
in self
.depotPaths
]):
2060 revisions
[file["depotFile"]] = file["rev"]
2061 change
= int(file["change"])
2062 if change
> newestChange
:
2063 newestChange
= change
2065 self
.labels
[newestChange
] = [output
, revisions
]
2068 print "Label changes: %s" % self
.labels
.keys()
2070 # Import p4 labels as git tags. A direct mapping does not
2071 # exist, so assume that if all the files are at the same revision
2072 # then we can use that, or it's something more complicated we should
2074 def importP4Labels(self
, stream
, p4Labels
):
2076 print "import p4 labels: " + ' '.join(p4Labels
)
2078 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2079 validLabelRegexp
= gitConfig("git-p4.validLabelRegexp")
2080 if len(validLabelRegexp
) == 0:
2081 validLabelRegexp
= defaultLabelRegexp
2082 m
= re
.compile(validLabelRegexp
)
2084 for name
in p4Labels
:
2087 if not m
.match(name
):
2089 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2092 if name
in ignoredP4Labels
:
2095 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2097 # get the most recent changelist for each file in this label
2098 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2099 for p
in self
.depotPaths
])
2101 if change
.has_key('change'):
2102 # find the corresponding git commit; take the oldest commit
2103 changelist
= int(change
['change'])
2104 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2105 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2106 if len(gitCommit
) == 0:
2107 print "could not find git commit for changelist %d" % changelist
2109 gitCommit
= gitCommit
.strip()
2111 # Convert from p4 time format
2113 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2115 print "Could not convert label time %s" % labelDetail
['Update']
2118 when
= int(time
.mktime(tmwhen
))
2119 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2121 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2124 print "Label %s has no changelists - possibly deleted?" % name
2127 # We can't import this label; don't try again as it will get very
2128 # expensive repeatedly fetching all the files for labels that will
2129 # never be imported. If the label is moved in the future, the
2130 # ignore will need to be removed manually.
2131 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2133 def guessProjectName(self
):
2134 for p
in self
.depotPaths
:
2137 p
= p
[p
.strip().rfind("/") + 1:]
2138 if not p
.endswith("/"):
2142 def getBranchMapping(self
):
2143 lostAndFoundBranches
= set()
2145 user
= gitConfig("git-p4.branchUser")
2147 command
= "branches -u %s" % user
2149 command
= "branches"
2151 for info
in p4CmdList(command
):
2152 details
= p4Cmd(["branch", "-o", info
["branch"]])
2154 while details
.has_key("View%s" % viewIdx
):
2155 paths
= details
["View%s" % viewIdx
].split(" ")
2156 viewIdx
= viewIdx
+ 1
2157 # require standard //depot/foo/... //depot/bar/... mapping
2158 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2161 destination
= paths
[1]
2163 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2164 source
= source
[len(self
.depotPaths
[0]):-4]
2165 destination
= destination
[len(self
.depotPaths
[0]):-4]
2167 if destination
in self
.knownBranches
:
2169 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2170 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2173 self
.knownBranches
[destination
] = source
2175 lostAndFoundBranches
.discard(destination
)
2177 if source
not in self
.knownBranches
:
2178 lostAndFoundBranches
.add(source
)
2180 # Perforce does not strictly require branches to be defined, so we also
2181 # check git config for a branch list.
2183 # Example of branch definition in git config file:
2185 # branchList=main:branchA
2186 # branchList=main:branchB
2187 # branchList=branchA:branchC
2188 configBranches
= gitConfigList("git-p4.branchList")
2189 for branch
in configBranches
:
2191 (source
, destination
) = branch
.split(":")
2192 self
.knownBranches
[destination
] = source
2194 lostAndFoundBranches
.discard(destination
)
2196 if source
not in self
.knownBranches
:
2197 lostAndFoundBranches
.add(source
)
2200 for branch
in lostAndFoundBranches
:
2201 self
.knownBranches
[branch
] = branch
2203 def getBranchMappingFromGitBranches(self
):
2204 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2205 for branch
in branches
.keys():
2206 if branch
== "master":
2209 branch
= branch
[len(self
.projectName
):]
2210 self
.knownBranches
[branch
] = branch
2212 def listExistingP4GitBranches(self
):
2213 # branches holds mapping from name to commit
2214 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2215 self
.p4BranchesInGit
= branches
.keys()
2216 for branch
in branches
.keys():
2217 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2219 def updateOptionDict(self
, d
):
2221 if self
.keepRepoPath
:
2222 option_keys
['keepRepoPath'] = 1
2224 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2226 def readOptions(self
, d
):
2227 self
.keepRepoPath
= (d
.has_key('options')
2228 and ('keepRepoPath' in d
['options']))
2230 def gitRefForBranch(self
, branch
):
2231 if branch
== "main":
2232 return self
.refPrefix
+ "master"
2234 if len(branch
) <= 0:
2237 return self
.refPrefix
+ self
.projectName
+ branch
2239 def gitCommitByP4Change(self
, ref
, change
):
2241 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2244 latestCommit
= parseRevision(ref
)
2248 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2249 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2254 log
= extractLogMessageFromGitCommit(next
)
2255 settings
= extractSettingsGitLog(log
)
2256 currentChange
= int(settings
['change'])
2258 print "current change %s" % currentChange
2260 if currentChange
== change
:
2262 print "found %s" % next
2265 if currentChange
< change
:
2266 earliestCommit
= "^%s" % next
2268 latestCommit
= "%s" % next
2272 def importNewBranch(self
, branch
, maxChange
):
2273 # make fast-import flush all changes to disk and update the refs using the checkpoint
2274 # command so that we can try to find the branch parent in the git history
2275 self
.gitStream
.write("checkpoint\n\n");
2276 self
.gitStream
.flush();
2277 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2278 range = "@1,%s" % maxChange
2279 #print "prefix" + branchPrefix
2280 changes
= p4ChangesForPaths([branchPrefix
], range)
2281 if len(changes
) <= 0:
2283 firstChange
= changes
[0]
2284 #print "first change in branch: %s" % firstChange
2285 sourceBranch
= self
.knownBranches
[branch
]
2286 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2287 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2288 #print "source " + sourceBranch
2290 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2291 #print "branch parent: %s" % branchParentChange
2292 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2293 if len(gitParent
) > 0:
2294 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2295 #print "parent git commit: %s" % gitParent
2297 self
.importChanges(changes
)
2300 def searchParent(self
, parent
, branch
, target
):
2302 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2304 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2307 print "Found parent of %s in commit %s" % (branch
, blob
)
2314 def importChanges(self
, changes
):
2316 for change
in changes
:
2317 description
= p4Cmd(["describe", str(change
)])
2318 self
.updateOptionDict(description
)
2321 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2326 if self
.detectBranches
:
2327 branches
= self
.splitFilesIntoBranches(description
)
2328 for branch
in branches
.keys():
2330 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2334 filesForCommit
= branches
[branch
]
2337 print "branch is %s" % branch
2339 self
.updatedBranches
.add(branch
)
2341 if branch
not in self
.createdBranches
:
2342 self
.createdBranches
.add(branch
)
2343 parent
= self
.knownBranches
[branch
]
2344 if parent
== branch
:
2347 fullBranch
= self
.projectName
+ branch
2348 if fullBranch
not in self
.p4BranchesInGit
:
2350 print("\n Importing new branch %s" % fullBranch
);
2351 if self
.importNewBranch(branch
, change
- 1):
2353 self
.p4BranchesInGit
.append(fullBranch
)
2355 print("\n Resuming with change %s" % change
);
2358 print "parent determined through known branches: %s" % parent
2360 branch
= self
.gitRefForBranch(branch
)
2361 parent
= self
.gitRefForBranch(parent
)
2364 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2366 if len(parent
) == 0 and branch
in self
.initialParents
:
2367 parent
= self
.initialParents
[branch
]
2368 del self
.initialParents
[branch
]
2372 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2374 print "Creating temporary branch: " + tempBranch
2375 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2376 self
.tempBranches
.append(tempBranch
)
2378 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2380 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2383 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2384 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2386 files
= self
.extractFilesFromCommit(description
)
2387 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2389 self
.initialParent
= ""
2391 print self
.gitError
.read()
2394 def importHeadRevision(self
, revision
):
2395 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2398 details
["user"] = "git perforce import user"
2399 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2400 % (' '.join(self
.depotPaths
), revision
))
2401 details
["change"] = revision
2405 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2407 for info
in p4CmdList(["files"] + fileArgs
):
2409 if 'code' in info
and info
['code'] == 'error':
2410 sys
.stderr
.write("p4 returned an error: %s\n"
2412 if info
['data'].find("must refer to client") >= 0:
2413 sys
.stderr
.write("This particular p4 error is misleading.\n")
2414 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2415 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2417 if 'p4ExitCode' in info
:
2418 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2422 change
= int(info
["change"])
2423 if change
> newestRevision
:
2424 newestRevision
= change
2426 if info
["action"] in self
.delete_actions
:
2427 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2428 #fileCnt = fileCnt + 1
2431 for prop
in ["depotFile", "rev", "action", "type" ]:
2432 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2434 fileCnt
= fileCnt
+ 1
2436 details
["change"] = newestRevision
2438 # Use time from top-most change so that all git p4 clones of
2439 # the same p4 repo have the same commit SHA1s.
2440 res
= p4CmdList("describe -s %d" % newestRevision
)
2443 if r
.has_key('time'):
2444 newestTime
= int(r
['time'])
2445 if newestTime
is None:
2446 die("\"describe -s\" on newest change %d did not give a time")
2447 details
["time"] = newestTime
2449 self
.updateOptionDict(details
)
2451 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2453 print "IO error with git fast-import. Is your git version recent enough?"
2454 print self
.gitError
.read()
2457 def run(self
, args
):
2458 self
.depotPaths
= []
2459 self
.changeRange
= ""
2460 self
.initialParent
= ""
2461 self
.previousDepotPaths
= []
2463 # map from branch depot path to parent branch
2464 self
.knownBranches
= {}
2465 self
.initialParents
= {}
2466 self
.hasOrigin
= originP4BranchesExist()
2467 if not self
.syncWithOrigin
:
2468 self
.hasOrigin
= False
2470 if self
.importIntoRemotes
:
2471 self
.refPrefix
= "refs/remotes/p4/"
2473 self
.refPrefix
= "refs/heads/p4/"
2475 if self
.syncWithOrigin
and self
.hasOrigin
:
2477 print "Syncing with origin first by calling git fetch origin"
2478 system("git fetch origin")
2480 if len(self
.branch
) == 0:
2481 self
.branch
= self
.refPrefix
+ "master"
2482 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2483 system("git update-ref %s refs/heads/p4" % self
.branch
)
2484 system("git branch -D p4");
2485 # create it /after/ importing, when master exists
2486 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2487 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2489 # accept either the command-line option, or the configuration variable
2490 if self
.useClientSpec
:
2491 # will use this after clone to set the variable
2492 self
.useClientSpec_from_options
= True
2494 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2495 self
.useClientSpec
= True
2496 if self
.useClientSpec
:
2497 self
.clientSpecDirs
= getClientSpec()
2499 # TODO: should always look at previous commits,
2500 # merge with previous imports, if possible.
2503 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2504 self
.listExistingP4GitBranches()
2506 if len(self
.p4BranchesInGit
) > 1:
2508 print "Importing from/into multiple branches"
2509 self
.detectBranches
= True
2512 print "branches: %s" % self
.p4BranchesInGit
2515 for branch
in self
.p4BranchesInGit
:
2516 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2518 settings
= extractSettingsGitLog(logMsg
)
2520 self
.readOptions(settings
)
2521 if (settings
.has_key('depot-paths')
2522 and settings
.has_key ('change')):
2523 change
= int(settings
['change']) + 1
2524 p4Change
= max(p4Change
, change
)
2526 depotPaths
= sorted(settings
['depot-paths'])
2527 if self
.previousDepotPaths
== []:
2528 self
.previousDepotPaths
= depotPaths
2531 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2532 prev_list
= prev
.split("/")
2533 cur_list
= cur
.split("/")
2534 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2535 if cur_list
[i
] <> prev_list
[i
]:
2539 paths
.append ("/".join(cur_list
[:i
+ 1]))
2541 self
.previousDepotPaths
= paths
2544 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2545 self
.changeRange
= "@%s,#head" % p4Change
2546 if not self
.detectBranches
:
2547 self
.initialParent
= parseRevision(self
.branch
)
2548 if not self
.silent
and not self
.detectBranches
:
2549 print "Performing incremental import into %s git branch" % self
.branch
2551 if not self
.branch
.startswith("refs/"):
2552 self
.branch
= "refs/heads/" + self
.branch
2554 if len(args
) == 0 and self
.depotPaths
:
2556 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2558 if self
.depotPaths
and self
.depotPaths
!= args
:
2559 print ("previous import used depot path %s and now %s was specified. "
2560 "This doesn't work!" % (' '.join (self
.depotPaths
),
2564 self
.depotPaths
= sorted(args
)
2569 # Make sure no revision specifiers are used when --changesfile
2571 bad_changesfile
= False
2572 if len(self
.changesFile
) > 0:
2573 for p
in self
.depotPaths
:
2574 if p
.find("@") >= 0 or p
.find("#") >= 0:
2575 bad_changesfile
= True
2578 die("Option --changesfile is incompatible with revision specifiers")
2581 for p
in self
.depotPaths
:
2582 if p
.find("@") != -1:
2583 atIdx
= p
.index("@")
2584 self
.changeRange
= p
[atIdx
:]
2585 if self
.changeRange
== "@all":
2586 self
.changeRange
= ""
2587 elif ',' not in self
.changeRange
:
2588 revision
= self
.changeRange
2589 self
.changeRange
= ""
2591 elif p
.find("#") != -1:
2592 hashIdx
= p
.index("#")
2593 revision
= p
[hashIdx
:]
2595 elif self
.previousDepotPaths
== []:
2596 # pay attention to changesfile, if given, else import
2597 # the entire p4 tree at the head revision
2598 if len(self
.changesFile
) == 0:
2601 p
= re
.sub ("\.\.\.$", "", p
)
2602 if not p
.endswith("/"):
2607 self
.depotPaths
= newPaths
2609 self
.loadUserMapFromCache()
2611 if self
.detectLabels
:
2614 if self
.detectBranches
:
2615 ## FIXME - what's a P4 projectName ?
2616 self
.projectName
= self
.guessProjectName()
2619 self
.getBranchMappingFromGitBranches()
2621 self
.getBranchMapping()
2623 print "p4-git branches: %s" % self
.p4BranchesInGit
2624 print "initial parents: %s" % self
.initialParents
2625 for b
in self
.p4BranchesInGit
:
2629 b
= b
[len(self
.projectName
):]
2630 self
.createdBranches
.add(b
)
2632 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2634 importProcess
= subprocess
.Popen(["git", "fast-import"],
2635 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2636 stderr
=subprocess
.PIPE
);
2637 self
.gitOutput
= importProcess
.stdout
2638 self
.gitStream
= importProcess
.stdin
2639 self
.gitError
= importProcess
.stderr
2642 self
.importHeadRevision(revision
)
2646 if len(self
.changesFile
) > 0:
2647 output
= open(self
.changesFile
).readlines()
2650 changeSet
.add(int(line
))
2652 for change
in changeSet
:
2653 changes
.append(change
)
2657 # catch "git p4 sync" with no new branches, in a repo that
2658 # does not have any existing p4 branches
2659 if len(args
) == 0 and not self
.p4BranchesInGit
:
2660 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2662 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2664 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2666 if len(self
.maxChanges
) > 0:
2667 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2669 if len(changes
) == 0:
2671 print "No changes to import!"
2673 if not self
.silent
and not self
.detectBranches
:
2674 print "Import destination: %s" % self
.branch
2676 self
.updatedBranches
= set()
2678 self
.importChanges(changes
)
2682 if len(self
.updatedBranches
) > 0:
2683 sys
.stdout
.write("Updated branches: ")
2684 for b
in self
.updatedBranches
:
2685 sys
.stdout
.write("%s " % b
)
2686 sys
.stdout
.write("\n")
2688 if gitConfig("git-p4.importLabels", "--bool") == "true":
2689 self
.importLabels
= true
2691 if self
.importLabels
:
2692 p4Labels
= getP4Labels(self
.depotPaths
)
2693 gitTags
= getGitTags()
2695 missingP4Labels
= p4Labels
- gitTags
2696 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2698 self
.gitStream
.close()
2699 if importProcess
.wait() != 0:
2700 die("fast-import failed: %s" % self
.gitError
.read())
2701 self
.gitOutput
.close()
2702 self
.gitError
.close()
2704 # Cleanup temporary branches created during import
2705 if self
.tempBranches
!= []:
2706 for branch
in self
.tempBranches
:
2707 read_pipe("git update-ref -d %s" % branch
)
2708 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2712 class P4Rebase(Command
):
2714 Command
.__init
__(self
)
2716 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2717 optparse
.make_option("--verbose", dest
="verbose", action
="store_true"),
2719 self
.verbose
= False
2720 self
.importLabels
= False
2721 self
.description
= ("Fetches the latest revision from perforce and "
2722 + "rebases the current work (branch) against it")
2724 def run(self
, args
):
2726 sync
.importLabels
= self
.importLabels
2729 return self
.rebase()
2732 if os
.system("git update-index --refresh") != 0:
2733 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2734 if len(read_pipe("git diff-index HEAD --")) > 0:
2735 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2737 [upstream
, settings
] = findUpstreamBranchPoint()
2738 if len(upstream
) == 0:
2739 die("Cannot find upstream branchpoint for rebase")
2741 # the branchpoint may be p4/foo~3, so strip off the parent
2742 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2744 print "Rebasing the current branch onto %s" % upstream
2745 oldHead
= read_pipe("git rev-parse HEAD").strip()
2746 system("git rebase %s" % upstream
)
2747 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2750 class P4Clone(P4Sync
):
2752 P4Sync
.__init
__(self
)
2753 self
.description
= "Creates a new git repository and imports from Perforce into it"
2754 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2756 optparse
.make_option("--destination", dest
="cloneDestination",
2757 action
='store', default
=None,
2758 help="where to leave result of the clone"),
2759 optparse
.make_option("-/", dest
="cloneExclude",
2760 action
="append", type="string",
2761 help="exclude depot path"),
2762 optparse
.make_option("--bare", dest
="cloneBare",
2763 action
="store_true", default
=False),
2765 self
.cloneDestination
= None
2766 self
.needsGit
= False
2767 self
.cloneBare
= False
2769 # This is required for the "append" cloneExclude action
2770 def ensure_value(self
, attr
, value
):
2771 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2772 setattr(self
, attr
, value
)
2773 return getattr(self
, attr
)
2775 def defaultDestination(self
, args
):
2776 ## TODO: use common prefix of args?
2778 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2779 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2780 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2781 depotDir
= re
.sub(r
"/$", "", depotDir
)
2782 return os
.path
.split(depotDir
)[1]
2784 def run(self
, args
):
2788 if self
.keepRepoPath
and not self
.cloneDestination
:
2789 sys
.stderr
.write("Must specify destination for --keep-path\n")
2794 if not self
.cloneDestination
and len(depotPaths
) > 1:
2795 self
.cloneDestination
= depotPaths
[-1]
2796 depotPaths
= depotPaths
[:-1]
2798 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2799 for p
in depotPaths
:
2800 if not p
.startswith("//"):
2803 if not self
.cloneDestination
:
2804 self
.cloneDestination
= self
.defaultDestination(args
)
2806 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2808 if not os
.path
.exists(self
.cloneDestination
):
2809 os
.makedirs(self
.cloneDestination
)
2810 chdir(self
.cloneDestination
)
2812 init_cmd
= [ "git", "init" ]
2814 init_cmd
.append("--bare")
2815 subprocess
.check_call(init_cmd
)
2817 if not P4Sync
.run(self
, depotPaths
):
2819 if self
.branch
!= "master":
2820 if self
.importIntoRemotes
:
2821 masterbranch
= "refs/remotes/p4/master"
2823 masterbranch
= "refs/heads/p4/master"
2824 if gitBranchExists(masterbranch
):
2825 system("git branch master %s" % masterbranch
)
2826 if not self
.cloneBare
:
2827 system("git checkout -f")
2829 print "Could not detect main branch. No checkout/master branch created."
2831 # auto-set this variable if invoked with --use-client-spec
2832 if self
.useClientSpec_from_options
:
2833 system("git config --bool git-p4.useclientspec true")
2837 class P4Branches(Command
):
2839 Command
.__init
__(self
)
2841 self
.description
= ("Shows the git branches that hold imports and their "
2842 + "corresponding perforce depot paths")
2843 self
.verbose
= False
2845 def run(self
, args
):
2846 if originP4BranchesExist():
2847 createOrUpdateBranchesFromOrigin()
2849 cmdline
= "git rev-parse --symbolic "
2850 cmdline
+= " --remotes"
2852 for line
in read_pipe_lines(cmdline
):
2855 if not line
.startswith('p4/') or line
== "p4/HEAD":
2859 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2860 settings
= extractSettingsGitLog(log
)
2862 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2865 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2867 optparse
.IndentedHelpFormatter
.__init
__(self
)
2869 def format_description(self
, description
):
2871 return description
+ "\n"
2875 def printUsage(commands
):
2876 print "usage: %s <command> [options]" % sys
.argv
[0]
2878 print "valid commands: %s" % ", ".join(commands
)
2880 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2885 "submit" : P4Submit
,
2886 "commit" : P4Submit
,
2888 "rebase" : P4Rebase
,
2890 "rollback" : P4RollBack
,
2891 "branches" : P4Branches
2896 if len(sys
.argv
[1:]) == 0:
2897 printUsage(commands
.keys())
2901 cmdName
= sys
.argv
[1]
2903 klass
= commands
[cmdName
]
2906 print "unknown command %s" % cmdName
2908 printUsage(commands
.keys())
2911 options
= cmd
.options
2912 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2916 if len(options
) > 0:
2918 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2920 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2922 description
= cmd
.description
,
2923 formatter
= HelpFormatter())
2925 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2927 verbose
= cmd
.verbose
2929 if cmd
.gitdir
== None:
2930 cmd
.gitdir
= os
.path
.abspath(".git")
2931 if not isValidGitDir(cmd
.gitdir
):
2932 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2933 if os
.path
.exists(cmd
.gitdir
):
2934 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2938 if not isValidGitDir(cmd
.gitdir
):
2939 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2940 cmd
.gitdir
+= "/.git"
2942 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2944 os
.environ
["GIT_DIR"] = cmd
.gitdir
2946 if not cmd
.run(args
):
2951 if __name__
== '__main__':