3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
123 def p4_has_command(cmd
):
124 """Ask p4 for help on this command. If it returns an error, the
125 command does not exist in this version of p4."""
126 real_cmd
= p4_build_cmd(["help", cmd
])
127 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
128 stderr
=subprocess
.PIPE
)
130 return p
.returncode
== 0
133 expand
= isinstance(cmd
,basestring
)
135 sys
.stderr
.write("executing %s\n" % str(cmd
))
136 subprocess
.check_call(cmd
, shell
=expand
)
139 """Specifically invoke p4 as the system command. """
140 real_cmd
= p4_build_cmd(cmd
)
141 expand
= isinstance(real_cmd
, basestring
)
142 subprocess
.check_call(real_cmd
, shell
=expand
)
144 def p4_integrate(src
, dest
):
145 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
147 def p4_sync(f
, *options
):
148 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
151 # forcibly add file names with wildcards
152 if wildcard_present(f
):
153 p4_system(["add", "-f", f
])
155 p4_system(["add", f
])
158 p4_system(["delete", wildcard_encode(f
)])
161 p4_system(["edit", wildcard_encode(f
)])
164 p4_system(["revert", wildcard_encode(f
)])
166 def p4_reopen(type, f
):
167 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
169 def p4_move(src
, dest
):
170 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers. See "p4 help filetypes"
175 # for a list and explanation.
177 def split_p4_type(p4type
):
179 p4_filetypes_historical
= {
180 "ctempobj": "binary+Sw",
186 "tempobj": "binary+FSw",
187 "ubinary": "binary+F",
188 "uresource": "resource+F",
189 "uxbinary": "binary+Fx",
190 "xbinary": "binary+x",
192 "xtempobj": "binary+Swx",
194 "xunicode": "unicode+x",
197 if p4type
in p4_filetypes_historical
:
198 p4type
= p4_filetypes_historical
[p4type
]
200 s
= p4type
.split("+")
208 # return the raw p4 type of a file (text, text+ko, etc)
211 results
= p4CmdList(["fstat", "-T", "headType", file])
212 return results
[0]['headType']
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
218 def p4_keywords_regexp_for_type(base
, type_mods
):
219 if base
in ("text", "unicode", "binary"):
221 if "ko" in type_mods
:
223 elif "k" in type_mods
:
224 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
228 \$ # Starts with a dollar, followed by...
229 (%s) # one of the keywords, followed by...
230 (:[^$]+)? # possibly an old expansion, followed by...
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
242 def p4_keywords_regexp_for_file(file):
243 if not os
.path
.exists(file):
246 (type_base
, type_mods
) = split_p4_type(p4_type(file))
247 return p4_keywords_regexp_for_type(type_base
, type_mods
)
249 def setP4ExecBit(file, mode
):
250 # Reopens an already open file and changes the execute bit to match
251 # the execute bit setting in the passed in mode.
255 if not isModeExec(mode
):
256 p4Type
= getP4OpenedType(file)
257 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
258 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
259 if p4Type
[-1] == "+":
260 p4Type
= p4Type
[0:-1]
262 p4_reopen(p4Type
, file)
264 def getP4OpenedType(file):
265 # Returns the perforce file type for the given file.
267 result
= p4_read_pipe(["opened", wildcard_encode(file)])
268 match
= re
.match(".*\((.+)\)\r?$", result
)
270 return match
.group(1)
272 die("Could not determine file type for %s (result: '%s')" % (file, result
))
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths
):
277 if isinstance(depotPaths
,basestring
):
278 depotPaths
= [depotPaths
]
280 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
286 # Return the set of all git tags
289 for line
in read_pipe_lines(["git", "tag"]):
294 def diffTreePattern():
295 # This is a simple generator for the diff tree regex pattern. This could be
296 # a class variable if this and parseDiffTreeEntry were a part of a class.
297 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
301 def parseDiffTreeEntry(entry
):
302 """Parses a single diff tree entry into its component elements.
304 See git-diff-tree(1) manpage for details about the format of the diff
305 output. This method returns a dictionary with the following elements:
307 src_mode - The mode of the source file
308 dst_mode - The mode of the destination file
309 src_sha1 - The sha1 for the source file
310 dst_sha1 - The sha1 fr the destination file
311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312 status_score - The score for the status (applicable for 'C' and 'R'
313 statuses). This is None if there is no score.
314 src - The path for the source file.
315 dst - The path for the destination file. This is only present for
316 copy or renames. If it is not present, this is None.
318 If the pattern is not matched, None is returned."""
320 match
= diffTreePattern().next().match(entry
)
323 'src_mode': match
.group(1),
324 'dst_mode': match
.group(2),
325 'src_sha1': match
.group(3),
326 'dst_sha1': match
.group(4),
327 'status': match
.group(5),
328 'status_score': match
.group(6),
329 'src': match
.group(7),
330 'dst': match
.group(10)
334 def isModeExec(mode
):
335 # Returns True if the given git mode represents an executable file,
337 return mode
[-3:] == "755"
339 def isModeExecChanged(src_mode
, dst_mode
):
340 return isModeExec(src_mode
) != isModeExec(dst_mode
)
342 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
344 if isinstance(cmd
,basestring
):
351 cmd
= p4_build_cmd(cmd
)
353 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
355 # Use a temporary file to avoid deadlocks without
356 # subprocess.communicate(), which would put another copy
357 # of stdout into memory.
359 if stdin
is not None:
360 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
361 if isinstance(stdin
,basestring
):
362 stdin_file
.write(stdin
)
365 stdin_file
.write(i
+ '\n')
369 p4
= subprocess
.Popen(cmd
,
372 stdout
=subprocess
.PIPE
)
377 entry
= marshal
.load(p4
.stdout
)
387 entry
["p4ExitCode"] = exitCode
393 list = p4CmdList(cmd
)
399 def p4Where(depotPath
):
400 if not depotPath
.endswith("/"):
402 depotPath
= depotPath
+ "..."
403 outputList
= p4CmdList(["where", depotPath
])
405 for entry
in outputList
:
406 if "depotFile" in entry
:
407 if entry
["depotFile"] == depotPath
:
410 elif "data" in entry
:
411 data
= entry
.get("data")
412 space
= data
.find(" ")
413 if data
[:space
] == depotPath
:
418 if output
["code"] == "error":
422 clientPath
= output
.get("path")
423 elif "data" in output
:
424 data
= output
.get("data")
425 lastSpace
= data
.rfind(" ")
426 clientPath
= data
[lastSpace
+ 1:]
428 if clientPath
.endswith("..."):
429 clientPath
= clientPath
[:-3]
432 def currentGitBranch():
433 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
435 def isValidGitDir(path
):
436 if (os
.path
.exists(path
+ "/HEAD")
437 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
441 def parseRevision(ref
):
442 return read_pipe("git rev-parse %s" % ref
).strip()
444 def branchExists(ref
):
445 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
449 def extractLogMessageFromGitCommit(commit
):
452 ## fixme: title is first line of commit, not 1st paragraph.
454 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
463 def extractSettingsGitLog(log
):
465 for line
in log
.split("\n"):
467 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
471 assignments
= m
.group(1).split (':')
472 for a
in assignments
:
474 key
= vals
[0].strip()
475 val
= ('='.join (vals
[1:])).strip()
476 if val
.endswith ('\"') and val
.startswith('"'):
481 paths
= values
.get("depot-paths")
483 paths
= values
.get("depot-path")
485 values
['depot-paths'] = paths
.split(',')
488 def gitBranchExists(branch
):
489 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
490 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
491 return proc
.wait() == 0;
494 def gitConfig(key
, args
= None): # set args to "--bool", for instance
495 if not _gitConfig
.has_key(key
):
498 argsFilter
= "%s " % args
499 cmd
= "git config %s%s" % (argsFilter
, key
)
500 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
501 return _gitConfig
[key
]
503 def gitConfigList(key
):
504 if not _gitConfig
.has_key(key
):
505 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
506 return _gitConfig
[key
]
508 def p4BranchesInGit(branchesAreInRemotes
= True):
511 cmdline
= "git rev-parse --symbolic "
512 if branchesAreInRemotes
:
513 cmdline
+= " --remotes"
515 cmdline
+= " --branches"
517 for line
in read_pipe_lines(cmdline
):
520 ## only import to p4/
521 if not line
.startswith('p4/') or line
== "p4/HEAD":
526 branch
= re
.sub ("^p4/", "", line
)
528 branches
[branch
] = parseRevision(line
)
531 def findUpstreamBranchPoint(head
= "HEAD"):
532 branches
= p4BranchesInGit()
533 # map from depot-path to branch name
534 branchByDepotPath
= {}
535 for branch
in branches
.keys():
536 tip
= branches
[branch
]
537 log
= extractLogMessageFromGitCommit(tip
)
538 settings
= extractSettingsGitLog(log
)
539 if settings
.has_key("depot-paths"):
540 paths
= ",".join(settings
["depot-paths"])
541 branchByDepotPath
[paths
] = "remotes/p4/" + branch
545 while parent
< 65535:
546 commit
= head
+ "~%s" % parent
547 log
= extractLogMessageFromGitCommit(commit
)
548 settings
= extractSettingsGitLog(log
)
549 if settings
.has_key("depot-paths"):
550 paths
= ",".join(settings
["depot-paths"])
551 if branchByDepotPath
.has_key(paths
):
552 return [branchByDepotPath
[paths
], settings
]
556 return ["", settings
]
558 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
560 print ("Creating/updating branch(es) in %s based on origin branch(es)"
563 originPrefix
= "origin/p4/"
565 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
567 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
570 headName
= line
[len(originPrefix
):]
571 remoteHead
= localRefPrefix
+ headName
574 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
575 if (not original
.has_key('depot-paths')
576 or not original
.has_key('change')):
580 if not gitBranchExists(remoteHead
):
582 print "creating %s" % remoteHead
585 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
586 if settings
.has_key('change') > 0:
587 if settings
['depot-paths'] == original
['depot-paths']:
588 originP4Change
= int(original
['change'])
589 p4Change
= int(settings
['change'])
590 if originP4Change
> p4Change
:
591 print ("%s (%s) is newer than %s (%s). "
592 "Updating p4 branch from origin."
593 % (originHead
, originP4Change
,
594 remoteHead
, p4Change
))
597 print ("Ignoring: %s was imported from %s while "
598 "%s was imported from %s"
599 % (originHead
, ','.join(original
['depot-paths']),
600 remoteHead
, ','.join(settings
['depot-paths'])))
603 system("git update-ref %s %s" % (remoteHead
, originHead
))
605 def originP4BranchesExist():
606 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
608 def p4ChangesForPaths(depotPaths
, changeRange
):
612 cmd
+= ["%s...%s" % (p
, changeRange
)]
613 output
= p4_read_pipe_lines(cmd
)
617 changeNum
= int(line
.split(" ")[1])
618 changes
[changeNum
] = True
620 changelist
= changes
.keys()
624 def p4PathStartsWith(path
, prefix
):
625 # This method tries to remedy a potential mixed-case issue:
627 # If UserA adds //depot/DirA/file1
628 # and UserB adds //depot/dira/file2
630 # we may or may not have a problem. If you have core.ignorecase=true,
631 # we treat DirA and dira as the same directory
632 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
634 return path
.lower().startswith(prefix
.lower())
635 return path
.startswith(prefix
)
638 """Look at the p4 client spec, create a View() object that contains
639 all the mappings, and return it."""
641 specList
= p4CmdList("client -o")
642 if len(specList
) != 1:
643 die('Output from "client -o" is %d lines, expecting 1' %
646 # dictionary of all client parameters
649 # just the keys that start with "View"
650 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
655 # append the lines, in order, to the view
656 for view_num
in range(len(view_keys
)):
657 k
= "View%d" % view_num
658 if k
not in view_keys
:
659 die("Expected view key %s missing" % k
)
660 view
.append(entry
[k
])
665 """Grab the client directory."""
667 output
= p4CmdList("client -o")
669 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
672 if "Root" not in entry
:
673 die('Client has no "Root"')
678 # P4 wildcards are not allowed in filenames. P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
682 def wildcard_decode(path
):
683 # Search for and fix just these four characters. Do % last so
684 # that fixing it does not inadvertently create new %-escapes.
685 # Cannot have * in a filename in windows; untested as to
686 # what p4 would do in such a case.
687 if not platform
.system() == "Windows":
688 path
= path
.replace("%2A", "*")
689 path
= path
.replace("%23", "#") \
690 .replace("%40", "@") \
694 def wildcard_encode(path
):
695 # do % first to avoid double-encoding the %s introduced here
696 path
= path
.replace("%", "%25") \
697 .replace("*", "%2A") \
698 .replace("#", "%23") \
702 def wildcard_present(path
):
703 return path
.translate(None, "*#@%") != path
707 self
.usage
= "usage: %prog [options]"
713 self
.userMapFromPerforceServer
= False
714 self
.myP4UserId
= None
718 return self
.myP4UserId
720 results
= p4CmdList("user -o")
722 if r
.has_key('User'):
723 self
.myP4UserId
= r
['User']
725 die("Could not find your p4 user id")
727 def p4UserIsMe(self
, p4User
):
728 # return True if the given p4 user is actually me
730 if not p4User
or p4User
!= me
:
735 def getUserCacheFilename(self
):
736 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
737 return home
+ "/.gitp4-usercache.txt"
739 def getUserMapFromPerforceServer(self
):
740 if self
.userMapFromPerforceServer
:
745 for output
in p4CmdList("users"):
746 if not output
.has_key("User"):
748 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
749 self
.emails
[output
["Email"]] = output
["User"]
753 for (key
, val
) in self
.users
.items():
754 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
756 open(self
.getUserCacheFilename(), "wb").write(s
)
757 self
.userMapFromPerforceServer
= True
759 def loadUserMapFromCache(self
):
761 self
.userMapFromPerforceServer
= False
763 cache
= open(self
.getUserCacheFilename(), "rb")
764 lines
= cache
.readlines()
767 entry
= line
.strip().split("\t")
768 self
.users
[entry
[0]] = entry
[1]
770 self
.getUserMapFromPerforceServer()
772 class P4Debug(Command
):
774 Command
.__init
__(self
)
776 self
.description
= "A tool to debug the output of p4 -G."
777 self
.needsGit
= False
781 for output
in p4CmdList(args
):
782 print 'Element: %d' % j
787 class P4RollBack(Command
):
789 Command
.__init
__(self
)
791 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
793 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
794 self
.rollbackLocalBranches
= False
799 maxChange
= int(args
[0])
801 if "p4ExitCode" in p4Cmd("changes -m 1"):
802 die("Problems executing p4");
804 if self
.rollbackLocalBranches
:
805 refPrefix
= "refs/heads/"
806 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
808 refPrefix
= "refs/remotes/"
809 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
812 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
814 ref
= refPrefix
+ line
815 log
= extractLogMessageFromGitCommit(ref
)
816 settings
= extractSettingsGitLog(log
)
818 depotPaths
= settings
['depot-paths']
819 change
= settings
['change']
823 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
824 for p
in depotPaths
]))) == 0:
825 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
826 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
829 while change
and int(change
) > maxChange
:
832 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
833 system("git update-ref %s \"%s^\"" % (ref
, ref
))
834 log
= extractLogMessageFromGitCommit(ref
)
835 settings
= extractSettingsGitLog(log
)
838 depotPaths
= settings
['depot-paths']
839 change
= settings
['change']
842 print "%s rewound to %s" % (ref
, change
)
846 class P4Submit(Command
, P4UserMap
):
848 Command
.__init
__(self
)
849 P4UserMap
.__init
__(self
)
851 optparse
.make_option("--origin", dest
="origin"),
852 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
853 # preserve the user, requires relevant p4 permissions
854 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
855 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
857 self
.description
= "Submit changes from git to the perforce depot."
858 self
.usage
+= " [name of git branch to submit into perforce depot]"
859 self
.interactive
= True
861 self
.detectRenames
= False
862 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
863 self
.isWindows
= (platform
.system() == "Windows")
864 self
.exportLabels
= False
865 self
.p4HasMoveCommand
= p4_has_command("move")
868 if len(p4CmdList("opened ...")) > 0:
869 die("You have files opened with perforce! Close them before starting the sync.")
871 # replaces everything between 'Description:' and the next P4 submit template field with the
873 def prepareLogMessage(self
, template
, message
):
876 inDescriptionSection
= False
878 for line
in template
.split("\n"):
879 if line
.startswith("#"):
880 result
+= line
+ "\n"
883 if inDescriptionSection
:
884 if line
.startswith("Files:") or line
.startswith("Jobs:"):
885 inDescriptionSection
= False
889 if line
.startswith("Description:"):
890 inDescriptionSection
= True
892 for messageLine
in message
.split("\n"):
893 line
+= "\t" + messageLine
+ "\n"
895 result
+= line
+ "\n"
899 def patchRCSKeywords(self
, file, pattern
):
900 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
901 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
903 outFile
= os
.fdopen(handle
, "w+")
904 inFile
= open(file, "r")
905 regexp
= re
.compile(pattern
, re
.VERBOSE
)
906 for line
in inFile
.readlines():
907 line
= regexp
.sub(r
'$\1$', line
)
911 # Forcibly overwrite the original file
913 shutil
.move(outFileName
, file)
915 # cleanup our temporary file
916 os
.unlink(outFileName
)
917 print "Failed to strip RCS keywords in %s" % file
920 print "Patched up RCS keywords in %s" % file
922 def p4UserForCommit(self
,id):
923 # Return the tuple (perforce user,git email) for a given git commit id
924 self
.getUserMapFromPerforceServer()
925 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
926 gitEmail
= gitEmail
.strip()
927 if not self
.emails
.has_key(gitEmail
):
928 return (None,gitEmail
)
930 return (self
.emails
[gitEmail
],gitEmail
)
932 def checkValidP4Users(self
,commits
):
933 # check if any git authors cannot be mapped to p4 users
935 (user
,email
) = self
.p4UserForCommit(id)
937 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
938 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
941 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
943 def lastP4Changelist(self
):
944 # Get back the last changelist number submitted in this client spec. This
945 # then gets used to patch up the username in the change. If the same
946 # client spec is being used by multiple processes then this might go
948 results
= p4CmdList("client -o") # find the current client
951 if r
.has_key('Client'):
955 die("could not get client spec")
956 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
958 if r
.has_key('change'):
960 die("Could not get changelist number for last submit - cannot patch up user details")
962 def modifyChangelistUser(self
, changelist
, newUser
):
963 # fixup the user field of a changelist after it has been submitted.
964 changes
= p4CmdList("change -o %s" % changelist
)
965 if len(changes
) != 1:
966 die("Bad output from p4 change modifying %s to user %s" %
967 (changelist
, newUser
))
970 if c
['User'] == newUser
: return # nothing to do
972 input = marshal
.dumps(c
)
974 result
= p4CmdList("change -f -i", stdin
=input)
976 if r
.has_key('code'):
977 if r
['code'] == 'error':
978 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
979 if r
.has_key('data'):
980 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
982 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
984 def canChangeChangelists(self
):
985 # check to see if we have p4 admin or super-user permissions, either of
986 # which are required to modify changelists.
987 results
= p4CmdList(["protects", self
.depotPath
])
989 if r
.has_key('perm'):
990 if r
['perm'] == 'admin':
992 if r
['perm'] == 'super':
996 def prepareSubmitTemplate(self
):
997 # remove lines in the Files section that show changes to files outside the depot path we're committing into
999 inFilesSection
= False
1000 for line
in p4_read_pipe_lines(['change', '-o']):
1001 if line
.endswith("\r\n"):
1002 line
= line
[:-2] + "\n"
1004 if line
.startswith("\t"):
1005 # path starts and ends with a tab
1007 lastTab
= path
.rfind("\t")
1009 path
= path
[:lastTab
]
1010 if not p4PathStartsWith(path
, self
.depotPath
):
1013 inFilesSection
= False
1015 if line
.startswith("Files:"):
1016 inFilesSection
= True
1022 def edit_template(self
, template_file
):
1023 """Invoke the editor to let the user change the submission
1024 message. Return true if okay to continue with the submit."""
1026 # if configured to skip the editing part, just submit
1027 if gitConfig("git-p4.skipSubmitEdit") == "true":
1030 # look at the modification time, to check later if the user saved
1032 mtime
= os
.stat(template_file
).st_mtime
1035 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1036 editor
= os
.environ
.get("P4EDITOR")
1038 editor
= read_pipe("git var GIT_EDITOR").strip()
1039 system(editor
+ " " + template_file
)
1041 # If the file was not saved, prompt to see if this patch should
1042 # be skipped. But skip this verification step if configured so.
1043 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1046 # modification time updated means user saved the file
1047 if os
.stat(template_file
).st_mtime
> mtime
:
1051 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1057 def applyCommit(self
, id):
1058 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1060 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1062 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1064 filesToDelete
= set()
1066 pureRenameCopy
= set()
1067 filesToChangeExecBit
= {}
1070 diff
= parseDiffTreeEntry(line
)
1071 modifier
= diff
['status']
1075 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1076 filesToChangeExecBit
[path
] = diff
['dst_mode']
1077 editedFiles
.add(path
)
1078 elif modifier
== "A":
1079 filesToAdd
.add(path
)
1080 filesToChangeExecBit
[path
] = diff
['dst_mode']
1081 if path
in filesToDelete
:
1082 filesToDelete
.remove(path
)
1083 elif modifier
== "D":
1084 filesToDelete
.add(path
)
1085 if path
in filesToAdd
:
1086 filesToAdd
.remove(path
)
1087 elif modifier
== "C":
1088 src
, dest
= diff
['src'], diff
['dst']
1089 p4_integrate(src
, dest
)
1090 pureRenameCopy
.add(dest
)
1091 if diff
['src_sha1'] != diff
['dst_sha1']:
1093 pureRenameCopy
.discard(dest
)
1094 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1096 pureRenameCopy
.discard(dest
)
1097 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1099 editedFiles
.add(dest
)
1100 elif modifier
== "R":
1101 src
, dest
= diff
['src'], diff
['dst']
1102 if self
.p4HasMoveCommand
:
1103 p4_edit(src
) # src must be open before move
1104 p4_move(src
, dest
) # opens for (move/delete, move/add)
1106 p4_integrate(src
, dest
)
1107 if diff
['src_sha1'] != diff
['dst_sha1']:
1110 pureRenameCopy
.add(dest
)
1111 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1112 if not self
.p4HasMoveCommand
:
1113 p4_edit(dest
) # with move: already open, writable
1114 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1115 if not self
.p4HasMoveCommand
:
1117 filesToDelete
.add(src
)
1118 editedFiles
.add(dest
)
1120 die("unknown modifier %s for %s" % (modifier
, path
))
1122 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1123 patchcmd
= diffcmd
+ " | git apply "
1124 tryPatchCmd
= patchcmd
+ "--check -"
1125 applyPatchCmd
= patchcmd
+ "--check --apply -"
1126 patch_succeeded
= True
1128 if os
.system(tryPatchCmd
) != 0:
1129 fixed_rcs_keywords
= False
1130 patch_succeeded
= False
1131 print "Unfortunately applying the change failed!"
1133 # Patch failed, maybe it's just RCS keyword woes. Look through
1134 # the patch to see if that's possible.
1135 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1139 for file in editedFiles | filesToDelete
:
1140 # did this file's delta contain RCS keywords?
1141 pattern
= p4_keywords_regexp_for_file(file)
1144 # this file is a possibility...look for RCS keywords.
1145 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1146 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1147 if regexp
.search(line
):
1149 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1150 kwfiles
[file] = pattern
1153 for file in kwfiles
:
1155 print "zapping %s with %s" % (line
,pattern
)
1156 self
.patchRCSKeywords(file, kwfiles
[file])
1157 fixed_rcs_keywords
= True
1159 if fixed_rcs_keywords
:
1160 print "Retrying the patch with RCS keywords cleaned up"
1161 if os
.system(tryPatchCmd
) == 0:
1162 patch_succeeded
= True
1164 if not patch_succeeded
:
1165 print "What do you want to do?"
1167 while response
!= "s" and response
!= "a" and response
!= "w":
1168 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1169 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1171 print "Skipping! Good luck with the next patches..."
1172 for f
in editedFiles
:
1174 for f
in filesToAdd
:
1177 elif response
== "a":
1178 os
.system(applyPatchCmd
)
1179 if len(filesToAdd
) > 0:
1180 print "You may also want to call p4 add on the following files:"
1181 print " ".join(filesToAdd
)
1182 if len(filesToDelete
):
1183 print "The following files should be scheduled for deletion with p4 delete:"
1184 print " ".join(filesToDelete
)
1185 die("Please resolve and submit the conflict manually and "
1186 + "continue afterwards with git p4 submit --continue")
1187 elif response
== "w":
1188 system(diffcmd
+ " > patch.txt")
1189 print "Patch saved to patch.txt in %s !" % self
.clientPath
1190 die("Please resolve and submit the conflict manually and "
1191 "continue afterwards with git p4 submit --continue")
1193 system(applyPatchCmd
)
1195 for f
in filesToAdd
:
1197 for f
in filesToDelete
:
1201 # Set/clear executable bits
1202 for f
in filesToChangeExecBit
.keys():
1203 mode
= filesToChangeExecBit
[f
]
1204 setP4ExecBit(f
, mode
)
1206 logMessage
= extractLogMessageFromGitCommit(id)
1207 logMessage
= logMessage
.strip()
1209 template
= self
.prepareSubmitTemplate()
1211 if self
.interactive
:
1212 submitTemplate
= self
.prepareLogMessage(template
, logMessage
)
1214 if self
.preserveUser
:
1215 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1217 if os
.environ
.has_key("P4DIFF"):
1218 del(os
.environ
["P4DIFF"])
1220 for editedFile
in editedFiles
:
1221 diff
+= p4_read_pipe(['diff', '-du',
1222 wildcard_encode(editedFile
)])
1225 for newFile
in filesToAdd
:
1226 newdiff
+= "==== new file ====\n"
1227 newdiff
+= "--- /dev/null\n"
1228 newdiff
+= "+++ %s\n" % newFile
1229 f
= open(newFile
, "r")
1230 for line
in f
.readlines():
1231 newdiff
+= "+" + line
1234 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1235 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1236 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1237 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1239 separatorLine
= "######## everything below this line is just the diff #######\n"
1241 (handle
, fileName
) = tempfile
.mkstemp()
1242 tmpFile
= os
.fdopen(handle
, "w+")
1244 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1245 separatorLine
= separatorLine
.replace("\n", "\r\n")
1246 newdiff
= newdiff
.replace("\n", "\r\n")
1247 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1250 if self
.edit_template(fileName
):
1251 # read the edited message and submit
1252 tmpFile
= open(fileName
, "rb")
1253 message
= tmpFile
.read()
1255 submitTemplate
= message
[:message
.index(separatorLine
)]
1257 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1258 p4_write_pipe(['submit', '-i'], submitTemplate
)
1260 if self
.preserveUser
:
1262 # Get last changelist number. Cannot easily get it from
1263 # the submit command output as the output is
1265 changelist
= self
.lastP4Changelist()
1266 self
.modifyChangelistUser(changelist
, p4User
)
1268 # The rename/copy happened by applying a patch that created a
1269 # new file. This leaves it writable, which confuses p4.
1270 for f
in pureRenameCopy
:
1275 print "Submission cancelled, undoing p4 changes."
1276 for f
in editedFiles
:
1278 for f
in filesToAdd
:
1284 fileName
= "submit.txt"
1285 file = open(fileName
, "w+")
1286 file.write(self
.prepareLogMessage(template
, logMessage
))
1288 print ("Perforce submit template written as %s. "
1289 + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1290 % (fileName
, fileName
))
1292 # Export git tags as p4 labels. Create a p4 label and then tag
1294 def exportGitTags(self
, gitTags
):
1295 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1296 if len(validLabelRegexp
) == 0:
1297 validLabelRegexp
= defaultLabelRegexp
1298 m
= re
.compile(validLabelRegexp
)
1300 for name
in gitTags
:
1302 if not m
.match(name
):
1304 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1307 # Get the p4 commit this corresponds to
1308 logMessage
= extractLogMessageFromGitCommit(name
)
1309 values
= extractSettingsGitLog(logMessage
)
1311 if not values
.has_key('change'):
1312 # a tag pointing to something not sent to p4; ignore
1314 print "git tag %s does not give a p4 commit" % name
1317 changelist
= values
['change']
1319 # Get the tag details.
1323 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1326 if re
.match(r
'tag\s+', l
):
1328 elif re
.match(r
'\s*$', l
):
1335 body
= ["lightweight tag imported by git p4\n"]
1337 # Create the label - use the same view as the client spec we are using
1338 clientSpec
= getClientSpec()
1340 labelTemplate
= "Label: %s\n" % name
1341 labelTemplate
+= "Description:\n"
1343 labelTemplate
+= "\t" + b
+ "\n"
1344 labelTemplate
+= "View:\n"
1345 for mapping
in clientSpec
.mappings
:
1346 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1348 p4_write_pipe(["label", "-i"], labelTemplate
)
1351 p4_system(["tag", "-l", name
] +
1352 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1355 print "created p4 label for tag %s" % name
1357 def run(self
, args
):
1359 self
.master
= currentGitBranch()
1360 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1361 die("Detecting current git branch failed!")
1362 elif len(args
) == 1:
1363 self
.master
= args
[0]
1364 if not branchExists(self
.master
):
1365 die("Branch %s does not exist" % self
.master
)
1369 allowSubmit
= gitConfig("git-p4.allowSubmit")
1370 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1371 die("%s is not in git-p4.allowSubmit" % self
.master
)
1373 [upstream
, settings
] = findUpstreamBranchPoint()
1374 self
.depotPath
= settings
['depot-paths'][0]
1375 if len(self
.origin
) == 0:
1376 self
.origin
= upstream
1378 if self
.preserveUser
:
1379 if not self
.canChangeChangelists():
1380 die("Cannot preserve user names without p4 super-user or admin permissions")
1383 print "Origin branch is " + self
.origin
1385 if len(self
.depotPath
) == 0:
1386 print "Internal error: cannot locate perforce depot path from existing branches"
1389 self
.useClientSpec
= False
1390 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1391 self
.useClientSpec
= True
1392 if self
.useClientSpec
:
1393 self
.clientSpecDirs
= getClientSpec()
1395 if self
.useClientSpec
:
1396 # all files are relative to the client spec
1397 self
.clientPath
= getClientRoot()
1399 self
.clientPath
= p4Where(self
.depotPath
)
1401 if self
.clientPath
== "":
1402 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1404 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1405 self
.oldWorkingDirectory
= os
.getcwd()
1407 # ensure the clientPath exists
1408 new_client_dir
= False
1409 if not os
.path
.exists(self
.clientPath
):
1410 new_client_dir
= True
1411 os
.makedirs(self
.clientPath
)
1413 chdir(self
.clientPath
)
1414 print "Synchronizing p4 checkout..."
1416 # old one was destroyed, and maybe nobody told p4
1417 p4_sync("...", "-f")
1423 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1424 commits
.append(line
.strip())
1427 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1428 self
.checkAuthorship
= False
1430 self
.checkAuthorship
= True
1432 if self
.preserveUser
:
1433 self
.checkValidP4Users(commits
)
1436 # Build up a set of options to be passed to diff when
1437 # submitting each commit to p4.
1439 if self
.detectRenames
:
1440 # command-line -M arg
1441 self
.diffOpts
= "-M"
1443 # If not explicitly set check the config variable
1444 detectRenames
= gitConfig("git-p4.detectRenames")
1446 if detectRenames
.lower() == "false" or detectRenames
== "":
1448 elif detectRenames
.lower() == "true":
1449 self
.diffOpts
= "-M"
1451 self
.diffOpts
= "-M%s" % detectRenames
1453 # no command-line arg for -C or --find-copies-harder, just
1455 detectCopies
= gitConfig("git-p4.detectCopies")
1456 if detectCopies
.lower() == "false" or detectCopies
== "":
1458 elif detectCopies
.lower() == "true":
1459 self
.diffOpts
+= " -C"
1461 self
.diffOpts
+= " -C%s" % detectCopies
1463 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1464 self
.diffOpts
+= " --find-copies-harder"
1466 while len(commits
) > 0:
1468 commits
= commits
[1:]
1469 self
.applyCommit(commit
)
1470 if not self
.interactive
:
1473 if len(commits
) == 0:
1474 print "All changes applied!"
1475 chdir(self
.oldWorkingDirectory
)
1483 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1484 self
.exportLabels
= True
1486 if self
.exportLabels
:
1487 p4Labels
= getP4Labels(self
.depotPath
)
1488 gitTags
= getGitTags()
1490 missingGitTags
= gitTags
- p4Labels
1491 self
.exportGitTags(missingGitTags
)
1496 """Represent a p4 view ("p4 help views"), and map files in a
1497 repo according to the view."""
1500 """A depot or client path, possibly containing wildcards.
1501 The only one supported is ... at the end, currently.
1502 Initialize with the full path, with //depot or //client."""
1504 def __init__(self
, path
, is_depot
):
1506 self
.is_depot
= is_depot
1507 self
.find_wildcards()
1508 # remember the prefix bit, useful for relative mappings
1509 m
= re
.match("(//[^/]+/)", self
.path
)
1511 die("Path %s does not start with //prefix/" % self
.path
)
1513 if not self
.is_depot
:
1514 # strip //client/ on client paths
1515 self
.path
= self
.path
[len(prefix
):]
1517 def find_wildcards(self
):
1518 """Make sure wildcards are valid, and set up internal
1521 self
.ends_triple_dot
= False
1522 # There are three wildcards allowed in p4 views
1523 # (see "p4 help views"). This code knows how to
1524 # handle "..." (only at the end), but cannot deal with
1525 # "%%n" or "*". Only check the depot_side, as p4 should
1526 # validate that the client_side matches too.
1527 if re
.search(r
'%%[1-9]', self
.path
):
1528 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1529 if self
.path
.find("*") >= 0:
1530 die("Can't handle * wildcards in view: %s" % self
.path
)
1531 triple_dot_index
= self
.path
.find("...")
1532 if triple_dot_index
>= 0:
1533 if triple_dot_index
!= len(self
.path
) - 3:
1534 die("Can handle only single ... wildcard, at end: %s" %
1536 self
.ends_triple_dot
= True
1538 def ensure_compatible(self
, other_path
):
1539 """Make sure the wildcards agree."""
1540 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1541 die("Both paths must end with ... if either does;\n" +
1542 "paths: %s %s" % (self
.path
, other_path
.path
))
1544 def match_wildcards(self
, test_path
):
1545 """See if this test_path matches us, and fill in the value
1546 of the wildcards if so. Returns a tuple of
1547 (True|False, wildcards[]). For now, only the ... at end
1548 is supported, so at most one wildcard."""
1549 if self
.ends_triple_dot
:
1550 dotless
= self
.path
[:-3]
1551 if test_path
.startswith(dotless
):
1552 wildcard
= test_path
[len(dotless
):]
1553 return (True, [ wildcard
])
1555 if test_path
== self
.path
:
1559 def match(self
, test_path
):
1560 """Just return if it matches; don't bother with the wildcards."""
1561 b
, _
= self
.match_wildcards(test_path
)
1564 def fill_in_wildcards(self
, wildcards
):
1565 """Return the relative path, with the wildcards filled in
1566 if there are any."""
1567 if self
.ends_triple_dot
:
1568 return self
.path
[:-3] + wildcards
[0]
1572 class Mapping(object):
1573 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1574 # depot_side is without the trailing /... if it had one
1575 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1576 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1577 self
.overlay
= overlay
# started with "+"
1578 self
.exclude
= exclude
# started with "-"
1579 assert not (self
.overlay
and self
.exclude
)
1580 self
.depot_side
.ensure_compatible(self
.client_side
)
1588 return "View.Mapping: %s%s -> %s" % \
1589 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1591 def map_depot_to_client(self
, depot_path
):
1592 """Calculate the client path if using this mapping on the
1593 given depot path; does not consider the effect of other
1594 mappings in a view. Even excluded mappings are returned."""
1595 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1598 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1607 def append(self
, view_line
):
1608 """Parse a view line, splitting it into depot and client
1609 sides. Append to self.mappings, preserving order."""
1611 # Split the view line into exactly two words. P4 enforces
1612 # structure on these lines that simplifies this quite a bit.
1614 # Either or both words may be double-quoted.
1615 # Single quotes do not matter.
1616 # Double-quote marks cannot occur inside the words.
1617 # A + or - prefix is also inside the quotes.
1618 # There are no quotes unless they contain a space.
1619 # The line is already white-space stripped.
1620 # The two words are separated by a single space.
1622 if view_line
[0] == '"':
1623 # First word is double quoted. Find its end.
1624 close_quote_index
= view_line
.find('"', 1)
1625 if close_quote_index
<= 0:
1626 die("No first-word closing quote found: %s" % view_line
)
1627 depot_side
= view_line
[1:close_quote_index
]
1628 # skip closing quote and space
1629 rhs_index
= close_quote_index
+ 1 + 1
1631 space_index
= view_line
.find(" ")
1632 if space_index
<= 0:
1633 die("No word-splitting space found: %s" % view_line
)
1634 depot_side
= view_line
[0:space_index
]
1635 rhs_index
= space_index
+ 1
1637 if view_line
[rhs_index
] == '"':
1638 # Second word is double quoted. Make sure there is a
1639 # double quote at the end too.
1640 if not view_line
.endswith('"'):
1641 die("View line with rhs quote should end with one: %s" %
1644 client_side
= view_line
[rhs_index
+1:-1]
1646 client_side
= view_line
[rhs_index
:]
1648 # prefix + means overlay on previous mapping
1650 if depot_side
.startswith("+"):
1652 depot_side
= depot_side
[1:]
1654 # prefix - means exclude this path
1656 if depot_side
.startswith("-"):
1658 depot_side
= depot_side
[1:]
1660 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1661 self
.mappings
.append(m
)
1663 def map_in_client(self
, depot_path
):
1664 """Return the relative location in the client where this
1665 depot file should live. Returns "" if the file should
1666 not be mapped in the client."""
1671 # look at later entries first
1672 for m
in self
.mappings
[::-1]:
1674 # see where will this path end up in the client
1675 p
= m
.map_depot_to_client(depot_path
)
1678 # Depot path does not belong in client. Must remember
1679 # this, as previous items should not cause files to
1680 # exist in this path either. Remember that the list is
1681 # being walked from the end, which has higher precedence.
1682 # Overlap mappings do not exclude previous mappings.
1684 paths_filled
.append(m
.client_side
)
1687 # This mapping matched; no need to search any further.
1688 # But, the mapping could be rejected if the client path
1689 # has already been claimed by an earlier mapping (i.e.
1690 # one later in the list, which we are walking backwards).
1691 already_mapped_in_client
= False
1692 for f
in paths_filled
:
1693 # this is View.Path.match
1695 already_mapped_in_client
= True
1697 if not already_mapped_in_client
:
1698 # Include this file, unless it is from a line that
1699 # explicitly said to exclude it.
1703 # a match, even if rejected, always stops the search
1708 class P4Sync(Command
, P4UserMap
):
1709 delete_actions
= ( "delete", "move/delete", "purge" )
1712 Command
.__init
__(self
)
1713 P4UserMap
.__init
__(self
)
1715 optparse
.make_option("--branch", dest
="branch"),
1716 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1717 optparse
.make_option("--changesfile", dest
="changesFile"),
1718 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1719 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1720 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1721 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1722 help="Import into refs/heads/ , not refs/remotes"),
1723 optparse
.make_option("--max-changes", dest
="maxChanges"),
1724 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1725 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1726 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1727 help="Only sync files that are included in the Perforce Client Spec")
1729 self
.description
= """Imports from Perforce into a git repository.\n
1731 //depot/my/project/ -- to import the current head
1732 //depot/my/project/@all -- to import everything
1733 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1735 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1737 self
.usage
+= " //depot/path[@revRange]"
1739 self
.createdBranches
= set()
1740 self
.committedChanges
= set()
1742 self
.detectBranches
= False
1743 self
.detectLabels
= False
1744 self
.importLabels
= False
1745 self
.changesFile
= ""
1746 self
.syncWithOrigin
= True
1747 self
.importIntoRemotes
= True
1748 self
.maxChanges
= ""
1749 self
.isWindows
= (platform
.system() == "Windows")
1750 self
.keepRepoPath
= False
1751 self
.depotPaths
= None
1752 self
.p4BranchesInGit
= []
1753 self
.cloneExclude
= []
1754 self
.useClientSpec
= False
1755 self
.useClientSpec_from_options
= False
1756 self
.clientSpecDirs
= None
1757 self
.tempBranches
= []
1758 self
.tempBranchLocation
= "git-p4-tmp"
1760 if gitConfig("git-p4.syncFromOrigin") == "false":
1761 self
.syncWithOrigin
= False
1763 # Force a checkpoint in fast-import and wait for it to finish
1764 def checkpoint(self
):
1765 self
.gitStream
.write("checkpoint\n\n")
1766 self
.gitStream
.write("progress checkpoint\n\n")
1767 out
= self
.gitOutput
.readline()
1769 print "checkpoint finished: " + out
1771 def extractFilesFromCommit(self
, commit
):
1772 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1773 for path
in self
.cloneExclude
]
1776 while commit
.has_key("depotFile%s" % fnum
):
1777 path
= commit
["depotFile%s" % fnum
]
1779 if [p
for p
in self
.cloneExclude
1780 if p4PathStartsWith(path
, p
)]:
1783 found
= [p
for p
in self
.depotPaths
1784 if p4PathStartsWith(path
, p
)]
1791 file["rev"] = commit
["rev%s" % fnum
]
1792 file["action"] = commit
["action%s" % fnum
]
1793 file["type"] = commit
["type%s" % fnum
]
1798 def stripRepoPath(self
, path
, prefixes
):
1799 if self
.useClientSpec
:
1800 return self
.clientSpecDirs
.map_in_client(path
)
1802 if self
.keepRepoPath
:
1803 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1806 if p4PathStartsWith(path
, p
):
1807 path
= path
[len(p
):]
1811 def splitFilesIntoBranches(self
, commit
):
1814 while commit
.has_key("depotFile%s" % fnum
):
1815 path
= commit
["depotFile%s" % fnum
]
1816 found
= [p
for p
in self
.depotPaths
1817 if p4PathStartsWith(path
, p
)]
1824 file["rev"] = commit
["rev%s" % fnum
]
1825 file["action"] = commit
["action%s" % fnum
]
1826 file["type"] = commit
["type%s" % fnum
]
1829 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1830 relPath
= wildcard_decode(relPath
)
1832 for branch
in self
.knownBranches
.keys():
1834 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1835 if relPath
.startswith(branch
+ "/"):
1836 if branch
not in branches
:
1837 branches
[branch
] = []
1838 branches
[branch
].append(file)
1843 # output one file from the P4 stream
1844 # - helper for streamP4Files
1846 def streamOneP4File(self
, file, contents
):
1847 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1848 relPath
= wildcard_decode(relPath
)
1850 sys
.stderr
.write("%s\n" % relPath
)
1852 (type_base
, type_mods
) = split_p4_type(file["type"])
1855 if "x" in type_mods
:
1857 if type_base
== "symlink":
1859 # p4 print on a symlink contains "target\n"; remove the newline
1860 data
= ''.join(contents
)
1861 contents
= [data
[:-1]]
1863 if type_base
== "utf16":
1864 # p4 delivers different text in the python output to -G
1865 # than it does when using "print -o", or normal p4 client
1866 # operations. utf16 is converted to ascii or utf8, perhaps.
1867 # But ascii text saved as -t utf16 is completely mangled.
1868 # Invoke print -o to get the real contents.
1869 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1872 if type_base
== "apple":
1873 # Apple filetype files will be streamed as a concatenation of
1874 # its appledouble header and the contents. This is useless
1875 # on both macs and non-macs. If using "print -q -o xx", it
1876 # will create "xx" with the data, and "%xx" with the header.
1877 # This is also not very useful.
1879 # Ideally, someday, this script can learn how to generate
1880 # appledouble files directly and import those to git, but
1881 # non-mac machines can never find a use for apple filetype.
1882 print "\nIgnoring apple filetype file %s" % file['depotFile']
1885 # Perhaps windows wants unicode, utf16 newlines translated too;
1886 # but this is not doing it.
1887 if self
.isWindows
and type_base
== "text":
1889 for data
in contents
:
1890 data
= data
.replace("\r\n", "\n")
1891 mangled
.append(data
)
1894 # Note that we do not try to de-mangle keywords on utf16 files,
1895 # even though in theory somebody may want that.
1896 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1898 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1899 text
= ''.join(contents
)
1900 text
= regexp
.sub(r
'$\1$', text
)
1903 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1908 length
= length
+ len(d
)
1910 self
.gitStream
.write("data %d\n" % length
)
1912 self
.gitStream
.write(d
)
1913 self
.gitStream
.write("\n")
1915 def streamOneP4Deletion(self
, file):
1916 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1917 relPath
= wildcard_decode(relPath
)
1919 sys
.stderr
.write("delete %s\n" % relPath
)
1920 self
.gitStream
.write("D %s\n" % relPath
)
1922 # handle another chunk of streaming data
1923 def streamP4FilesCb(self
, marshalled
):
1925 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1926 # start of a new file - output the old one first
1927 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1928 self
.stream_file
= {}
1929 self
.stream_contents
= []
1930 self
.stream_have_file_info
= False
1932 # pick up the new file information... for the
1933 # 'data' field we need to append to our array
1934 for k
in marshalled
.keys():
1936 self
.stream_contents
.append(marshalled
['data'])
1938 self
.stream_file
[k
] = marshalled
[k
]
1940 self
.stream_have_file_info
= True
1942 # Stream directly from "p4 files" into "git fast-import"
1943 def streamP4Files(self
, files
):
1949 # if using a client spec, only add the files that have
1950 # a path in the client
1951 if self
.clientSpecDirs
:
1952 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
1955 filesForCommit
.append(f
)
1956 if f
['action'] in self
.delete_actions
:
1957 filesToDelete
.append(f
)
1959 filesToRead
.append(f
)
1962 for f
in filesToDelete
:
1963 self
.streamOneP4Deletion(f
)
1965 if len(filesToRead
) > 0:
1966 self
.stream_file
= {}
1967 self
.stream_contents
= []
1968 self
.stream_have_file_info
= False
1970 # curry self argument
1971 def streamP4FilesCbSelf(entry
):
1972 self
.streamP4FilesCb(entry
)
1974 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
1976 p4CmdList(["-x", "-", "print"],
1978 cb
=streamP4FilesCbSelf
)
1981 if self
.stream_file
.has_key('depotFile'):
1982 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1984 def make_email(self
, userid
):
1985 if userid
in self
.users
:
1986 return self
.users
[userid
]
1988 return "%s <a@b>" % userid
1991 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
1993 print "writing tag %s for commit %s" % (labelName
, commit
)
1994 gitStream
.write("tag %s\n" % labelName
)
1995 gitStream
.write("from %s\n" % commit
)
1997 if labelDetails
.has_key('Owner'):
1998 owner
= labelDetails
["Owner"]
2002 # Try to use the owner of the p4 label, or failing that,
2003 # the current p4 user id.
2005 email
= self
.make_email(owner
)
2007 email
= self
.make_email(self
.p4UserId())
2008 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2010 gitStream
.write("tagger %s\n" % tagger
)
2012 print "labelDetails=",labelDetails
2013 if labelDetails
.has_key('Description'):
2014 description
= labelDetails
['Description']
2016 description
= 'Label from git p4'
2018 gitStream
.write("data %d\n" % len(description
))
2019 gitStream
.write(description
)
2020 gitStream
.write("\n")
2022 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
2023 epoch
= details
["time"]
2024 author
= details
["user"]
2025 self
.branchPrefixes
= branchPrefixes
2028 print "commit into %s" % branch
2030 # start with reading files; if that fails, we should not
2034 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2035 new_files
.append (f
)
2037 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2039 self
.gitStream
.write("commit %s\n" % branch
)
2040 # gitStream.write("mark :%s\n" % details["change"])
2041 self
.committedChanges
.add(int(details
["change"]))
2043 if author
not in self
.users
:
2044 self
.getUserMapFromPerforceServer()
2045 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2047 self
.gitStream
.write("committer %s\n" % committer
)
2049 self
.gitStream
.write("data <<EOT\n")
2050 self
.gitStream
.write(details
["desc"])
2051 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2052 % (','.join (branchPrefixes
), details
["change"]))
2053 if len(details
['options']) > 0:
2054 self
.gitStream
.write(": options = %s" % details
['options'])
2055 self
.gitStream
.write("]\nEOT\n\n")
2059 print "parent %s" % parent
2060 self
.gitStream
.write("from %s\n" % parent
)
2062 self
.streamP4Files(new_files
)
2063 self
.gitStream
.write("\n")
2065 change
= int(details
["change"])
2067 if self
.labels
.has_key(change
):
2068 label
= self
.labels
[change
]
2069 labelDetails
= label
[0]
2070 labelRevisions
= label
[1]
2072 print "Change %s is labelled %s" % (change
, labelDetails
)
2074 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2075 for p
in branchPrefixes
])
2077 if len(files
) == len(labelRevisions
):
2081 if info
["action"] in self
.delete_actions
:
2083 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2085 if cleanedFiles
== labelRevisions
:
2086 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2090 print ("Tag %s does not match with change %s: files do not match."
2091 % (labelDetails
["label"], change
))
2095 print ("Tag %s does not match with change %s: file count is different."
2096 % (labelDetails
["label"], change
))
2098 # Build a dictionary of changelists and labels, for "detect-labels" option.
2099 def getLabels(self
):
2102 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2103 if len(l
) > 0 and not self
.silent
:
2104 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2107 label
= output
["label"]
2111 print "Querying files for label %s" % label
2112 for file in p4CmdList(["files"] +
2113 ["%s...@%s" % (p
, label
)
2114 for p
in self
.depotPaths
]):
2115 revisions
[file["depotFile"]] = file["rev"]
2116 change
= int(file["change"])
2117 if change
> newestChange
:
2118 newestChange
= change
2120 self
.labels
[newestChange
] = [output
, revisions
]
2123 print "Label changes: %s" % self
.labels
.keys()
2125 # Import p4 labels as git tags. A direct mapping does not
2126 # exist, so assume that if all the files are at the same revision
2127 # then we can use that, or it's something more complicated we should
2129 def importP4Labels(self
, stream
, p4Labels
):
2131 print "import p4 labels: " + ' '.join(p4Labels
)
2133 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2134 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2135 if len(validLabelRegexp
) == 0:
2136 validLabelRegexp
= defaultLabelRegexp
2137 m
= re
.compile(validLabelRegexp
)
2139 for name
in p4Labels
:
2142 if not m
.match(name
):
2144 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2147 if name
in ignoredP4Labels
:
2150 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2152 # get the most recent changelist for each file in this label
2153 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2154 for p
in self
.depotPaths
])
2156 if change
.has_key('change'):
2157 # find the corresponding git commit; take the oldest commit
2158 changelist
= int(change
['change'])
2159 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2160 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2161 if len(gitCommit
) == 0:
2162 print "could not find git commit for changelist %d" % changelist
2164 gitCommit
= gitCommit
.strip()
2166 # Convert from p4 time format
2168 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2170 print "Could not convert label time %s" % labelDetail
['Update']
2173 when
= int(time
.mktime(tmwhen
))
2174 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2176 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2179 print "Label %s has no changelists - possibly deleted?" % name
2182 # We can't import this label; don't try again as it will get very
2183 # expensive repeatedly fetching all the files for labels that will
2184 # never be imported. If the label is moved in the future, the
2185 # ignore will need to be removed manually.
2186 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2188 def guessProjectName(self
):
2189 for p
in self
.depotPaths
:
2192 p
= p
[p
.strip().rfind("/") + 1:]
2193 if not p
.endswith("/"):
2197 def getBranchMapping(self
):
2198 lostAndFoundBranches
= set()
2200 user
= gitConfig("git-p4.branchUser")
2202 command
= "branches -u %s" % user
2204 command
= "branches"
2206 for info
in p4CmdList(command
):
2207 details
= p4Cmd(["branch", "-o", info
["branch"]])
2209 while details
.has_key("View%s" % viewIdx
):
2210 paths
= details
["View%s" % viewIdx
].split(" ")
2211 viewIdx
= viewIdx
+ 1
2212 # require standard //depot/foo/... //depot/bar/... mapping
2213 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2216 destination
= paths
[1]
2218 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2219 source
= source
[len(self
.depotPaths
[0]):-4]
2220 destination
= destination
[len(self
.depotPaths
[0]):-4]
2222 if destination
in self
.knownBranches
:
2224 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2225 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2228 self
.knownBranches
[destination
] = source
2230 lostAndFoundBranches
.discard(destination
)
2232 if source
not in self
.knownBranches
:
2233 lostAndFoundBranches
.add(source
)
2235 # Perforce does not strictly require branches to be defined, so we also
2236 # check git config for a branch list.
2238 # Example of branch definition in git config file:
2240 # branchList=main:branchA
2241 # branchList=main:branchB
2242 # branchList=branchA:branchC
2243 configBranches
= gitConfigList("git-p4.branchList")
2244 for branch
in configBranches
:
2246 (source
, destination
) = branch
.split(":")
2247 self
.knownBranches
[destination
] = source
2249 lostAndFoundBranches
.discard(destination
)
2251 if source
not in self
.knownBranches
:
2252 lostAndFoundBranches
.add(source
)
2255 for branch
in lostAndFoundBranches
:
2256 self
.knownBranches
[branch
] = branch
2258 def getBranchMappingFromGitBranches(self
):
2259 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2260 for branch
in branches
.keys():
2261 if branch
== "master":
2264 branch
= branch
[len(self
.projectName
):]
2265 self
.knownBranches
[branch
] = branch
2267 def listExistingP4GitBranches(self
):
2268 # branches holds mapping from name to commit
2269 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2270 self
.p4BranchesInGit
= branches
.keys()
2271 for branch
in branches
.keys():
2272 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2274 def updateOptionDict(self
, d
):
2276 if self
.keepRepoPath
:
2277 option_keys
['keepRepoPath'] = 1
2279 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2281 def readOptions(self
, d
):
2282 self
.keepRepoPath
= (d
.has_key('options')
2283 and ('keepRepoPath' in d
['options']))
2285 def gitRefForBranch(self
, branch
):
2286 if branch
== "main":
2287 return self
.refPrefix
+ "master"
2289 if len(branch
) <= 0:
2292 return self
.refPrefix
+ self
.projectName
+ branch
2294 def gitCommitByP4Change(self
, ref
, change
):
2296 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2299 latestCommit
= parseRevision(ref
)
2303 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2304 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2309 log
= extractLogMessageFromGitCommit(next
)
2310 settings
= extractSettingsGitLog(log
)
2311 currentChange
= int(settings
['change'])
2313 print "current change %s" % currentChange
2315 if currentChange
== change
:
2317 print "found %s" % next
2320 if currentChange
< change
:
2321 earliestCommit
= "^%s" % next
2323 latestCommit
= "%s" % next
2327 def importNewBranch(self
, branch
, maxChange
):
2328 # make fast-import flush all changes to disk and update the refs using the checkpoint
2329 # command so that we can try to find the branch parent in the git history
2330 self
.gitStream
.write("checkpoint\n\n");
2331 self
.gitStream
.flush();
2332 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2333 range = "@1,%s" % maxChange
2334 #print "prefix" + branchPrefix
2335 changes
= p4ChangesForPaths([branchPrefix
], range)
2336 if len(changes
) <= 0:
2338 firstChange
= changes
[0]
2339 #print "first change in branch: %s" % firstChange
2340 sourceBranch
= self
.knownBranches
[branch
]
2341 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2342 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2343 #print "source " + sourceBranch
2345 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2346 #print "branch parent: %s" % branchParentChange
2347 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2348 if len(gitParent
) > 0:
2349 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2350 #print "parent git commit: %s" % gitParent
2352 self
.importChanges(changes
)
2355 def searchParent(self
, parent
, branch
, target
):
2357 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2359 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2362 print "Found parent of %s in commit %s" % (branch
, blob
)
2369 def importChanges(self
, changes
):
2371 for change
in changes
:
2372 description
= p4Cmd(["describe", str(change
)])
2373 self
.updateOptionDict(description
)
2376 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2381 if self
.detectBranches
:
2382 branches
= self
.splitFilesIntoBranches(description
)
2383 for branch
in branches
.keys():
2385 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2389 filesForCommit
= branches
[branch
]
2392 print "branch is %s" % branch
2394 self
.updatedBranches
.add(branch
)
2396 if branch
not in self
.createdBranches
:
2397 self
.createdBranches
.add(branch
)
2398 parent
= self
.knownBranches
[branch
]
2399 if parent
== branch
:
2402 fullBranch
= self
.projectName
+ branch
2403 if fullBranch
not in self
.p4BranchesInGit
:
2405 print("\n Importing new branch %s" % fullBranch
);
2406 if self
.importNewBranch(branch
, change
- 1):
2408 self
.p4BranchesInGit
.append(fullBranch
)
2410 print("\n Resuming with change %s" % change
);
2413 print "parent determined through known branches: %s" % parent
2415 branch
= self
.gitRefForBranch(branch
)
2416 parent
= self
.gitRefForBranch(parent
)
2419 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2421 if len(parent
) == 0 and branch
in self
.initialParents
:
2422 parent
= self
.initialParents
[branch
]
2423 del self
.initialParents
[branch
]
2427 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2429 print "Creating temporary branch: " + tempBranch
2430 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2431 self
.tempBranches
.append(tempBranch
)
2433 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2435 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2438 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2439 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2441 files
= self
.extractFilesFromCommit(description
)
2442 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2444 self
.initialParent
= ""
2446 print self
.gitError
.read()
2449 def importHeadRevision(self
, revision
):
2450 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2453 details
["user"] = "git perforce import user"
2454 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2455 % (' '.join(self
.depotPaths
), revision
))
2456 details
["change"] = revision
2460 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2462 for info
in p4CmdList(["files"] + fileArgs
):
2464 if 'code' in info
and info
['code'] == 'error':
2465 sys
.stderr
.write("p4 returned an error: %s\n"
2467 if info
['data'].find("must refer to client") >= 0:
2468 sys
.stderr
.write("This particular p4 error is misleading.\n")
2469 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2470 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2472 if 'p4ExitCode' in info
:
2473 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2477 change
= int(info
["change"])
2478 if change
> newestRevision
:
2479 newestRevision
= change
2481 if info
["action"] in self
.delete_actions
:
2482 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2483 #fileCnt = fileCnt + 1
2486 for prop
in ["depotFile", "rev", "action", "type" ]:
2487 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2489 fileCnt
= fileCnt
+ 1
2491 details
["change"] = newestRevision
2493 # Use time from top-most change so that all git p4 clones of
2494 # the same p4 repo have the same commit SHA1s.
2495 res
= p4CmdList("describe -s %d" % newestRevision
)
2498 if r
.has_key('time'):
2499 newestTime
= int(r
['time'])
2500 if newestTime
is None:
2501 die("\"describe -s\" on newest change %d did not give a time")
2502 details
["time"] = newestTime
2504 self
.updateOptionDict(details
)
2506 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2508 print "IO error with git fast-import. Is your git version recent enough?"
2509 print self
.gitError
.read()
2512 def run(self
, args
):
2513 self
.depotPaths
= []
2514 self
.changeRange
= ""
2515 self
.initialParent
= ""
2516 self
.previousDepotPaths
= []
2518 # map from branch depot path to parent branch
2519 self
.knownBranches
= {}
2520 self
.initialParents
= {}
2521 self
.hasOrigin
= originP4BranchesExist()
2522 if not self
.syncWithOrigin
:
2523 self
.hasOrigin
= False
2525 if self
.importIntoRemotes
:
2526 self
.refPrefix
= "refs/remotes/p4/"
2528 self
.refPrefix
= "refs/heads/p4/"
2530 if self
.syncWithOrigin
and self
.hasOrigin
:
2532 print "Syncing with origin first by calling git fetch origin"
2533 system("git fetch origin")
2535 if len(self
.branch
) == 0:
2536 self
.branch
= self
.refPrefix
+ "master"
2537 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2538 system("git update-ref %s refs/heads/p4" % self
.branch
)
2539 system("git branch -D p4");
2540 # create it /after/ importing, when master exists
2541 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2542 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2544 # accept either the command-line option, or the configuration variable
2545 if self
.useClientSpec
:
2546 # will use this after clone to set the variable
2547 self
.useClientSpec_from_options
= True
2549 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2550 self
.useClientSpec
= True
2551 if self
.useClientSpec
:
2552 self
.clientSpecDirs
= getClientSpec()
2554 # TODO: should always look at previous commits,
2555 # merge with previous imports, if possible.
2558 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2559 self
.listExistingP4GitBranches()
2561 if len(self
.p4BranchesInGit
) > 1:
2563 print "Importing from/into multiple branches"
2564 self
.detectBranches
= True
2567 print "branches: %s" % self
.p4BranchesInGit
2570 for branch
in self
.p4BranchesInGit
:
2571 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2573 settings
= extractSettingsGitLog(logMsg
)
2575 self
.readOptions(settings
)
2576 if (settings
.has_key('depot-paths')
2577 and settings
.has_key ('change')):
2578 change
= int(settings
['change']) + 1
2579 p4Change
= max(p4Change
, change
)
2581 depotPaths
= sorted(settings
['depot-paths'])
2582 if self
.previousDepotPaths
== []:
2583 self
.previousDepotPaths
= depotPaths
2586 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2587 prev_list
= prev
.split("/")
2588 cur_list
= cur
.split("/")
2589 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2590 if cur_list
[i
] <> prev_list
[i
]:
2594 paths
.append ("/".join(cur_list
[:i
+ 1]))
2596 self
.previousDepotPaths
= paths
2599 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2600 self
.changeRange
= "@%s,#head" % p4Change
2601 if not self
.detectBranches
:
2602 self
.initialParent
= parseRevision(self
.branch
)
2603 if not self
.silent
and not self
.detectBranches
:
2604 print "Performing incremental import into %s git branch" % self
.branch
2606 if not self
.branch
.startswith("refs/"):
2607 self
.branch
= "refs/heads/" + self
.branch
2609 if len(args
) == 0 and self
.depotPaths
:
2611 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2613 if self
.depotPaths
and self
.depotPaths
!= args
:
2614 print ("previous import used depot path %s and now %s was specified. "
2615 "This doesn't work!" % (' '.join (self
.depotPaths
),
2619 self
.depotPaths
= sorted(args
)
2624 # Make sure no revision specifiers are used when --changesfile
2626 bad_changesfile
= False
2627 if len(self
.changesFile
) > 0:
2628 for p
in self
.depotPaths
:
2629 if p
.find("@") >= 0 or p
.find("#") >= 0:
2630 bad_changesfile
= True
2633 die("Option --changesfile is incompatible with revision specifiers")
2636 for p
in self
.depotPaths
:
2637 if p
.find("@") != -1:
2638 atIdx
= p
.index("@")
2639 self
.changeRange
= p
[atIdx
:]
2640 if self
.changeRange
== "@all":
2641 self
.changeRange
= ""
2642 elif ',' not in self
.changeRange
:
2643 revision
= self
.changeRange
2644 self
.changeRange
= ""
2646 elif p
.find("#") != -1:
2647 hashIdx
= p
.index("#")
2648 revision
= p
[hashIdx
:]
2650 elif self
.previousDepotPaths
== []:
2651 # pay attention to changesfile, if given, else import
2652 # the entire p4 tree at the head revision
2653 if len(self
.changesFile
) == 0:
2656 p
= re
.sub ("\.\.\.$", "", p
)
2657 if not p
.endswith("/"):
2662 self
.depotPaths
= newPaths
2664 self
.loadUserMapFromCache()
2666 if self
.detectLabels
:
2669 if self
.detectBranches
:
2670 ## FIXME - what's a P4 projectName ?
2671 self
.projectName
= self
.guessProjectName()
2674 self
.getBranchMappingFromGitBranches()
2676 self
.getBranchMapping()
2678 print "p4-git branches: %s" % self
.p4BranchesInGit
2679 print "initial parents: %s" % self
.initialParents
2680 for b
in self
.p4BranchesInGit
:
2684 b
= b
[len(self
.projectName
):]
2685 self
.createdBranches
.add(b
)
2687 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2689 importProcess
= subprocess
.Popen(["git", "fast-import"],
2690 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2691 stderr
=subprocess
.PIPE
);
2692 self
.gitOutput
= importProcess
.stdout
2693 self
.gitStream
= importProcess
.stdin
2694 self
.gitError
= importProcess
.stderr
2697 self
.importHeadRevision(revision
)
2701 if len(self
.changesFile
) > 0:
2702 output
= open(self
.changesFile
).readlines()
2705 changeSet
.add(int(line
))
2707 for change
in changeSet
:
2708 changes
.append(change
)
2712 # catch "git p4 sync" with no new branches, in a repo that
2713 # does not have any existing p4 branches
2714 if len(args
) == 0 and not self
.p4BranchesInGit
:
2715 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2717 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2719 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2721 if len(self
.maxChanges
) > 0:
2722 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2724 if len(changes
) == 0:
2726 print "No changes to import!"
2728 if not self
.silent
and not self
.detectBranches
:
2729 print "Import destination: %s" % self
.branch
2731 self
.updatedBranches
= set()
2733 self
.importChanges(changes
)
2737 if len(self
.updatedBranches
) > 0:
2738 sys
.stdout
.write("Updated branches: ")
2739 for b
in self
.updatedBranches
:
2740 sys
.stdout
.write("%s " % b
)
2741 sys
.stdout
.write("\n")
2743 if gitConfig("git-p4.importLabels", "--bool") == "true":
2744 self
.importLabels
= True
2746 if self
.importLabels
:
2747 p4Labels
= getP4Labels(self
.depotPaths
)
2748 gitTags
= getGitTags()
2750 missingP4Labels
= p4Labels
- gitTags
2751 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2753 self
.gitStream
.close()
2754 if importProcess
.wait() != 0:
2755 die("fast-import failed: %s" % self
.gitError
.read())
2756 self
.gitOutput
.close()
2757 self
.gitError
.close()
2759 # Cleanup temporary branches created during import
2760 if self
.tempBranches
!= []:
2761 for branch
in self
.tempBranches
:
2762 read_pipe("git update-ref -d %s" % branch
)
2763 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2767 class P4Rebase(Command
):
2769 Command
.__init
__(self
)
2771 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2773 self
.importLabels
= False
2774 self
.description
= ("Fetches the latest revision from perforce and "
2775 + "rebases the current work (branch) against it")
2777 def run(self
, args
):
2779 sync
.importLabels
= self
.importLabels
2782 return self
.rebase()
2785 if os
.system("git update-index --refresh") != 0:
2786 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2787 if len(read_pipe("git diff-index HEAD --")) > 0:
2788 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2790 [upstream
, settings
] = findUpstreamBranchPoint()
2791 if len(upstream
) == 0:
2792 die("Cannot find upstream branchpoint for rebase")
2794 # the branchpoint may be p4/foo~3, so strip off the parent
2795 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2797 print "Rebasing the current branch onto %s" % upstream
2798 oldHead
= read_pipe("git rev-parse HEAD").strip()
2799 system("git rebase %s" % upstream
)
2800 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2803 class P4Clone(P4Sync
):
2805 P4Sync
.__init
__(self
)
2806 self
.description
= "Creates a new git repository and imports from Perforce into it"
2807 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2809 optparse
.make_option("--destination", dest
="cloneDestination",
2810 action
='store', default
=None,
2811 help="where to leave result of the clone"),
2812 optparse
.make_option("-/", dest
="cloneExclude",
2813 action
="append", type="string",
2814 help="exclude depot path"),
2815 optparse
.make_option("--bare", dest
="cloneBare",
2816 action
="store_true", default
=False),
2818 self
.cloneDestination
= None
2819 self
.needsGit
= False
2820 self
.cloneBare
= False
2822 # This is required for the "append" cloneExclude action
2823 def ensure_value(self
, attr
, value
):
2824 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2825 setattr(self
, attr
, value
)
2826 return getattr(self
, attr
)
2828 def defaultDestination(self
, args
):
2829 ## TODO: use common prefix of args?
2831 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2832 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2833 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2834 depotDir
= re
.sub(r
"/$", "", depotDir
)
2835 return os
.path
.split(depotDir
)[1]
2837 def run(self
, args
):
2841 if self
.keepRepoPath
and not self
.cloneDestination
:
2842 sys
.stderr
.write("Must specify destination for --keep-path\n")
2847 if not self
.cloneDestination
and len(depotPaths
) > 1:
2848 self
.cloneDestination
= depotPaths
[-1]
2849 depotPaths
= depotPaths
[:-1]
2851 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2852 for p
in depotPaths
:
2853 if not p
.startswith("//"):
2856 if not self
.cloneDestination
:
2857 self
.cloneDestination
= self
.defaultDestination(args
)
2859 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2861 if not os
.path
.exists(self
.cloneDestination
):
2862 os
.makedirs(self
.cloneDestination
)
2863 chdir(self
.cloneDestination
)
2865 init_cmd
= [ "git", "init" ]
2867 init_cmd
.append("--bare")
2868 subprocess
.check_call(init_cmd
)
2870 if not P4Sync
.run(self
, depotPaths
):
2872 if self
.branch
!= "master":
2873 if self
.importIntoRemotes
:
2874 masterbranch
= "refs/remotes/p4/master"
2876 masterbranch
= "refs/heads/p4/master"
2877 if gitBranchExists(masterbranch
):
2878 system("git branch master %s" % masterbranch
)
2879 if not self
.cloneBare
:
2880 system("git checkout -f")
2882 print "Could not detect main branch. No checkout/master branch created."
2884 # auto-set this variable if invoked with --use-client-spec
2885 if self
.useClientSpec_from_options
:
2886 system("git config --bool git-p4.useclientspec true")
2890 class P4Branches(Command
):
2892 Command
.__init
__(self
)
2894 self
.description
= ("Shows the git branches that hold imports and their "
2895 + "corresponding perforce depot paths")
2896 self
.verbose
= False
2898 def run(self
, args
):
2899 if originP4BranchesExist():
2900 createOrUpdateBranchesFromOrigin()
2902 cmdline
= "git rev-parse --symbolic "
2903 cmdline
+= " --remotes"
2905 for line
in read_pipe_lines(cmdline
):
2908 if not line
.startswith('p4/') or line
== "p4/HEAD":
2912 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2913 settings
= extractSettingsGitLog(log
)
2915 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2918 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2920 optparse
.IndentedHelpFormatter
.__init
__(self
)
2922 def format_description(self
, description
):
2924 return description
+ "\n"
2928 def printUsage(commands
):
2929 print "usage: %s <command> [options]" % sys
.argv
[0]
2931 print "valid commands: %s" % ", ".join(commands
)
2933 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2938 "submit" : P4Submit
,
2939 "commit" : P4Submit
,
2941 "rebase" : P4Rebase
,
2943 "rollback" : P4RollBack
,
2944 "branches" : P4Branches
2949 if len(sys
.argv
[1:]) == 0:
2950 printUsage(commands
.keys())
2954 cmdName
= sys
.argv
[1]
2956 klass
= commands
[cmdName
]
2959 print "unknown command %s" % cmdName
2961 printUsage(commands
.keys())
2964 options
= cmd
.options
2965 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2969 options
.append(optparse
.make_option("--verbose", dest
="verbose", action
="store_true"))
2971 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2973 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2975 description
= cmd
.description
,
2976 formatter
= HelpFormatter())
2978 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2980 verbose
= cmd
.verbose
2982 if cmd
.gitdir
== None:
2983 cmd
.gitdir
= os
.path
.abspath(".git")
2984 if not isValidGitDir(cmd
.gitdir
):
2985 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2986 if os
.path
.exists(cmd
.gitdir
):
2987 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2991 if not isValidGitDir(cmd
.gitdir
):
2992 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2993 cmd
.gitdir
+= "/.git"
2995 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2997 os
.environ
["GIT_DIR"] = cmd
.gitdir
2999 if not cmd
.run(args
):
3004 if __name__
== '__main__':