3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
123 def p4_has_command(cmd
):
124 """Ask p4 for help on this command. If it returns an error, the
125 command does not exist in this version of p4."""
126 real_cmd
= p4_build_cmd(["help", cmd
])
127 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
128 stderr
=subprocess
.PIPE
)
130 return p
.returncode
== 0
133 expand
= isinstance(cmd
,basestring
)
135 sys
.stderr
.write("executing %s\n" % str(cmd
))
136 subprocess
.check_call(cmd
, shell
=expand
)
139 """Specifically invoke p4 as the system command. """
140 real_cmd
= p4_build_cmd(cmd
)
141 expand
= isinstance(real_cmd
, basestring
)
142 subprocess
.check_call(real_cmd
, shell
=expand
)
144 def p4_integrate(src
, dest
):
145 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
147 def p4_sync(f
, *options
):
148 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
151 # forcibly add file names with wildcards
152 if wildcard_present(f
):
153 p4_system(["add", "-f", f
])
155 p4_system(["add", f
])
158 p4_system(["delete", wildcard_encode(f
)])
161 p4_system(["edit", wildcard_encode(f
)])
164 p4_system(["revert", wildcard_encode(f
)])
166 def p4_reopen(type, f
):
167 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
169 def p4_move(src
, dest
):
170 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers. See "p4 help filetypes"
175 # for a list and explanation.
177 def split_p4_type(p4type
):
179 p4_filetypes_historical
= {
180 "ctempobj": "binary+Sw",
186 "tempobj": "binary+FSw",
187 "ubinary": "binary+F",
188 "uresource": "resource+F",
189 "uxbinary": "binary+Fx",
190 "xbinary": "binary+x",
192 "xtempobj": "binary+Swx",
194 "xunicode": "unicode+x",
197 if p4type
in p4_filetypes_historical
:
198 p4type
= p4_filetypes_historical
[p4type
]
200 s
= p4type
.split("+")
208 # return the raw p4 type of a file (text, text+ko, etc)
211 results
= p4CmdList(["fstat", "-T", "headType", file])
212 return results
[0]['headType']
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
218 def p4_keywords_regexp_for_type(base
, type_mods
):
219 if base
in ("text", "unicode", "binary"):
221 if "ko" in type_mods
:
223 elif "k" in type_mods
:
224 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
228 \$ # Starts with a dollar, followed by...
229 (%s) # one of the keywords, followed by...
230 (:[^$]+)? # possibly an old expansion, followed by...
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
242 def p4_keywords_regexp_for_file(file):
243 if not os
.path
.exists(file):
246 (type_base
, type_mods
) = split_p4_type(p4_type(file))
247 return p4_keywords_regexp_for_type(type_base
, type_mods
)
249 def setP4ExecBit(file, mode
):
250 # Reopens an already open file and changes the execute bit to match
251 # the execute bit setting in the passed in mode.
255 if not isModeExec(mode
):
256 p4Type
= getP4OpenedType(file)
257 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
258 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
259 if p4Type
[-1] == "+":
260 p4Type
= p4Type
[0:-1]
262 p4_reopen(p4Type
, file)
264 def getP4OpenedType(file):
265 # Returns the perforce file type for the given file.
267 result
= p4_read_pipe(["opened", wildcard_encode(file)])
268 match
= re
.match(".*\((.+)\)\r?$", result
)
270 return match
.group(1)
272 die("Could not determine file type for %s (result: '%s')" % (file, result
))
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths
):
277 if isinstance(depotPaths
,basestring
):
278 depotPaths
= [depotPaths
]
280 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
286 # Return the set of all git tags
289 for line
in read_pipe_lines(["git", "tag"]):
294 def diffTreePattern():
295 # This is a simple generator for the diff tree regex pattern. This could be
296 # a class variable if this and parseDiffTreeEntry were a part of a class.
297 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
301 def parseDiffTreeEntry(entry
):
302 """Parses a single diff tree entry into its component elements.
304 See git-diff-tree(1) manpage for details about the format of the diff
305 output. This method returns a dictionary with the following elements:
307 src_mode - The mode of the source file
308 dst_mode - The mode of the destination file
309 src_sha1 - The sha1 for the source file
310 dst_sha1 - The sha1 fr the destination file
311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312 status_score - The score for the status (applicable for 'C' and 'R'
313 statuses). This is None if there is no score.
314 src - The path for the source file.
315 dst - The path for the destination file. This is only present for
316 copy or renames. If it is not present, this is None.
318 If the pattern is not matched, None is returned."""
320 match
= diffTreePattern().next().match(entry
)
323 'src_mode': match
.group(1),
324 'dst_mode': match
.group(2),
325 'src_sha1': match
.group(3),
326 'dst_sha1': match
.group(4),
327 'status': match
.group(5),
328 'status_score': match
.group(6),
329 'src': match
.group(7),
330 'dst': match
.group(10)
334 def isModeExec(mode
):
335 # Returns True if the given git mode represents an executable file,
337 return mode
[-3:] == "755"
339 def isModeExecChanged(src_mode
, dst_mode
):
340 return isModeExec(src_mode
) != isModeExec(dst_mode
)
342 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
344 if isinstance(cmd
,basestring
):
351 cmd
= p4_build_cmd(cmd
)
353 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
355 # Use a temporary file to avoid deadlocks without
356 # subprocess.communicate(), which would put another copy
357 # of stdout into memory.
359 if stdin
is not None:
360 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
361 if isinstance(stdin
,basestring
):
362 stdin_file
.write(stdin
)
365 stdin_file
.write(i
+ '\n')
369 p4
= subprocess
.Popen(cmd
,
372 stdout
=subprocess
.PIPE
)
377 entry
= marshal
.load(p4
.stdout
)
387 entry
["p4ExitCode"] = exitCode
393 list = p4CmdList(cmd
)
399 def p4Where(depotPath
):
400 if not depotPath
.endswith("/"):
402 depotPath
= depotPath
+ "..."
403 outputList
= p4CmdList(["where", depotPath
])
405 for entry
in outputList
:
406 if "depotFile" in entry
:
407 if entry
["depotFile"] == depotPath
:
410 elif "data" in entry
:
411 data
= entry
.get("data")
412 space
= data
.find(" ")
413 if data
[:space
] == depotPath
:
418 if output
["code"] == "error":
422 clientPath
= output
.get("path")
423 elif "data" in output
:
424 data
= output
.get("data")
425 lastSpace
= data
.rfind(" ")
426 clientPath
= data
[lastSpace
+ 1:]
428 if clientPath
.endswith("..."):
429 clientPath
= clientPath
[:-3]
432 def currentGitBranch():
433 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
435 def isValidGitDir(path
):
436 if (os
.path
.exists(path
+ "/HEAD")
437 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
441 def parseRevision(ref
):
442 return read_pipe("git rev-parse %s" % ref
).strip()
444 def branchExists(ref
):
445 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
449 def extractLogMessageFromGitCommit(commit
):
452 ## fixme: title is first line of commit, not 1st paragraph.
454 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
463 def extractSettingsGitLog(log
):
465 for line
in log
.split("\n"):
467 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
471 assignments
= m
.group(1).split (':')
472 for a
in assignments
:
474 key
= vals
[0].strip()
475 val
= ('='.join (vals
[1:])).strip()
476 if val
.endswith ('\"') and val
.startswith('"'):
481 paths
= values
.get("depot-paths")
483 paths
= values
.get("depot-path")
485 values
['depot-paths'] = paths
.split(',')
488 def gitBranchExists(branch
):
489 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
490 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
491 return proc
.wait() == 0;
494 def gitConfig(key
, args
= None): # set args to "--bool", for instance
495 if not _gitConfig
.has_key(key
):
498 argsFilter
= "%s " % args
499 cmd
= "git config %s%s" % (argsFilter
, key
)
500 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
501 return _gitConfig
[key
]
503 def gitConfigList(key
):
504 if not _gitConfig
.has_key(key
):
505 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
506 return _gitConfig
[key
]
508 def p4BranchesInGit(branchesAreInRemotes
= True):
511 cmdline
= "git rev-parse --symbolic "
512 if branchesAreInRemotes
:
513 cmdline
+= " --remotes"
515 cmdline
+= " --branches"
517 for line
in read_pipe_lines(cmdline
):
520 ## only import to p4/
521 if not line
.startswith('p4/') or line
== "p4/HEAD":
526 branch
= re
.sub ("^p4/", "", line
)
528 branches
[branch
] = parseRevision(line
)
531 def findUpstreamBranchPoint(head
= "HEAD"):
532 branches
= p4BranchesInGit()
533 # map from depot-path to branch name
534 branchByDepotPath
= {}
535 for branch
in branches
.keys():
536 tip
= branches
[branch
]
537 log
= extractLogMessageFromGitCommit(tip
)
538 settings
= extractSettingsGitLog(log
)
539 if settings
.has_key("depot-paths"):
540 paths
= ",".join(settings
["depot-paths"])
541 branchByDepotPath
[paths
] = "remotes/p4/" + branch
545 while parent
< 65535:
546 commit
= head
+ "~%s" % parent
547 log
= extractLogMessageFromGitCommit(commit
)
548 settings
= extractSettingsGitLog(log
)
549 if settings
.has_key("depot-paths"):
550 paths
= ",".join(settings
["depot-paths"])
551 if branchByDepotPath
.has_key(paths
):
552 return [branchByDepotPath
[paths
], settings
]
556 return ["", settings
]
558 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
560 print ("Creating/updating branch(es) in %s based on origin branch(es)"
563 originPrefix
= "origin/p4/"
565 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
567 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
570 headName
= line
[len(originPrefix
):]
571 remoteHead
= localRefPrefix
+ headName
574 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
575 if (not original
.has_key('depot-paths')
576 or not original
.has_key('change')):
580 if not gitBranchExists(remoteHead
):
582 print "creating %s" % remoteHead
585 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
586 if settings
.has_key('change') > 0:
587 if settings
['depot-paths'] == original
['depot-paths']:
588 originP4Change
= int(original
['change'])
589 p4Change
= int(settings
['change'])
590 if originP4Change
> p4Change
:
591 print ("%s (%s) is newer than %s (%s). "
592 "Updating p4 branch from origin."
593 % (originHead
, originP4Change
,
594 remoteHead
, p4Change
))
597 print ("Ignoring: %s was imported from %s while "
598 "%s was imported from %s"
599 % (originHead
, ','.join(original
['depot-paths']),
600 remoteHead
, ','.join(settings
['depot-paths'])))
603 system("git update-ref %s %s" % (remoteHead
, originHead
))
605 def originP4BranchesExist():
606 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
608 def p4ChangesForPaths(depotPaths
, changeRange
):
612 cmd
+= ["%s...%s" % (p
, changeRange
)]
613 output
= p4_read_pipe_lines(cmd
)
617 changeNum
= int(line
.split(" ")[1])
618 changes
[changeNum
] = True
620 changelist
= changes
.keys()
624 def p4PathStartsWith(path
, prefix
):
625 # This method tries to remedy a potential mixed-case issue:
627 # If UserA adds //depot/DirA/file1
628 # and UserB adds //depot/dira/file2
630 # we may or may not have a problem. If you have core.ignorecase=true,
631 # we treat DirA and dira as the same directory
632 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
634 return path
.lower().startswith(prefix
.lower())
635 return path
.startswith(prefix
)
638 """Look at the p4 client spec, create a View() object that contains
639 all the mappings, and return it."""
641 specList
= p4CmdList("client -o")
642 if len(specList
) != 1:
643 die('Output from "client -o" is %d lines, expecting 1' %
646 # dictionary of all client parameters
649 # just the keys that start with "View"
650 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
655 # append the lines, in order, to the view
656 for view_num
in range(len(view_keys
)):
657 k
= "View%d" % view_num
658 if k
not in view_keys
:
659 die("Expected view key %s missing" % k
)
660 view
.append(entry
[k
])
665 """Grab the client directory."""
667 output
= p4CmdList("client -o")
669 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
672 if "Root" not in entry
:
673 die('Client has no "Root"')
678 # P4 wildcards are not allowed in filenames. P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
682 def wildcard_decode(path
):
683 # Search for and fix just these four characters. Do % last so
684 # that fixing it does not inadvertently create new %-escapes.
685 # Cannot have * in a filename in windows; untested as to
686 # what p4 would do in such a case.
687 if not platform
.system() == "Windows":
688 path
= path
.replace("%2A", "*")
689 path
= path
.replace("%23", "#") \
690 .replace("%40", "@") \
694 def wildcard_encode(path
):
695 # do % first to avoid double-encoding the %s introduced here
696 path
= path
.replace("%", "%25") \
697 .replace("*", "%2A") \
698 .replace("#", "%23") \
702 def wildcard_present(path
):
703 return path
.translate(None, "*#@%") != path
707 self
.usage
= "usage: %prog [options]"
713 self
.userMapFromPerforceServer
= False
714 self
.myP4UserId
= None
718 return self
.myP4UserId
720 results
= p4CmdList("user -o")
722 if r
.has_key('User'):
723 self
.myP4UserId
= r
['User']
725 die("Could not find your p4 user id")
727 def p4UserIsMe(self
, p4User
):
728 # return True if the given p4 user is actually me
730 if not p4User
or p4User
!= me
:
735 def getUserCacheFilename(self
):
736 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
737 return home
+ "/.gitp4-usercache.txt"
739 def getUserMapFromPerforceServer(self
):
740 if self
.userMapFromPerforceServer
:
745 for output
in p4CmdList("users"):
746 if not output
.has_key("User"):
748 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
749 self
.emails
[output
["Email"]] = output
["User"]
753 for (key
, val
) in self
.users
.items():
754 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
756 open(self
.getUserCacheFilename(), "wb").write(s
)
757 self
.userMapFromPerforceServer
= True
759 def loadUserMapFromCache(self
):
761 self
.userMapFromPerforceServer
= False
763 cache
= open(self
.getUserCacheFilename(), "rb")
764 lines
= cache
.readlines()
767 entry
= line
.strip().split("\t")
768 self
.users
[entry
[0]] = entry
[1]
770 self
.getUserMapFromPerforceServer()
772 class P4Debug(Command
):
774 Command
.__init
__(self
)
776 self
.description
= "A tool to debug the output of p4 -G."
777 self
.needsGit
= False
781 for output
in p4CmdList(args
):
782 print 'Element: %d' % j
787 class P4RollBack(Command
):
789 Command
.__init
__(self
)
791 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
793 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
794 self
.rollbackLocalBranches
= False
799 maxChange
= int(args
[0])
801 if "p4ExitCode" in p4Cmd("changes -m 1"):
802 die("Problems executing p4");
804 if self
.rollbackLocalBranches
:
805 refPrefix
= "refs/heads/"
806 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
808 refPrefix
= "refs/remotes/"
809 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
812 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
814 ref
= refPrefix
+ line
815 log
= extractLogMessageFromGitCommit(ref
)
816 settings
= extractSettingsGitLog(log
)
818 depotPaths
= settings
['depot-paths']
819 change
= settings
['change']
823 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
824 for p
in depotPaths
]))) == 0:
825 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
826 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
829 while change
and int(change
) > maxChange
:
832 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
833 system("git update-ref %s \"%s^\"" % (ref
, ref
))
834 log
= extractLogMessageFromGitCommit(ref
)
835 settings
= extractSettingsGitLog(log
)
838 depotPaths
= settings
['depot-paths']
839 change
= settings
['change']
842 print "%s rewound to %s" % (ref
, change
)
846 class P4Submit(Command
, P4UserMap
):
848 Command
.__init
__(self
)
849 P4UserMap
.__init
__(self
)
851 optparse
.make_option("--origin", dest
="origin"),
852 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
853 # preserve the user, requires relevant p4 permissions
854 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
855 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
857 self
.description
= "Submit changes from git to the perforce depot."
858 self
.usage
+= " [name of git branch to submit into perforce depot]"
860 self
.detectRenames
= False
861 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
862 self
.isWindows
= (platform
.system() == "Windows")
863 self
.exportLabels
= False
864 self
.p4HasMoveCommand
= p4_has_command("move")
867 if len(p4CmdList("opened ...")) > 0:
868 die("You have files opened with perforce! Close them before starting the sync.")
870 def separate_jobs_from_description(self
, message
):
871 """Extract and return a possible Jobs field in the commit
872 message. It goes into a separate section in the p4 change
875 A jobs line starts with "Jobs:" and looks like a new field
876 in a form. Values are white-space separated on the same
877 line or on following lines that start with a tab.
879 This does not parse and extract the full git commit message
880 like a p4 form. It just sees the Jobs: line as a marker
881 to pass everything from then on directly into the p4 form,
882 but outside the description section.
884 Return a tuple (stripped log message, jobs string)."""
886 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
888 return (message
, None)
890 jobtext
= message
[m
.start():]
891 stripped_message
= message
[:m
.start()].rstrip()
892 return (stripped_message
, jobtext
)
894 def prepareLogMessage(self
, template
, message
, jobs
):
895 """Edits the template returned from "p4 change -o" to insert
896 the message in the Description field, and the jobs text in
900 inDescriptionSection
= False
902 for line
in template
.split("\n"):
903 if line
.startswith("#"):
904 result
+= line
+ "\n"
907 if inDescriptionSection
:
908 if line
.startswith("Files:") or line
.startswith("Jobs:"):
909 inDescriptionSection
= False
910 # insert Jobs section
912 result
+= jobs
+ "\n"
916 if line
.startswith("Description:"):
917 inDescriptionSection
= True
919 for messageLine
in message
.split("\n"):
920 line
+= "\t" + messageLine
+ "\n"
922 result
+= line
+ "\n"
926 def patchRCSKeywords(self
, file, pattern
):
927 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
928 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
930 outFile
= os
.fdopen(handle
, "w+")
931 inFile
= open(file, "r")
932 regexp
= re
.compile(pattern
, re
.VERBOSE
)
933 for line
in inFile
.readlines():
934 line
= regexp
.sub(r
'$\1$', line
)
938 # Forcibly overwrite the original file
940 shutil
.move(outFileName
, file)
942 # cleanup our temporary file
943 os
.unlink(outFileName
)
944 print "Failed to strip RCS keywords in %s" % file
947 print "Patched up RCS keywords in %s" % file
949 def p4UserForCommit(self
,id):
950 # Return the tuple (perforce user,git email) for a given git commit id
951 self
.getUserMapFromPerforceServer()
952 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
953 gitEmail
= gitEmail
.strip()
954 if not self
.emails
.has_key(gitEmail
):
955 return (None,gitEmail
)
957 return (self
.emails
[gitEmail
],gitEmail
)
959 def checkValidP4Users(self
,commits
):
960 # check if any git authors cannot be mapped to p4 users
962 (user
,email
) = self
.p4UserForCommit(id)
964 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
965 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
968 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
970 def lastP4Changelist(self
):
971 # Get back the last changelist number submitted in this client spec. This
972 # then gets used to patch up the username in the change. If the same
973 # client spec is being used by multiple processes then this might go
975 results
= p4CmdList("client -o") # find the current client
978 if r
.has_key('Client'):
982 die("could not get client spec")
983 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
985 if r
.has_key('change'):
987 die("Could not get changelist number for last submit - cannot patch up user details")
989 def modifyChangelistUser(self
, changelist
, newUser
):
990 # fixup the user field of a changelist after it has been submitted.
991 changes
= p4CmdList("change -o %s" % changelist
)
992 if len(changes
) != 1:
993 die("Bad output from p4 change modifying %s to user %s" %
994 (changelist
, newUser
))
997 if c
['User'] == newUser
: return # nothing to do
999 input = marshal
.dumps(c
)
1001 result
= p4CmdList("change -f -i", stdin
=input)
1003 if r
.has_key('code'):
1004 if r
['code'] == 'error':
1005 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1006 if r
.has_key('data'):
1007 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1009 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1011 def canChangeChangelists(self
):
1012 # check to see if we have p4 admin or super-user permissions, either of
1013 # which are required to modify changelists.
1014 results
= p4CmdList(["protects", self
.depotPath
])
1016 if r
.has_key('perm'):
1017 if r
['perm'] == 'admin':
1019 if r
['perm'] == 'super':
1023 def prepareSubmitTemplate(self
):
1024 """Run "p4 change -o" to grab a change specification template.
1025 This does not use "p4 -G", as it is nice to keep the submission
1026 template in original order, since a human might edit it.
1028 Remove lines in the Files section that show changes to files
1029 outside the depot path we're committing into."""
1032 inFilesSection
= False
1033 for line
in p4_read_pipe_lines(['change', '-o']):
1034 if line
.endswith("\r\n"):
1035 line
= line
[:-2] + "\n"
1037 if line
.startswith("\t"):
1038 # path starts and ends with a tab
1040 lastTab
= path
.rfind("\t")
1042 path
= path
[:lastTab
]
1043 if not p4PathStartsWith(path
, self
.depotPath
):
1046 inFilesSection
= False
1048 if line
.startswith("Files:"):
1049 inFilesSection
= True
1055 def edit_template(self
, template_file
):
1056 """Invoke the editor to let the user change the submission
1057 message. Return true if okay to continue with the submit."""
1059 # if configured to skip the editing part, just submit
1060 if gitConfig("git-p4.skipSubmitEdit") == "true":
1063 # look at the modification time, to check later if the user saved
1065 mtime
= os
.stat(template_file
).st_mtime
1068 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1069 editor
= os
.environ
.get("P4EDITOR")
1071 editor
= read_pipe("git var GIT_EDITOR").strip()
1072 system(editor
+ " " + template_file
)
1074 # If the file was not saved, prompt to see if this patch should
1075 # be skipped. But skip this verification step if configured so.
1076 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1079 # modification time updated means user saved the file
1080 if os
.stat(template_file
).st_mtime
> mtime
:
1084 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1090 def applyCommit(self
, id):
1091 """Apply one commit, return True if it succeeded."""
1093 print "Applying", read_pipe(["git", "show", "-s",
1094 "--format=format:%h %s", id])
1096 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1098 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1100 filesToDelete
= set()
1102 pureRenameCopy
= set()
1103 filesToChangeExecBit
= {}
1106 diff
= parseDiffTreeEntry(line
)
1107 modifier
= diff
['status']
1111 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1112 filesToChangeExecBit
[path
] = diff
['dst_mode']
1113 editedFiles
.add(path
)
1114 elif modifier
== "A":
1115 filesToAdd
.add(path
)
1116 filesToChangeExecBit
[path
] = diff
['dst_mode']
1117 if path
in filesToDelete
:
1118 filesToDelete
.remove(path
)
1119 elif modifier
== "D":
1120 filesToDelete
.add(path
)
1121 if path
in filesToAdd
:
1122 filesToAdd
.remove(path
)
1123 elif modifier
== "C":
1124 src
, dest
= diff
['src'], diff
['dst']
1125 p4_integrate(src
, dest
)
1126 pureRenameCopy
.add(dest
)
1127 if diff
['src_sha1'] != diff
['dst_sha1']:
1129 pureRenameCopy
.discard(dest
)
1130 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1132 pureRenameCopy
.discard(dest
)
1133 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1135 editedFiles
.add(dest
)
1136 elif modifier
== "R":
1137 src
, dest
= diff
['src'], diff
['dst']
1138 if self
.p4HasMoveCommand
:
1139 p4_edit(src
) # src must be open before move
1140 p4_move(src
, dest
) # opens for (move/delete, move/add)
1142 p4_integrate(src
, dest
)
1143 if diff
['src_sha1'] != diff
['dst_sha1']:
1146 pureRenameCopy
.add(dest
)
1147 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1148 if not self
.p4HasMoveCommand
:
1149 p4_edit(dest
) # with move: already open, writable
1150 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1151 if not self
.p4HasMoveCommand
:
1153 filesToDelete
.add(src
)
1154 editedFiles
.add(dest
)
1156 die("unknown modifier %s for %s" % (modifier
, path
))
1158 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1159 patchcmd
= diffcmd
+ " | git apply "
1160 tryPatchCmd
= patchcmd
+ "--check -"
1161 applyPatchCmd
= patchcmd
+ "--check --apply -"
1162 patch_succeeded
= True
1164 if os
.system(tryPatchCmd
) != 0:
1165 fixed_rcs_keywords
= False
1166 patch_succeeded
= False
1167 print "Unfortunately applying the change failed!"
1169 # Patch failed, maybe it's just RCS keyword woes. Look through
1170 # the patch to see if that's possible.
1171 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1175 for file in editedFiles | filesToDelete
:
1176 # did this file's delta contain RCS keywords?
1177 pattern
= p4_keywords_regexp_for_file(file)
1180 # this file is a possibility...look for RCS keywords.
1181 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1182 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1183 if regexp
.search(line
):
1185 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1186 kwfiles
[file] = pattern
1189 for file in kwfiles
:
1191 print "zapping %s with %s" % (line
,pattern
)
1192 self
.patchRCSKeywords(file, kwfiles
[file])
1193 fixed_rcs_keywords
= True
1195 if fixed_rcs_keywords
:
1196 print "Retrying the patch with RCS keywords cleaned up"
1197 if os
.system(tryPatchCmd
) == 0:
1198 patch_succeeded
= True
1200 if not patch_succeeded
:
1201 print "What do you want to do?"
1203 while response
!= "s" and response
!= "a" and response
!= "w":
1204 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1205 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1207 print "Skipping! Good luck with the next patches..."
1208 for f
in editedFiles
:
1210 for f
in filesToAdd
:
1213 elif response
== "a":
1214 os
.system(applyPatchCmd
)
1215 if len(filesToAdd
) > 0:
1216 print "You may also want to call p4 add on the following files:"
1217 print " ".join(filesToAdd
)
1218 if len(filesToDelete
):
1219 print "The following files should be scheduled for deletion with p4 delete:"
1220 print " ".join(filesToDelete
)
1221 die("Please resolve and submit the conflict manually and "
1222 + "continue afterwards with git p4 submit --continue")
1223 elif response
== "w":
1224 system(diffcmd
+ " > patch.txt")
1225 print "Patch saved to patch.txt in %s !" % self
.clientPath
1226 die("Please resolve and submit the conflict manually and "
1227 "continue afterwards with git p4 submit --continue")
1229 system(applyPatchCmd
)
1231 for f
in filesToAdd
:
1233 for f
in filesToDelete
:
1237 # Set/clear executable bits
1238 for f
in filesToChangeExecBit
.keys():
1239 mode
= filesToChangeExecBit
[f
]
1240 setP4ExecBit(f
, mode
)
1242 logMessage
= extractLogMessageFromGitCommit(id)
1243 logMessage
= logMessage
.strip()
1244 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1246 template
= self
.prepareSubmitTemplate()
1247 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1249 if self
.preserveUser
:
1250 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1252 if os
.environ
.has_key("P4DIFF"):
1253 del(os
.environ
["P4DIFF"])
1255 for editedFile
in editedFiles
:
1256 diff
+= p4_read_pipe(['diff', '-du',
1257 wildcard_encode(editedFile
)])
1260 for newFile
in filesToAdd
:
1261 newdiff
+= "==== new file ====\n"
1262 newdiff
+= "--- /dev/null\n"
1263 newdiff
+= "+++ %s\n" % newFile
1264 f
= open(newFile
, "r")
1265 for line
in f
.readlines():
1266 newdiff
+= "+" + line
1269 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1270 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1271 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1272 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1274 separatorLine
= "######## everything below this line is just the diff #######\n"
1276 (handle
, fileName
) = tempfile
.mkstemp()
1277 tmpFile
= os
.fdopen(handle
, "w+")
1279 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1280 separatorLine
= separatorLine
.replace("\n", "\r\n")
1281 newdiff
= newdiff
.replace("\n", "\r\n")
1282 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1285 if self
.edit_template(fileName
):
1286 # read the edited message and submit
1287 tmpFile
= open(fileName
, "rb")
1288 message
= tmpFile
.read()
1290 submitTemplate
= message
[:message
.index(separatorLine
)]
1292 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1293 p4_write_pipe(['submit', '-i'], submitTemplate
)
1295 if self
.preserveUser
:
1297 # Get last changelist number. Cannot easily get it from
1298 # the submit command output as the output is
1300 changelist
= self
.lastP4Changelist()
1301 self
.modifyChangelistUser(changelist
, p4User
)
1303 # The rename/copy happened by applying a patch that created a
1304 # new file. This leaves it writable, which confuses p4.
1305 for f
in pureRenameCopy
:
1310 print "Submission cancelled, undoing p4 changes."
1311 for f
in editedFiles
:
1313 for f
in filesToAdd
:
1318 return True # success
1320 # Export git tags as p4 labels. Create a p4 label and then tag
1322 def exportGitTags(self
, gitTags
):
1323 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1324 if len(validLabelRegexp
) == 0:
1325 validLabelRegexp
= defaultLabelRegexp
1326 m
= re
.compile(validLabelRegexp
)
1328 for name
in gitTags
:
1330 if not m
.match(name
):
1332 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1335 # Get the p4 commit this corresponds to
1336 logMessage
= extractLogMessageFromGitCommit(name
)
1337 values
= extractSettingsGitLog(logMessage
)
1339 if not values
.has_key('change'):
1340 # a tag pointing to something not sent to p4; ignore
1342 print "git tag %s does not give a p4 commit" % name
1345 changelist
= values
['change']
1347 # Get the tag details.
1351 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1354 if re
.match(r
'tag\s+', l
):
1356 elif re
.match(r
'\s*$', l
):
1363 body
= ["lightweight tag imported by git p4\n"]
1365 # Create the label - use the same view as the client spec we are using
1366 clientSpec
= getClientSpec()
1368 labelTemplate
= "Label: %s\n" % name
1369 labelTemplate
+= "Description:\n"
1371 labelTemplate
+= "\t" + b
+ "\n"
1372 labelTemplate
+= "View:\n"
1373 for mapping
in clientSpec
.mappings
:
1374 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1376 p4_write_pipe(["label", "-i"], labelTemplate
)
1379 p4_system(["tag", "-l", name
] +
1380 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1383 print "created p4 label for tag %s" % name
1385 def run(self
, args
):
1387 self
.master
= currentGitBranch()
1388 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1389 die("Detecting current git branch failed!")
1390 elif len(args
) == 1:
1391 self
.master
= args
[0]
1392 if not branchExists(self
.master
):
1393 die("Branch %s does not exist" % self
.master
)
1397 allowSubmit
= gitConfig("git-p4.allowSubmit")
1398 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1399 die("%s is not in git-p4.allowSubmit" % self
.master
)
1401 [upstream
, settings
] = findUpstreamBranchPoint()
1402 self
.depotPath
= settings
['depot-paths'][0]
1403 if len(self
.origin
) == 0:
1404 self
.origin
= upstream
1406 if self
.preserveUser
:
1407 if not self
.canChangeChangelists():
1408 die("Cannot preserve user names without p4 super-user or admin permissions")
1411 print "Origin branch is " + self
.origin
1413 if len(self
.depotPath
) == 0:
1414 print "Internal error: cannot locate perforce depot path from existing branches"
1417 self
.useClientSpec
= False
1418 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1419 self
.useClientSpec
= True
1420 if self
.useClientSpec
:
1421 self
.clientSpecDirs
= getClientSpec()
1423 if self
.useClientSpec
:
1424 # all files are relative to the client spec
1425 self
.clientPath
= getClientRoot()
1427 self
.clientPath
= p4Where(self
.depotPath
)
1429 if self
.clientPath
== "":
1430 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1432 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1433 self
.oldWorkingDirectory
= os
.getcwd()
1435 # ensure the clientPath exists
1436 new_client_dir
= False
1437 if not os
.path
.exists(self
.clientPath
):
1438 new_client_dir
= True
1439 os
.makedirs(self
.clientPath
)
1441 chdir(self
.clientPath
)
1442 print "Synchronizing p4 checkout..."
1444 # old one was destroyed, and maybe nobody told p4
1445 p4_sync("...", "-f")
1451 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1452 commits
.append(line
.strip())
1455 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1456 self
.checkAuthorship
= False
1458 self
.checkAuthorship
= True
1460 if self
.preserveUser
:
1461 self
.checkValidP4Users(commits
)
1464 # Build up a set of options to be passed to diff when
1465 # submitting each commit to p4.
1467 if self
.detectRenames
:
1468 # command-line -M arg
1469 self
.diffOpts
= "-M"
1471 # If not explicitly set check the config variable
1472 detectRenames
= gitConfig("git-p4.detectRenames")
1474 if detectRenames
.lower() == "false" or detectRenames
== "":
1476 elif detectRenames
.lower() == "true":
1477 self
.diffOpts
= "-M"
1479 self
.diffOpts
= "-M%s" % detectRenames
1481 # no command-line arg for -C or --find-copies-harder, just
1483 detectCopies
= gitConfig("git-p4.detectCopies")
1484 if detectCopies
.lower() == "false" or detectCopies
== "":
1486 elif detectCopies
.lower() == "true":
1487 self
.diffOpts
+= " -C"
1489 self
.diffOpts
+= " -C%s" % detectCopies
1491 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1492 self
.diffOpts
+= " --find-copies-harder"
1495 for commit
in commits
:
1496 ok
= self
.applyCommit(commit
)
1498 applied
.append(commit
)
1500 chdir(self
.oldWorkingDirectory
)
1502 if len(commits
) == len(applied
):
1503 print "All commits applied!"
1512 if len(applied
) == 0:
1513 print "No commits applied."
1515 print "Applied only the commits marked with '*':"
1521 print star
, read_pipe(["git", "show", "-s",
1522 "--format=format:%h %s", c
])
1523 print "You will have to do 'git p4 sync' and rebase."
1525 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1526 self
.exportLabels
= True
1528 if self
.exportLabels
:
1529 p4Labels
= getP4Labels(self
.depotPath
)
1530 gitTags
= getGitTags()
1532 missingGitTags
= gitTags
- p4Labels
1533 self
.exportGitTags(missingGitTags
)
1535 # exit with error unless everything applied perfecly
1536 if len(commits
) != len(applied
):
1542 """Represent a p4 view ("p4 help views"), and map files in a
1543 repo according to the view."""
1546 """A depot or client path, possibly containing wildcards.
1547 The only one supported is ... at the end, currently.
1548 Initialize with the full path, with //depot or //client."""
1550 def __init__(self
, path
, is_depot
):
1552 self
.is_depot
= is_depot
1553 self
.find_wildcards()
1554 # remember the prefix bit, useful for relative mappings
1555 m
= re
.match("(//[^/]+/)", self
.path
)
1557 die("Path %s does not start with //prefix/" % self
.path
)
1559 if not self
.is_depot
:
1560 # strip //client/ on client paths
1561 self
.path
= self
.path
[len(prefix
):]
1563 def find_wildcards(self
):
1564 """Make sure wildcards are valid, and set up internal
1567 self
.ends_triple_dot
= False
1568 # There are three wildcards allowed in p4 views
1569 # (see "p4 help views"). This code knows how to
1570 # handle "..." (only at the end), but cannot deal with
1571 # "%%n" or "*". Only check the depot_side, as p4 should
1572 # validate that the client_side matches too.
1573 if re
.search(r
'%%[1-9]', self
.path
):
1574 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1575 if self
.path
.find("*") >= 0:
1576 die("Can't handle * wildcards in view: %s" % self
.path
)
1577 triple_dot_index
= self
.path
.find("...")
1578 if triple_dot_index
>= 0:
1579 if triple_dot_index
!= len(self
.path
) - 3:
1580 die("Can handle only single ... wildcard, at end: %s" %
1582 self
.ends_triple_dot
= True
1584 def ensure_compatible(self
, other_path
):
1585 """Make sure the wildcards agree."""
1586 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1587 die("Both paths must end with ... if either does;\n" +
1588 "paths: %s %s" % (self
.path
, other_path
.path
))
1590 def match_wildcards(self
, test_path
):
1591 """See if this test_path matches us, and fill in the value
1592 of the wildcards if so. Returns a tuple of
1593 (True|False, wildcards[]). For now, only the ... at end
1594 is supported, so at most one wildcard."""
1595 if self
.ends_triple_dot
:
1596 dotless
= self
.path
[:-3]
1597 if test_path
.startswith(dotless
):
1598 wildcard
= test_path
[len(dotless
):]
1599 return (True, [ wildcard
])
1601 if test_path
== self
.path
:
1605 def match(self
, test_path
):
1606 """Just return if it matches; don't bother with the wildcards."""
1607 b
, _
= self
.match_wildcards(test_path
)
1610 def fill_in_wildcards(self
, wildcards
):
1611 """Return the relative path, with the wildcards filled in
1612 if there are any."""
1613 if self
.ends_triple_dot
:
1614 return self
.path
[:-3] + wildcards
[0]
1618 class Mapping(object):
1619 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1620 # depot_side is without the trailing /... if it had one
1621 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1622 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1623 self
.overlay
= overlay
# started with "+"
1624 self
.exclude
= exclude
# started with "-"
1625 assert not (self
.overlay
and self
.exclude
)
1626 self
.depot_side
.ensure_compatible(self
.client_side
)
1634 return "View.Mapping: %s%s -> %s" % \
1635 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1637 def map_depot_to_client(self
, depot_path
):
1638 """Calculate the client path if using this mapping on the
1639 given depot path; does not consider the effect of other
1640 mappings in a view. Even excluded mappings are returned."""
1641 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1644 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1653 def append(self
, view_line
):
1654 """Parse a view line, splitting it into depot and client
1655 sides. Append to self.mappings, preserving order."""
1657 # Split the view line into exactly two words. P4 enforces
1658 # structure on these lines that simplifies this quite a bit.
1660 # Either or both words may be double-quoted.
1661 # Single quotes do not matter.
1662 # Double-quote marks cannot occur inside the words.
1663 # A + or - prefix is also inside the quotes.
1664 # There are no quotes unless they contain a space.
1665 # The line is already white-space stripped.
1666 # The two words are separated by a single space.
1668 if view_line
[0] == '"':
1669 # First word is double quoted. Find its end.
1670 close_quote_index
= view_line
.find('"', 1)
1671 if close_quote_index
<= 0:
1672 die("No first-word closing quote found: %s" % view_line
)
1673 depot_side
= view_line
[1:close_quote_index
]
1674 # skip closing quote and space
1675 rhs_index
= close_quote_index
+ 1 + 1
1677 space_index
= view_line
.find(" ")
1678 if space_index
<= 0:
1679 die("No word-splitting space found: %s" % view_line
)
1680 depot_side
= view_line
[0:space_index
]
1681 rhs_index
= space_index
+ 1
1683 if view_line
[rhs_index
] == '"':
1684 # Second word is double quoted. Make sure there is a
1685 # double quote at the end too.
1686 if not view_line
.endswith('"'):
1687 die("View line with rhs quote should end with one: %s" %
1690 client_side
= view_line
[rhs_index
+1:-1]
1692 client_side
= view_line
[rhs_index
:]
1694 # prefix + means overlay on previous mapping
1696 if depot_side
.startswith("+"):
1698 depot_side
= depot_side
[1:]
1700 # prefix - means exclude this path
1702 if depot_side
.startswith("-"):
1704 depot_side
= depot_side
[1:]
1706 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1707 self
.mappings
.append(m
)
1709 def map_in_client(self
, depot_path
):
1710 """Return the relative location in the client where this
1711 depot file should live. Returns "" if the file should
1712 not be mapped in the client."""
1717 # look at later entries first
1718 for m
in self
.mappings
[::-1]:
1720 # see where will this path end up in the client
1721 p
= m
.map_depot_to_client(depot_path
)
1724 # Depot path does not belong in client. Must remember
1725 # this, as previous items should not cause files to
1726 # exist in this path either. Remember that the list is
1727 # being walked from the end, which has higher precedence.
1728 # Overlap mappings do not exclude previous mappings.
1730 paths_filled
.append(m
.client_side
)
1733 # This mapping matched; no need to search any further.
1734 # But, the mapping could be rejected if the client path
1735 # has already been claimed by an earlier mapping (i.e.
1736 # one later in the list, which we are walking backwards).
1737 already_mapped_in_client
= False
1738 for f
in paths_filled
:
1739 # this is View.Path.match
1741 already_mapped_in_client
= True
1743 if not already_mapped_in_client
:
1744 # Include this file, unless it is from a line that
1745 # explicitly said to exclude it.
1749 # a match, even if rejected, always stops the search
1754 class P4Sync(Command
, P4UserMap
):
1755 delete_actions
= ( "delete", "move/delete", "purge" )
1758 Command
.__init
__(self
)
1759 P4UserMap
.__init
__(self
)
1761 optparse
.make_option("--branch", dest
="branch"),
1762 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1763 optparse
.make_option("--changesfile", dest
="changesFile"),
1764 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1765 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1766 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1767 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1768 help="Import into refs/heads/ , not refs/remotes"),
1769 optparse
.make_option("--max-changes", dest
="maxChanges"),
1770 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1771 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1772 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1773 help="Only sync files that are included in the Perforce Client Spec")
1775 self
.description
= """Imports from Perforce into a git repository.\n
1777 //depot/my/project/ -- to import the current head
1778 //depot/my/project/@all -- to import everything
1779 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1781 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1783 self
.usage
+= " //depot/path[@revRange]"
1785 self
.createdBranches
= set()
1786 self
.committedChanges
= set()
1788 self
.detectBranches
= False
1789 self
.detectLabels
= False
1790 self
.importLabels
= False
1791 self
.changesFile
= ""
1792 self
.syncWithOrigin
= True
1793 self
.importIntoRemotes
= True
1794 self
.maxChanges
= ""
1795 self
.isWindows
= (platform
.system() == "Windows")
1796 self
.keepRepoPath
= False
1797 self
.depotPaths
= None
1798 self
.p4BranchesInGit
= []
1799 self
.cloneExclude
= []
1800 self
.useClientSpec
= False
1801 self
.useClientSpec_from_options
= False
1802 self
.clientSpecDirs
= None
1803 self
.tempBranches
= []
1804 self
.tempBranchLocation
= "git-p4-tmp"
1806 if gitConfig("git-p4.syncFromOrigin") == "false":
1807 self
.syncWithOrigin
= False
1809 # Force a checkpoint in fast-import and wait for it to finish
1810 def checkpoint(self
):
1811 self
.gitStream
.write("checkpoint\n\n")
1812 self
.gitStream
.write("progress checkpoint\n\n")
1813 out
= self
.gitOutput
.readline()
1815 print "checkpoint finished: " + out
1817 def extractFilesFromCommit(self
, commit
):
1818 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1819 for path
in self
.cloneExclude
]
1822 while commit
.has_key("depotFile%s" % fnum
):
1823 path
= commit
["depotFile%s" % fnum
]
1825 if [p
for p
in self
.cloneExclude
1826 if p4PathStartsWith(path
, p
)]:
1829 found
= [p
for p
in self
.depotPaths
1830 if p4PathStartsWith(path
, p
)]
1837 file["rev"] = commit
["rev%s" % fnum
]
1838 file["action"] = commit
["action%s" % fnum
]
1839 file["type"] = commit
["type%s" % fnum
]
1844 def stripRepoPath(self
, path
, prefixes
):
1845 if self
.useClientSpec
:
1846 return self
.clientSpecDirs
.map_in_client(path
)
1848 if self
.keepRepoPath
:
1849 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1852 if p4PathStartsWith(path
, p
):
1853 path
= path
[len(p
):]
1857 def splitFilesIntoBranches(self
, commit
):
1860 while commit
.has_key("depotFile%s" % fnum
):
1861 path
= commit
["depotFile%s" % fnum
]
1862 found
= [p
for p
in self
.depotPaths
1863 if p4PathStartsWith(path
, p
)]
1870 file["rev"] = commit
["rev%s" % fnum
]
1871 file["action"] = commit
["action%s" % fnum
]
1872 file["type"] = commit
["type%s" % fnum
]
1875 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1876 relPath
= wildcard_decode(relPath
)
1878 for branch
in self
.knownBranches
.keys():
1880 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1881 if relPath
.startswith(branch
+ "/"):
1882 if branch
not in branches
:
1883 branches
[branch
] = []
1884 branches
[branch
].append(file)
1889 # output one file from the P4 stream
1890 # - helper for streamP4Files
1892 def streamOneP4File(self
, file, contents
):
1893 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1894 relPath
= wildcard_decode(relPath
)
1896 sys
.stderr
.write("%s\n" % relPath
)
1898 (type_base
, type_mods
) = split_p4_type(file["type"])
1901 if "x" in type_mods
:
1903 if type_base
== "symlink":
1905 # p4 print on a symlink contains "target\n"; remove the newline
1906 data
= ''.join(contents
)
1907 contents
= [data
[:-1]]
1909 if type_base
== "utf16":
1910 # p4 delivers different text in the python output to -G
1911 # than it does when using "print -o", or normal p4 client
1912 # operations. utf16 is converted to ascii or utf8, perhaps.
1913 # But ascii text saved as -t utf16 is completely mangled.
1914 # Invoke print -o to get the real contents.
1915 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1918 if type_base
== "apple":
1919 # Apple filetype files will be streamed as a concatenation of
1920 # its appledouble header and the contents. This is useless
1921 # on both macs and non-macs. If using "print -q -o xx", it
1922 # will create "xx" with the data, and "%xx" with the header.
1923 # This is also not very useful.
1925 # Ideally, someday, this script can learn how to generate
1926 # appledouble files directly and import those to git, but
1927 # non-mac machines can never find a use for apple filetype.
1928 print "\nIgnoring apple filetype file %s" % file['depotFile']
1931 # Perhaps windows wants unicode, utf16 newlines translated too;
1932 # but this is not doing it.
1933 if self
.isWindows
and type_base
== "text":
1935 for data
in contents
:
1936 data
= data
.replace("\r\n", "\n")
1937 mangled
.append(data
)
1940 # Note that we do not try to de-mangle keywords on utf16 files,
1941 # even though in theory somebody may want that.
1942 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1944 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1945 text
= ''.join(contents
)
1946 text
= regexp
.sub(r
'$\1$', text
)
1949 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1954 length
= length
+ len(d
)
1956 self
.gitStream
.write("data %d\n" % length
)
1958 self
.gitStream
.write(d
)
1959 self
.gitStream
.write("\n")
1961 def streamOneP4Deletion(self
, file):
1962 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1963 relPath
= wildcard_decode(relPath
)
1965 sys
.stderr
.write("delete %s\n" % relPath
)
1966 self
.gitStream
.write("D %s\n" % relPath
)
1968 # handle another chunk of streaming data
1969 def streamP4FilesCb(self
, marshalled
):
1971 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1972 # start of a new file - output the old one first
1973 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1974 self
.stream_file
= {}
1975 self
.stream_contents
= []
1976 self
.stream_have_file_info
= False
1978 # pick up the new file information... for the
1979 # 'data' field we need to append to our array
1980 for k
in marshalled
.keys():
1982 self
.stream_contents
.append(marshalled
['data'])
1984 self
.stream_file
[k
] = marshalled
[k
]
1986 self
.stream_have_file_info
= True
1988 # Stream directly from "p4 files" into "git fast-import"
1989 def streamP4Files(self
, files
):
1995 # if using a client spec, only add the files that have
1996 # a path in the client
1997 if self
.clientSpecDirs
:
1998 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
2001 filesForCommit
.append(f
)
2002 if f
['action'] in self
.delete_actions
:
2003 filesToDelete
.append(f
)
2005 filesToRead
.append(f
)
2008 for f
in filesToDelete
:
2009 self
.streamOneP4Deletion(f
)
2011 if len(filesToRead
) > 0:
2012 self
.stream_file
= {}
2013 self
.stream_contents
= []
2014 self
.stream_have_file_info
= False
2016 # curry self argument
2017 def streamP4FilesCbSelf(entry
):
2018 self
.streamP4FilesCb(entry
)
2020 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2022 p4CmdList(["-x", "-", "print"],
2024 cb
=streamP4FilesCbSelf
)
2027 if self
.stream_file
.has_key('depotFile'):
2028 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2030 def make_email(self
, userid
):
2031 if userid
in self
.users
:
2032 return self
.users
[userid
]
2034 return "%s <a@b>" % userid
2037 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2039 print "writing tag %s for commit %s" % (labelName
, commit
)
2040 gitStream
.write("tag %s\n" % labelName
)
2041 gitStream
.write("from %s\n" % commit
)
2043 if labelDetails
.has_key('Owner'):
2044 owner
= labelDetails
["Owner"]
2048 # Try to use the owner of the p4 label, or failing that,
2049 # the current p4 user id.
2051 email
= self
.make_email(owner
)
2053 email
= self
.make_email(self
.p4UserId())
2054 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2056 gitStream
.write("tagger %s\n" % tagger
)
2058 print "labelDetails=",labelDetails
2059 if labelDetails
.has_key('Description'):
2060 description
= labelDetails
['Description']
2062 description
= 'Label from git p4'
2064 gitStream
.write("data %d\n" % len(description
))
2065 gitStream
.write(description
)
2066 gitStream
.write("\n")
2068 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
2069 epoch
= details
["time"]
2070 author
= details
["user"]
2071 self
.branchPrefixes
= branchPrefixes
2074 print "commit into %s" % branch
2076 # start with reading files; if that fails, we should not
2080 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2081 new_files
.append (f
)
2083 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2085 self
.gitStream
.write("commit %s\n" % branch
)
2086 # gitStream.write("mark :%s\n" % details["change"])
2087 self
.committedChanges
.add(int(details
["change"]))
2089 if author
not in self
.users
:
2090 self
.getUserMapFromPerforceServer()
2091 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2093 self
.gitStream
.write("committer %s\n" % committer
)
2095 self
.gitStream
.write("data <<EOT\n")
2096 self
.gitStream
.write(details
["desc"])
2097 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2098 % (','.join (branchPrefixes
), details
["change"]))
2099 if len(details
['options']) > 0:
2100 self
.gitStream
.write(": options = %s" % details
['options'])
2101 self
.gitStream
.write("]\nEOT\n\n")
2105 print "parent %s" % parent
2106 self
.gitStream
.write("from %s\n" % parent
)
2108 self
.streamP4Files(new_files
)
2109 self
.gitStream
.write("\n")
2111 change
= int(details
["change"])
2113 if self
.labels
.has_key(change
):
2114 label
= self
.labels
[change
]
2115 labelDetails
= label
[0]
2116 labelRevisions
= label
[1]
2118 print "Change %s is labelled %s" % (change
, labelDetails
)
2120 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2121 for p
in branchPrefixes
])
2123 if len(files
) == len(labelRevisions
):
2127 if info
["action"] in self
.delete_actions
:
2129 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2131 if cleanedFiles
== labelRevisions
:
2132 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2136 print ("Tag %s does not match with change %s: files do not match."
2137 % (labelDetails
["label"], change
))
2141 print ("Tag %s does not match with change %s: file count is different."
2142 % (labelDetails
["label"], change
))
2144 # Build a dictionary of changelists and labels, for "detect-labels" option.
2145 def getLabels(self
):
2148 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2149 if len(l
) > 0 and not self
.silent
:
2150 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2153 label
= output
["label"]
2157 print "Querying files for label %s" % label
2158 for file in p4CmdList(["files"] +
2159 ["%s...@%s" % (p
, label
)
2160 for p
in self
.depotPaths
]):
2161 revisions
[file["depotFile"]] = file["rev"]
2162 change
= int(file["change"])
2163 if change
> newestChange
:
2164 newestChange
= change
2166 self
.labels
[newestChange
] = [output
, revisions
]
2169 print "Label changes: %s" % self
.labels
.keys()
2171 # Import p4 labels as git tags. A direct mapping does not
2172 # exist, so assume that if all the files are at the same revision
2173 # then we can use that, or it's something more complicated we should
2175 def importP4Labels(self
, stream
, p4Labels
):
2177 print "import p4 labels: " + ' '.join(p4Labels
)
2179 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2180 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2181 if len(validLabelRegexp
) == 0:
2182 validLabelRegexp
= defaultLabelRegexp
2183 m
= re
.compile(validLabelRegexp
)
2185 for name
in p4Labels
:
2188 if not m
.match(name
):
2190 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2193 if name
in ignoredP4Labels
:
2196 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2198 # get the most recent changelist for each file in this label
2199 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2200 for p
in self
.depotPaths
])
2202 if change
.has_key('change'):
2203 # find the corresponding git commit; take the oldest commit
2204 changelist
= int(change
['change'])
2205 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2206 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2207 if len(gitCommit
) == 0:
2208 print "could not find git commit for changelist %d" % changelist
2210 gitCommit
= gitCommit
.strip()
2212 # Convert from p4 time format
2214 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2216 print "Could not convert label time %s" % labelDetail
['Update']
2219 when
= int(time
.mktime(tmwhen
))
2220 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2222 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2225 print "Label %s has no changelists - possibly deleted?" % name
2228 # We can't import this label; don't try again as it will get very
2229 # expensive repeatedly fetching all the files for labels that will
2230 # never be imported. If the label is moved in the future, the
2231 # ignore will need to be removed manually.
2232 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2234 def guessProjectName(self
):
2235 for p
in self
.depotPaths
:
2238 p
= p
[p
.strip().rfind("/") + 1:]
2239 if not p
.endswith("/"):
2243 def getBranchMapping(self
):
2244 lostAndFoundBranches
= set()
2246 user
= gitConfig("git-p4.branchUser")
2248 command
= "branches -u %s" % user
2250 command
= "branches"
2252 for info
in p4CmdList(command
):
2253 details
= p4Cmd(["branch", "-o", info
["branch"]])
2255 while details
.has_key("View%s" % viewIdx
):
2256 paths
= details
["View%s" % viewIdx
].split(" ")
2257 viewIdx
= viewIdx
+ 1
2258 # require standard //depot/foo/... //depot/bar/... mapping
2259 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2262 destination
= paths
[1]
2264 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2265 source
= source
[len(self
.depotPaths
[0]):-4]
2266 destination
= destination
[len(self
.depotPaths
[0]):-4]
2268 if destination
in self
.knownBranches
:
2270 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2271 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2274 self
.knownBranches
[destination
] = source
2276 lostAndFoundBranches
.discard(destination
)
2278 if source
not in self
.knownBranches
:
2279 lostAndFoundBranches
.add(source
)
2281 # Perforce does not strictly require branches to be defined, so we also
2282 # check git config for a branch list.
2284 # Example of branch definition in git config file:
2286 # branchList=main:branchA
2287 # branchList=main:branchB
2288 # branchList=branchA:branchC
2289 configBranches
= gitConfigList("git-p4.branchList")
2290 for branch
in configBranches
:
2292 (source
, destination
) = branch
.split(":")
2293 self
.knownBranches
[destination
] = source
2295 lostAndFoundBranches
.discard(destination
)
2297 if source
not in self
.knownBranches
:
2298 lostAndFoundBranches
.add(source
)
2301 for branch
in lostAndFoundBranches
:
2302 self
.knownBranches
[branch
] = branch
2304 def getBranchMappingFromGitBranches(self
):
2305 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2306 for branch
in branches
.keys():
2307 if branch
== "master":
2310 branch
= branch
[len(self
.projectName
):]
2311 self
.knownBranches
[branch
] = branch
2313 def listExistingP4GitBranches(self
):
2314 # branches holds mapping from name to commit
2315 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2316 self
.p4BranchesInGit
= branches
.keys()
2317 for branch
in branches
.keys():
2318 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2320 def updateOptionDict(self
, d
):
2322 if self
.keepRepoPath
:
2323 option_keys
['keepRepoPath'] = 1
2325 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2327 def readOptions(self
, d
):
2328 self
.keepRepoPath
= (d
.has_key('options')
2329 and ('keepRepoPath' in d
['options']))
2331 def gitRefForBranch(self
, branch
):
2332 if branch
== "main":
2333 return self
.refPrefix
+ "master"
2335 if len(branch
) <= 0:
2338 return self
.refPrefix
+ self
.projectName
+ branch
2340 def gitCommitByP4Change(self
, ref
, change
):
2342 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2345 latestCommit
= parseRevision(ref
)
2349 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2350 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2355 log
= extractLogMessageFromGitCommit(next
)
2356 settings
= extractSettingsGitLog(log
)
2357 currentChange
= int(settings
['change'])
2359 print "current change %s" % currentChange
2361 if currentChange
== change
:
2363 print "found %s" % next
2366 if currentChange
< change
:
2367 earliestCommit
= "^%s" % next
2369 latestCommit
= "%s" % next
2373 def importNewBranch(self
, branch
, maxChange
):
2374 # make fast-import flush all changes to disk and update the refs using the checkpoint
2375 # command so that we can try to find the branch parent in the git history
2376 self
.gitStream
.write("checkpoint\n\n");
2377 self
.gitStream
.flush();
2378 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2379 range = "@1,%s" % maxChange
2380 #print "prefix" + branchPrefix
2381 changes
= p4ChangesForPaths([branchPrefix
], range)
2382 if len(changes
) <= 0:
2384 firstChange
= changes
[0]
2385 #print "first change in branch: %s" % firstChange
2386 sourceBranch
= self
.knownBranches
[branch
]
2387 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2388 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2389 #print "source " + sourceBranch
2391 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2392 #print "branch parent: %s" % branchParentChange
2393 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2394 if len(gitParent
) > 0:
2395 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2396 #print "parent git commit: %s" % gitParent
2398 self
.importChanges(changes
)
2401 def searchParent(self
, parent
, branch
, target
):
2403 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2405 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2408 print "Found parent of %s in commit %s" % (branch
, blob
)
2415 def importChanges(self
, changes
):
2417 for change
in changes
:
2418 description
= p4Cmd(["describe", str(change
)])
2419 self
.updateOptionDict(description
)
2422 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2427 if self
.detectBranches
:
2428 branches
= self
.splitFilesIntoBranches(description
)
2429 for branch
in branches
.keys():
2431 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2435 filesForCommit
= branches
[branch
]
2438 print "branch is %s" % branch
2440 self
.updatedBranches
.add(branch
)
2442 if branch
not in self
.createdBranches
:
2443 self
.createdBranches
.add(branch
)
2444 parent
= self
.knownBranches
[branch
]
2445 if parent
== branch
:
2448 fullBranch
= self
.projectName
+ branch
2449 if fullBranch
not in self
.p4BranchesInGit
:
2451 print("\n Importing new branch %s" % fullBranch
);
2452 if self
.importNewBranch(branch
, change
- 1):
2454 self
.p4BranchesInGit
.append(fullBranch
)
2456 print("\n Resuming with change %s" % change
);
2459 print "parent determined through known branches: %s" % parent
2461 branch
= self
.gitRefForBranch(branch
)
2462 parent
= self
.gitRefForBranch(parent
)
2465 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2467 if len(parent
) == 0 and branch
in self
.initialParents
:
2468 parent
= self
.initialParents
[branch
]
2469 del self
.initialParents
[branch
]
2473 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2475 print "Creating temporary branch: " + tempBranch
2476 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2477 self
.tempBranches
.append(tempBranch
)
2479 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2481 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2484 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2485 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2487 files
= self
.extractFilesFromCommit(description
)
2488 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2490 self
.initialParent
= ""
2492 print self
.gitError
.read()
2495 def importHeadRevision(self
, revision
):
2496 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2499 details
["user"] = "git perforce import user"
2500 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2501 % (' '.join(self
.depotPaths
), revision
))
2502 details
["change"] = revision
2506 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2508 for info
in p4CmdList(["files"] + fileArgs
):
2510 if 'code' in info
and info
['code'] == 'error':
2511 sys
.stderr
.write("p4 returned an error: %s\n"
2513 if info
['data'].find("must refer to client") >= 0:
2514 sys
.stderr
.write("This particular p4 error is misleading.\n")
2515 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2516 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2518 if 'p4ExitCode' in info
:
2519 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2523 change
= int(info
["change"])
2524 if change
> newestRevision
:
2525 newestRevision
= change
2527 if info
["action"] in self
.delete_actions
:
2528 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2529 #fileCnt = fileCnt + 1
2532 for prop
in ["depotFile", "rev", "action", "type" ]:
2533 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2535 fileCnt
= fileCnt
+ 1
2537 details
["change"] = newestRevision
2539 # Use time from top-most change so that all git p4 clones of
2540 # the same p4 repo have the same commit SHA1s.
2541 res
= p4CmdList("describe -s %d" % newestRevision
)
2544 if r
.has_key('time'):
2545 newestTime
= int(r
['time'])
2546 if newestTime
is None:
2547 die("\"describe -s\" on newest change %d did not give a time")
2548 details
["time"] = newestTime
2550 self
.updateOptionDict(details
)
2552 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2554 print "IO error with git fast-import. Is your git version recent enough?"
2555 print self
.gitError
.read()
2558 def run(self
, args
):
2559 self
.depotPaths
= []
2560 self
.changeRange
= ""
2561 self
.initialParent
= ""
2562 self
.previousDepotPaths
= []
2564 # map from branch depot path to parent branch
2565 self
.knownBranches
= {}
2566 self
.initialParents
= {}
2567 self
.hasOrigin
= originP4BranchesExist()
2568 if not self
.syncWithOrigin
:
2569 self
.hasOrigin
= False
2571 if self
.importIntoRemotes
:
2572 self
.refPrefix
= "refs/remotes/p4/"
2574 self
.refPrefix
= "refs/heads/p4/"
2576 if self
.syncWithOrigin
and self
.hasOrigin
:
2578 print "Syncing with origin first by calling git fetch origin"
2579 system("git fetch origin")
2581 if len(self
.branch
) == 0:
2582 self
.branch
= self
.refPrefix
+ "master"
2583 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2584 system("git update-ref %s refs/heads/p4" % self
.branch
)
2585 system("git branch -D p4");
2586 # create it /after/ importing, when master exists
2587 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2588 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2590 # accept either the command-line option, or the configuration variable
2591 if self
.useClientSpec
:
2592 # will use this after clone to set the variable
2593 self
.useClientSpec_from_options
= True
2595 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2596 self
.useClientSpec
= True
2597 if self
.useClientSpec
:
2598 self
.clientSpecDirs
= getClientSpec()
2600 # TODO: should always look at previous commits,
2601 # merge with previous imports, if possible.
2604 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2605 self
.listExistingP4GitBranches()
2607 if len(self
.p4BranchesInGit
) > 1:
2609 print "Importing from/into multiple branches"
2610 self
.detectBranches
= True
2613 print "branches: %s" % self
.p4BranchesInGit
2616 for branch
in self
.p4BranchesInGit
:
2617 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2619 settings
= extractSettingsGitLog(logMsg
)
2621 self
.readOptions(settings
)
2622 if (settings
.has_key('depot-paths')
2623 and settings
.has_key ('change')):
2624 change
= int(settings
['change']) + 1
2625 p4Change
= max(p4Change
, change
)
2627 depotPaths
= sorted(settings
['depot-paths'])
2628 if self
.previousDepotPaths
== []:
2629 self
.previousDepotPaths
= depotPaths
2632 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2633 prev_list
= prev
.split("/")
2634 cur_list
= cur
.split("/")
2635 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2636 if cur_list
[i
] <> prev_list
[i
]:
2640 paths
.append ("/".join(cur_list
[:i
+ 1]))
2642 self
.previousDepotPaths
= paths
2645 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2646 self
.changeRange
= "@%s,#head" % p4Change
2647 if not self
.detectBranches
:
2648 self
.initialParent
= parseRevision(self
.branch
)
2649 if not self
.silent
and not self
.detectBranches
:
2650 print "Performing incremental import into %s git branch" % self
.branch
2652 if not self
.branch
.startswith("refs/"):
2653 self
.branch
= "refs/heads/" + self
.branch
2655 if len(args
) == 0 and self
.depotPaths
:
2657 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2659 if self
.depotPaths
and self
.depotPaths
!= args
:
2660 print ("previous import used depot path %s and now %s was specified. "
2661 "This doesn't work!" % (' '.join (self
.depotPaths
),
2665 self
.depotPaths
= sorted(args
)
2670 # Make sure no revision specifiers are used when --changesfile
2672 bad_changesfile
= False
2673 if len(self
.changesFile
) > 0:
2674 for p
in self
.depotPaths
:
2675 if p
.find("@") >= 0 or p
.find("#") >= 0:
2676 bad_changesfile
= True
2679 die("Option --changesfile is incompatible with revision specifiers")
2682 for p
in self
.depotPaths
:
2683 if p
.find("@") != -1:
2684 atIdx
= p
.index("@")
2685 self
.changeRange
= p
[atIdx
:]
2686 if self
.changeRange
== "@all":
2687 self
.changeRange
= ""
2688 elif ',' not in self
.changeRange
:
2689 revision
= self
.changeRange
2690 self
.changeRange
= ""
2692 elif p
.find("#") != -1:
2693 hashIdx
= p
.index("#")
2694 revision
= p
[hashIdx
:]
2696 elif self
.previousDepotPaths
== []:
2697 # pay attention to changesfile, if given, else import
2698 # the entire p4 tree at the head revision
2699 if len(self
.changesFile
) == 0:
2702 p
= re
.sub ("\.\.\.$", "", p
)
2703 if not p
.endswith("/"):
2708 self
.depotPaths
= newPaths
2710 self
.loadUserMapFromCache()
2712 if self
.detectLabels
:
2715 if self
.detectBranches
:
2716 ## FIXME - what's a P4 projectName ?
2717 self
.projectName
= self
.guessProjectName()
2720 self
.getBranchMappingFromGitBranches()
2722 self
.getBranchMapping()
2724 print "p4-git branches: %s" % self
.p4BranchesInGit
2725 print "initial parents: %s" % self
.initialParents
2726 for b
in self
.p4BranchesInGit
:
2730 b
= b
[len(self
.projectName
):]
2731 self
.createdBranches
.add(b
)
2733 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2735 importProcess
= subprocess
.Popen(["git", "fast-import"],
2736 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2737 stderr
=subprocess
.PIPE
);
2738 self
.gitOutput
= importProcess
.stdout
2739 self
.gitStream
= importProcess
.stdin
2740 self
.gitError
= importProcess
.stderr
2743 self
.importHeadRevision(revision
)
2747 if len(self
.changesFile
) > 0:
2748 output
= open(self
.changesFile
).readlines()
2751 changeSet
.add(int(line
))
2753 for change
in changeSet
:
2754 changes
.append(change
)
2758 # catch "git p4 sync" with no new branches, in a repo that
2759 # does not have any existing p4 branches
2760 if len(args
) == 0 and not self
.p4BranchesInGit
:
2761 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2763 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2765 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2767 if len(self
.maxChanges
) > 0:
2768 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2770 if len(changes
) == 0:
2772 print "No changes to import!"
2774 if not self
.silent
and not self
.detectBranches
:
2775 print "Import destination: %s" % self
.branch
2777 self
.updatedBranches
= set()
2779 self
.importChanges(changes
)
2783 if len(self
.updatedBranches
) > 0:
2784 sys
.stdout
.write("Updated branches: ")
2785 for b
in self
.updatedBranches
:
2786 sys
.stdout
.write("%s " % b
)
2787 sys
.stdout
.write("\n")
2789 if gitConfig("git-p4.importLabels", "--bool") == "true":
2790 self
.importLabels
= True
2792 if self
.importLabels
:
2793 p4Labels
= getP4Labels(self
.depotPaths
)
2794 gitTags
= getGitTags()
2796 missingP4Labels
= p4Labels
- gitTags
2797 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2799 self
.gitStream
.close()
2800 if importProcess
.wait() != 0:
2801 die("fast-import failed: %s" % self
.gitError
.read())
2802 self
.gitOutput
.close()
2803 self
.gitError
.close()
2805 # Cleanup temporary branches created during import
2806 if self
.tempBranches
!= []:
2807 for branch
in self
.tempBranches
:
2808 read_pipe("git update-ref -d %s" % branch
)
2809 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2813 class P4Rebase(Command
):
2815 Command
.__init
__(self
)
2817 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2819 self
.importLabels
= False
2820 self
.description
= ("Fetches the latest revision from perforce and "
2821 + "rebases the current work (branch) against it")
2823 def run(self
, args
):
2825 sync
.importLabels
= self
.importLabels
2828 return self
.rebase()
2831 if os
.system("git update-index --refresh") != 0:
2832 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2833 if len(read_pipe("git diff-index HEAD --")) > 0:
2834 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2836 [upstream
, settings
] = findUpstreamBranchPoint()
2837 if len(upstream
) == 0:
2838 die("Cannot find upstream branchpoint for rebase")
2840 # the branchpoint may be p4/foo~3, so strip off the parent
2841 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2843 print "Rebasing the current branch onto %s" % upstream
2844 oldHead
= read_pipe("git rev-parse HEAD").strip()
2845 system("git rebase %s" % upstream
)
2846 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2849 class P4Clone(P4Sync
):
2851 P4Sync
.__init
__(self
)
2852 self
.description
= "Creates a new git repository and imports from Perforce into it"
2853 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2855 optparse
.make_option("--destination", dest
="cloneDestination",
2856 action
='store', default
=None,
2857 help="where to leave result of the clone"),
2858 optparse
.make_option("-/", dest
="cloneExclude",
2859 action
="append", type="string",
2860 help="exclude depot path"),
2861 optparse
.make_option("--bare", dest
="cloneBare",
2862 action
="store_true", default
=False),
2864 self
.cloneDestination
= None
2865 self
.needsGit
= False
2866 self
.cloneBare
= False
2868 # This is required for the "append" cloneExclude action
2869 def ensure_value(self
, attr
, value
):
2870 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2871 setattr(self
, attr
, value
)
2872 return getattr(self
, attr
)
2874 def defaultDestination(self
, args
):
2875 ## TODO: use common prefix of args?
2877 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2878 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2879 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2880 depotDir
= re
.sub(r
"/$", "", depotDir
)
2881 return os
.path
.split(depotDir
)[1]
2883 def run(self
, args
):
2887 if self
.keepRepoPath
and not self
.cloneDestination
:
2888 sys
.stderr
.write("Must specify destination for --keep-path\n")
2893 if not self
.cloneDestination
and len(depotPaths
) > 1:
2894 self
.cloneDestination
= depotPaths
[-1]
2895 depotPaths
= depotPaths
[:-1]
2897 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2898 for p
in depotPaths
:
2899 if not p
.startswith("//"):
2902 if not self
.cloneDestination
:
2903 self
.cloneDestination
= self
.defaultDestination(args
)
2905 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2907 if not os
.path
.exists(self
.cloneDestination
):
2908 os
.makedirs(self
.cloneDestination
)
2909 chdir(self
.cloneDestination
)
2911 init_cmd
= [ "git", "init" ]
2913 init_cmd
.append("--bare")
2914 subprocess
.check_call(init_cmd
)
2916 if not P4Sync
.run(self
, depotPaths
):
2918 if self
.branch
!= "master":
2919 if self
.importIntoRemotes
:
2920 masterbranch
= "refs/remotes/p4/master"
2922 masterbranch
= "refs/heads/p4/master"
2923 if gitBranchExists(masterbranch
):
2924 system("git branch master %s" % masterbranch
)
2925 if not self
.cloneBare
:
2926 system("git checkout -f")
2928 print "Could not detect main branch. No checkout/master branch created."
2930 # auto-set this variable if invoked with --use-client-spec
2931 if self
.useClientSpec_from_options
:
2932 system("git config --bool git-p4.useclientspec true")
2936 class P4Branches(Command
):
2938 Command
.__init
__(self
)
2940 self
.description
= ("Shows the git branches that hold imports and their "
2941 + "corresponding perforce depot paths")
2942 self
.verbose
= False
2944 def run(self
, args
):
2945 if originP4BranchesExist():
2946 createOrUpdateBranchesFromOrigin()
2948 cmdline
= "git rev-parse --symbolic "
2949 cmdline
+= " --remotes"
2951 for line
in read_pipe_lines(cmdline
):
2954 if not line
.startswith('p4/') or line
== "p4/HEAD":
2958 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2959 settings
= extractSettingsGitLog(log
)
2961 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2964 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2966 optparse
.IndentedHelpFormatter
.__init
__(self
)
2968 def format_description(self
, description
):
2970 return description
+ "\n"
2974 def printUsage(commands
):
2975 print "usage: %s <command> [options]" % sys
.argv
[0]
2977 print "valid commands: %s" % ", ".join(commands
)
2979 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2984 "submit" : P4Submit
,
2985 "commit" : P4Submit
,
2987 "rebase" : P4Rebase
,
2989 "rollback" : P4RollBack
,
2990 "branches" : P4Branches
2995 if len(sys
.argv
[1:]) == 0:
2996 printUsage(commands
.keys())
3000 cmdName
= sys
.argv
[1]
3002 klass
= commands
[cmdName
]
3005 print "unknown command %s" % cmdName
3007 printUsage(commands
.keys())
3010 options
= cmd
.options
3011 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3015 options
.append(optparse
.make_option("--verbose", dest
="verbose", action
="store_true"))
3017 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3019 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3021 description
= cmd
.description
,
3022 formatter
= HelpFormatter())
3024 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3026 verbose
= cmd
.verbose
3028 if cmd
.gitdir
== None:
3029 cmd
.gitdir
= os
.path
.abspath(".git")
3030 if not isValidGitDir(cmd
.gitdir
):
3031 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3032 if os
.path
.exists(cmd
.gitdir
):
3033 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3037 if not isValidGitDir(cmd
.gitdir
):
3038 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3039 cmd
.gitdir
+= "/.git"
3041 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3043 os
.environ
["GIT_DIR"] = cmd
.gitdir
3045 if not cmd
.run(args
):
3050 if __name__
== '__main__':