3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
123 def p4_has_command(cmd
):
124 """Ask p4 for help on this command. If it returns an error, the
125 command does not exist in this version of p4."""
126 real_cmd
= p4_build_cmd(["help", cmd
])
127 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
128 stderr
=subprocess
.PIPE
)
130 return p
.returncode
== 0
133 expand
= isinstance(cmd
,basestring
)
135 sys
.stderr
.write("executing %s\n" % str(cmd
))
136 subprocess
.check_call(cmd
, shell
=expand
)
139 """Specifically invoke p4 as the system command. """
140 real_cmd
= p4_build_cmd(cmd
)
141 expand
= isinstance(real_cmd
, basestring
)
142 subprocess
.check_call(real_cmd
, shell
=expand
)
144 def p4_integrate(src
, dest
):
145 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
147 def p4_sync(f
, *options
):
148 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
151 # forcibly add file names with wildcards
152 if wildcard_present(f
):
153 p4_system(["add", "-f", f
])
155 p4_system(["add", f
])
158 p4_system(["delete", wildcard_encode(f
)])
161 p4_system(["edit", wildcard_encode(f
)])
164 p4_system(["revert", wildcard_encode(f
)])
166 def p4_reopen(type, f
):
167 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
169 def p4_move(src
, dest
):
170 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers. See "p4 help filetypes"
175 # for a list and explanation.
177 def split_p4_type(p4type
):
179 p4_filetypes_historical
= {
180 "ctempobj": "binary+Sw",
186 "tempobj": "binary+FSw",
187 "ubinary": "binary+F",
188 "uresource": "resource+F",
189 "uxbinary": "binary+Fx",
190 "xbinary": "binary+x",
192 "xtempobj": "binary+Swx",
194 "xunicode": "unicode+x",
197 if p4type
in p4_filetypes_historical
:
198 p4type
= p4_filetypes_historical
[p4type
]
200 s
= p4type
.split("+")
208 # return the raw p4 type of a file (text, text+ko, etc)
211 results
= p4CmdList(["fstat", "-T", "headType", file])
212 return results
[0]['headType']
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
218 def p4_keywords_regexp_for_type(base
, type_mods
):
219 if base
in ("text", "unicode", "binary"):
221 if "ko" in type_mods
:
223 elif "k" in type_mods
:
224 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
228 \$ # Starts with a dollar, followed by...
229 (%s) # one of the keywords, followed by...
230 (:[^$]+)? # possibly an old expansion, followed by...
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
242 def p4_keywords_regexp_for_file(file):
243 if not os
.path
.exists(file):
246 (type_base
, type_mods
) = split_p4_type(p4_type(file))
247 return p4_keywords_regexp_for_type(type_base
, type_mods
)
249 def setP4ExecBit(file, mode
):
250 # Reopens an already open file and changes the execute bit to match
251 # the execute bit setting in the passed in mode.
255 if not isModeExec(mode
):
256 p4Type
= getP4OpenedType(file)
257 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
258 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
259 if p4Type
[-1] == "+":
260 p4Type
= p4Type
[0:-1]
262 p4_reopen(p4Type
, file)
264 def getP4OpenedType(file):
265 # Returns the perforce file type for the given file.
267 result
= p4_read_pipe(["opened", wildcard_encode(file)])
268 match
= re
.match(".*\((.+)\)\r?$", result
)
270 return match
.group(1)
272 die("Could not determine file type for %s (result: '%s')" % (file, result
))
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths
):
277 if isinstance(depotPaths
,basestring
):
278 depotPaths
= [depotPaths
]
280 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
286 # Return the set of all git tags
289 for line
in read_pipe_lines(["git", "tag"]):
294 def diffTreePattern():
295 # This is a simple generator for the diff tree regex pattern. This could be
296 # a class variable if this and parseDiffTreeEntry were a part of a class.
297 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
301 def parseDiffTreeEntry(entry
):
302 """Parses a single diff tree entry into its component elements.
304 See git-diff-tree(1) manpage for details about the format of the diff
305 output. This method returns a dictionary with the following elements:
307 src_mode - The mode of the source file
308 dst_mode - The mode of the destination file
309 src_sha1 - The sha1 for the source file
310 dst_sha1 - The sha1 fr the destination file
311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312 status_score - The score for the status (applicable for 'C' and 'R'
313 statuses). This is None if there is no score.
314 src - The path for the source file.
315 dst - The path for the destination file. This is only present for
316 copy or renames. If it is not present, this is None.
318 If the pattern is not matched, None is returned."""
320 match
= diffTreePattern().next().match(entry
)
323 'src_mode': match
.group(1),
324 'dst_mode': match
.group(2),
325 'src_sha1': match
.group(3),
326 'dst_sha1': match
.group(4),
327 'status': match
.group(5),
328 'status_score': match
.group(6),
329 'src': match
.group(7),
330 'dst': match
.group(10)
334 def isModeExec(mode
):
335 # Returns True if the given git mode represents an executable file,
337 return mode
[-3:] == "755"
339 def isModeExecChanged(src_mode
, dst_mode
):
340 return isModeExec(src_mode
) != isModeExec(dst_mode
)
342 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
344 if isinstance(cmd
,basestring
):
351 cmd
= p4_build_cmd(cmd
)
353 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
355 # Use a temporary file to avoid deadlocks without
356 # subprocess.communicate(), which would put another copy
357 # of stdout into memory.
359 if stdin
is not None:
360 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
361 if isinstance(stdin
,basestring
):
362 stdin_file
.write(stdin
)
365 stdin_file
.write(i
+ '\n')
369 p4
= subprocess
.Popen(cmd
,
372 stdout
=subprocess
.PIPE
)
377 entry
= marshal
.load(p4
.stdout
)
387 entry
["p4ExitCode"] = exitCode
393 list = p4CmdList(cmd
)
399 def p4Where(depotPath
):
400 if not depotPath
.endswith("/"):
402 depotPath
= depotPath
+ "..."
403 outputList
= p4CmdList(["where", depotPath
])
405 for entry
in outputList
:
406 if "depotFile" in entry
:
407 if entry
["depotFile"] == depotPath
:
410 elif "data" in entry
:
411 data
= entry
.get("data")
412 space
= data
.find(" ")
413 if data
[:space
] == depotPath
:
418 if output
["code"] == "error":
422 clientPath
= output
.get("path")
423 elif "data" in output
:
424 data
= output
.get("data")
425 lastSpace
= data
.rfind(" ")
426 clientPath
= data
[lastSpace
+ 1:]
428 if clientPath
.endswith("..."):
429 clientPath
= clientPath
[:-3]
432 def currentGitBranch():
433 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
435 def isValidGitDir(path
):
436 if (os
.path
.exists(path
+ "/HEAD")
437 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
441 def parseRevision(ref
):
442 return read_pipe("git rev-parse %s" % ref
).strip()
444 def branchExists(ref
):
445 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
449 def extractLogMessageFromGitCommit(commit
):
452 ## fixme: title is first line of commit, not 1st paragraph.
454 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
463 def extractSettingsGitLog(log
):
465 for line
in log
.split("\n"):
467 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
471 assignments
= m
.group(1).split (':')
472 for a
in assignments
:
474 key
= vals
[0].strip()
475 val
= ('='.join (vals
[1:])).strip()
476 if val
.endswith ('\"') and val
.startswith('"'):
481 paths
= values
.get("depot-paths")
483 paths
= values
.get("depot-path")
485 values
['depot-paths'] = paths
.split(',')
488 def gitBranchExists(branch
):
489 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
490 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
491 return proc
.wait() == 0;
494 def gitConfig(key
, args
= None): # set args to "--bool", for instance
495 if not _gitConfig
.has_key(key
):
498 argsFilter
= "%s " % args
499 cmd
= "git config %s%s" % (argsFilter
, key
)
500 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
501 return _gitConfig
[key
]
503 def gitConfigList(key
):
504 if not _gitConfig
.has_key(key
):
505 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
506 return _gitConfig
[key
]
508 def p4BranchesInGit(branchesAreInRemotes
= True):
511 cmdline
= "git rev-parse --symbolic "
512 if branchesAreInRemotes
:
513 cmdline
+= " --remotes"
515 cmdline
+= " --branches"
517 for line
in read_pipe_lines(cmdline
):
520 ## only import to p4/
521 if not line
.startswith('p4/') or line
== "p4/HEAD":
526 branch
= re
.sub ("^p4/", "", line
)
528 branches
[branch
] = parseRevision(line
)
531 def findUpstreamBranchPoint(head
= "HEAD"):
532 branches
= p4BranchesInGit()
533 # map from depot-path to branch name
534 branchByDepotPath
= {}
535 for branch
in branches
.keys():
536 tip
= branches
[branch
]
537 log
= extractLogMessageFromGitCommit(tip
)
538 settings
= extractSettingsGitLog(log
)
539 if settings
.has_key("depot-paths"):
540 paths
= ",".join(settings
["depot-paths"])
541 branchByDepotPath
[paths
] = "remotes/p4/" + branch
545 while parent
< 65535:
546 commit
= head
+ "~%s" % parent
547 log
= extractLogMessageFromGitCommit(commit
)
548 settings
= extractSettingsGitLog(log
)
549 if settings
.has_key("depot-paths"):
550 paths
= ",".join(settings
["depot-paths"])
551 if branchByDepotPath
.has_key(paths
):
552 return [branchByDepotPath
[paths
], settings
]
556 return ["", settings
]
558 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
560 print ("Creating/updating branch(es) in %s based on origin branch(es)"
563 originPrefix
= "origin/p4/"
565 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
567 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
570 headName
= line
[len(originPrefix
):]
571 remoteHead
= localRefPrefix
+ headName
574 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
575 if (not original
.has_key('depot-paths')
576 or not original
.has_key('change')):
580 if not gitBranchExists(remoteHead
):
582 print "creating %s" % remoteHead
585 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
586 if settings
.has_key('change') > 0:
587 if settings
['depot-paths'] == original
['depot-paths']:
588 originP4Change
= int(original
['change'])
589 p4Change
= int(settings
['change'])
590 if originP4Change
> p4Change
:
591 print ("%s (%s) is newer than %s (%s). "
592 "Updating p4 branch from origin."
593 % (originHead
, originP4Change
,
594 remoteHead
, p4Change
))
597 print ("Ignoring: %s was imported from %s while "
598 "%s was imported from %s"
599 % (originHead
, ','.join(original
['depot-paths']),
600 remoteHead
, ','.join(settings
['depot-paths'])))
603 system("git update-ref %s %s" % (remoteHead
, originHead
))
605 def originP4BranchesExist():
606 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
608 def p4ChangesForPaths(depotPaths
, changeRange
):
612 cmd
+= ["%s...%s" % (p
, changeRange
)]
613 output
= p4_read_pipe_lines(cmd
)
617 changeNum
= int(line
.split(" ")[1])
618 changes
[changeNum
] = True
620 changelist
= changes
.keys()
624 def p4PathStartsWith(path
, prefix
):
625 # This method tries to remedy a potential mixed-case issue:
627 # If UserA adds //depot/DirA/file1
628 # and UserB adds //depot/dira/file2
630 # we may or may not have a problem. If you have core.ignorecase=true,
631 # we treat DirA and dira as the same directory
632 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
634 return path
.lower().startswith(prefix
.lower())
635 return path
.startswith(prefix
)
638 """Look at the p4 client spec, create a View() object that contains
639 all the mappings, and return it."""
641 specList
= p4CmdList("client -o")
642 if len(specList
) != 1:
643 die('Output from "client -o" is %d lines, expecting 1' %
646 # dictionary of all client parameters
649 # just the keys that start with "View"
650 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
655 # append the lines, in order, to the view
656 for view_num
in range(len(view_keys
)):
657 k
= "View%d" % view_num
658 if k
not in view_keys
:
659 die("Expected view key %s missing" % k
)
660 view
.append(entry
[k
])
665 """Grab the client directory."""
667 output
= p4CmdList("client -o")
669 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
672 if "Root" not in entry
:
673 die('Client has no "Root"')
678 # P4 wildcards are not allowed in filenames. P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
682 def wildcard_decode(path
):
683 # Search for and fix just these four characters. Do % last so
684 # that fixing it does not inadvertently create new %-escapes.
685 # Cannot have * in a filename in windows; untested as to
686 # what p4 would do in such a case.
687 if not platform
.system() == "Windows":
688 path
= path
.replace("%2A", "*")
689 path
= path
.replace("%23", "#") \
690 .replace("%40", "@") \
694 def wildcard_encode(path
):
695 # do % first to avoid double-encoding the %s introduced here
696 path
= path
.replace("%", "%25") \
697 .replace("*", "%2A") \
698 .replace("#", "%23") \
702 def wildcard_present(path
):
703 return path
.translate(None, "*#@%") != path
707 self
.usage
= "usage: %prog [options]"
713 self
.userMapFromPerforceServer
= False
714 self
.myP4UserId
= None
718 return self
.myP4UserId
720 results
= p4CmdList("user -o")
722 if r
.has_key('User'):
723 self
.myP4UserId
= r
['User']
725 die("Could not find your p4 user id")
727 def p4UserIsMe(self
, p4User
):
728 # return True if the given p4 user is actually me
730 if not p4User
or p4User
!= me
:
735 def getUserCacheFilename(self
):
736 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
737 return home
+ "/.gitp4-usercache.txt"
739 def getUserMapFromPerforceServer(self
):
740 if self
.userMapFromPerforceServer
:
745 for output
in p4CmdList("users"):
746 if not output
.has_key("User"):
748 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
749 self
.emails
[output
["Email"]] = output
["User"]
753 for (key
, val
) in self
.users
.items():
754 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
756 open(self
.getUserCacheFilename(), "wb").write(s
)
757 self
.userMapFromPerforceServer
= True
759 def loadUserMapFromCache(self
):
761 self
.userMapFromPerforceServer
= False
763 cache
= open(self
.getUserCacheFilename(), "rb")
764 lines
= cache
.readlines()
767 entry
= line
.strip().split("\t")
768 self
.users
[entry
[0]] = entry
[1]
770 self
.getUserMapFromPerforceServer()
772 class P4Debug(Command
):
774 Command
.__init
__(self
)
776 self
.description
= "A tool to debug the output of p4 -G."
777 self
.needsGit
= False
781 for output
in p4CmdList(args
):
782 print 'Element: %d' % j
787 class P4RollBack(Command
):
789 Command
.__init
__(self
)
791 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
793 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
794 self
.rollbackLocalBranches
= False
799 maxChange
= int(args
[0])
801 if "p4ExitCode" in p4Cmd("changes -m 1"):
802 die("Problems executing p4");
804 if self
.rollbackLocalBranches
:
805 refPrefix
= "refs/heads/"
806 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
808 refPrefix
= "refs/remotes/"
809 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
812 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
814 ref
= refPrefix
+ line
815 log
= extractLogMessageFromGitCommit(ref
)
816 settings
= extractSettingsGitLog(log
)
818 depotPaths
= settings
['depot-paths']
819 change
= settings
['change']
823 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
824 for p
in depotPaths
]))) == 0:
825 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
826 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
829 while change
and int(change
) > maxChange
:
832 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
833 system("git update-ref %s \"%s^\"" % (ref
, ref
))
834 log
= extractLogMessageFromGitCommit(ref
)
835 settings
= extractSettingsGitLog(log
)
838 depotPaths
= settings
['depot-paths']
839 change
= settings
['change']
842 print "%s rewound to %s" % (ref
, change
)
846 class P4Submit(Command
, P4UserMap
):
848 Command
.__init
__(self
)
849 P4UserMap
.__init
__(self
)
851 optparse
.make_option("--origin", dest
="origin"),
852 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
853 # preserve the user, requires relevant p4 permissions
854 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
855 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
857 self
.description
= "Submit changes from git to the perforce depot."
858 self
.usage
+= " [name of git branch to submit into perforce depot]"
860 self
.detectRenames
= False
861 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
862 self
.isWindows
= (platform
.system() == "Windows")
863 self
.exportLabels
= False
864 self
.p4HasMoveCommand
= p4_has_command("move")
867 if len(p4CmdList("opened ...")) > 0:
868 die("You have files opened with perforce! Close them before starting the sync.")
870 def separate_jobs_from_description(self
, message
):
871 """Extract and return a possible Jobs field in the commit
872 message. It goes into a separate section in the p4 change
875 A jobs line starts with "Jobs:" and looks like a new field
876 in a form. Values are white-space separated on the same
877 line or on following lines that start with a tab.
879 This does not parse and extract the full git commit message
880 like a p4 form. It just sees the Jobs: line as a marker
881 to pass everything from then on directly into the p4 form,
882 but outside the description section.
884 Return a tuple (stripped log message, jobs string)."""
886 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
888 return (message
, None)
890 jobtext
= message
[m
.start():]
891 stripped_message
= message
[:m
.start()].rstrip()
892 return (stripped_message
, jobtext
)
894 def prepareLogMessage(self
, template
, message
, jobs
):
895 """Edits the template returned from "p4 change -o" to insert
896 the message in the Description field, and the jobs text in
900 inDescriptionSection
= False
902 for line
in template
.split("\n"):
903 if line
.startswith("#"):
904 result
+= line
+ "\n"
907 if inDescriptionSection
:
908 if line
.startswith("Files:") or line
.startswith("Jobs:"):
909 inDescriptionSection
= False
910 # insert Jobs section
912 result
+= jobs
+ "\n"
916 if line
.startswith("Description:"):
917 inDescriptionSection
= True
919 for messageLine
in message
.split("\n"):
920 line
+= "\t" + messageLine
+ "\n"
922 result
+= line
+ "\n"
926 def patchRCSKeywords(self
, file, pattern
):
927 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
928 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
930 outFile
= os
.fdopen(handle
, "w+")
931 inFile
= open(file, "r")
932 regexp
= re
.compile(pattern
, re
.VERBOSE
)
933 for line
in inFile
.readlines():
934 line
= regexp
.sub(r
'$\1$', line
)
938 # Forcibly overwrite the original file
940 shutil
.move(outFileName
, file)
942 # cleanup our temporary file
943 os
.unlink(outFileName
)
944 print "Failed to strip RCS keywords in %s" % file
947 print "Patched up RCS keywords in %s" % file
949 def p4UserForCommit(self
,id):
950 # Return the tuple (perforce user,git email) for a given git commit id
951 self
.getUserMapFromPerforceServer()
952 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
953 gitEmail
= gitEmail
.strip()
954 if not self
.emails
.has_key(gitEmail
):
955 return (None,gitEmail
)
957 return (self
.emails
[gitEmail
],gitEmail
)
959 def checkValidP4Users(self
,commits
):
960 # check if any git authors cannot be mapped to p4 users
962 (user
,email
) = self
.p4UserForCommit(id)
964 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
965 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
968 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
970 def lastP4Changelist(self
):
971 # Get back the last changelist number submitted in this client spec. This
972 # then gets used to patch up the username in the change. If the same
973 # client spec is being used by multiple processes then this might go
975 results
= p4CmdList("client -o") # find the current client
978 if r
.has_key('Client'):
982 die("could not get client spec")
983 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
985 if r
.has_key('change'):
987 die("Could not get changelist number for last submit - cannot patch up user details")
989 def modifyChangelistUser(self
, changelist
, newUser
):
990 # fixup the user field of a changelist after it has been submitted.
991 changes
= p4CmdList("change -o %s" % changelist
)
992 if len(changes
) != 1:
993 die("Bad output from p4 change modifying %s to user %s" %
994 (changelist
, newUser
))
997 if c
['User'] == newUser
: return # nothing to do
999 input = marshal
.dumps(c
)
1001 result
= p4CmdList("change -f -i", stdin
=input)
1003 if r
.has_key('code'):
1004 if r
['code'] == 'error':
1005 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1006 if r
.has_key('data'):
1007 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1009 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1011 def canChangeChangelists(self
):
1012 # check to see if we have p4 admin or super-user permissions, either of
1013 # which are required to modify changelists.
1014 results
= p4CmdList(["protects", self
.depotPath
])
1016 if r
.has_key('perm'):
1017 if r
['perm'] == 'admin':
1019 if r
['perm'] == 'super':
1023 def prepareSubmitTemplate(self
):
1024 """Run "p4 change -o" to grab a change specification template.
1025 This does not use "p4 -G", as it is nice to keep the submission
1026 template in original order, since a human might edit it.
1028 Remove lines in the Files section that show changes to files
1029 outside the depot path we're committing into."""
1032 inFilesSection
= False
1033 for line
in p4_read_pipe_lines(['change', '-o']):
1034 if line
.endswith("\r\n"):
1035 line
= line
[:-2] + "\n"
1037 if line
.startswith("\t"):
1038 # path starts and ends with a tab
1040 lastTab
= path
.rfind("\t")
1042 path
= path
[:lastTab
]
1043 if not p4PathStartsWith(path
, self
.depotPath
):
1046 inFilesSection
= False
1048 if line
.startswith("Files:"):
1049 inFilesSection
= True
1055 def edit_template(self
, template_file
):
1056 """Invoke the editor to let the user change the submission
1057 message. Return true if okay to continue with the submit."""
1059 # if configured to skip the editing part, just submit
1060 if gitConfig("git-p4.skipSubmitEdit") == "true":
1063 # look at the modification time, to check later if the user saved
1065 mtime
= os
.stat(template_file
).st_mtime
1068 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1069 editor
= os
.environ
.get("P4EDITOR")
1071 editor
= read_pipe("git var GIT_EDITOR").strip()
1072 system(editor
+ " " + template_file
)
1074 # If the file was not saved, prompt to see if this patch should
1075 # be skipped. But skip this verification step if configured so.
1076 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1079 # modification time updated means user saved the file
1080 if os
.stat(template_file
).st_mtime
> mtime
:
1084 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1090 def applyCommit(self
, id):
1091 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1093 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1095 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1097 filesToDelete
= set()
1099 pureRenameCopy
= set()
1100 filesToChangeExecBit
= {}
1103 diff
= parseDiffTreeEntry(line
)
1104 modifier
= diff
['status']
1108 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1109 filesToChangeExecBit
[path
] = diff
['dst_mode']
1110 editedFiles
.add(path
)
1111 elif modifier
== "A":
1112 filesToAdd
.add(path
)
1113 filesToChangeExecBit
[path
] = diff
['dst_mode']
1114 if path
in filesToDelete
:
1115 filesToDelete
.remove(path
)
1116 elif modifier
== "D":
1117 filesToDelete
.add(path
)
1118 if path
in filesToAdd
:
1119 filesToAdd
.remove(path
)
1120 elif modifier
== "C":
1121 src
, dest
= diff
['src'], diff
['dst']
1122 p4_integrate(src
, dest
)
1123 pureRenameCopy
.add(dest
)
1124 if diff
['src_sha1'] != diff
['dst_sha1']:
1126 pureRenameCopy
.discard(dest
)
1127 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1129 pureRenameCopy
.discard(dest
)
1130 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1132 editedFiles
.add(dest
)
1133 elif modifier
== "R":
1134 src
, dest
= diff
['src'], diff
['dst']
1135 if self
.p4HasMoveCommand
:
1136 p4_edit(src
) # src must be open before move
1137 p4_move(src
, dest
) # opens for (move/delete, move/add)
1139 p4_integrate(src
, dest
)
1140 if diff
['src_sha1'] != diff
['dst_sha1']:
1143 pureRenameCopy
.add(dest
)
1144 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1145 if not self
.p4HasMoveCommand
:
1146 p4_edit(dest
) # with move: already open, writable
1147 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1148 if not self
.p4HasMoveCommand
:
1150 filesToDelete
.add(src
)
1151 editedFiles
.add(dest
)
1153 die("unknown modifier %s for %s" % (modifier
, path
))
1155 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1156 patchcmd
= diffcmd
+ " | git apply "
1157 tryPatchCmd
= patchcmd
+ "--check -"
1158 applyPatchCmd
= patchcmd
+ "--check --apply -"
1159 patch_succeeded
= True
1161 if os
.system(tryPatchCmd
) != 0:
1162 fixed_rcs_keywords
= False
1163 patch_succeeded
= False
1164 print "Unfortunately applying the change failed!"
1166 # Patch failed, maybe it's just RCS keyword woes. Look through
1167 # the patch to see if that's possible.
1168 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1172 for file in editedFiles | filesToDelete
:
1173 # did this file's delta contain RCS keywords?
1174 pattern
= p4_keywords_regexp_for_file(file)
1177 # this file is a possibility...look for RCS keywords.
1178 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1179 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1180 if regexp
.search(line
):
1182 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1183 kwfiles
[file] = pattern
1186 for file in kwfiles
:
1188 print "zapping %s with %s" % (line
,pattern
)
1189 self
.patchRCSKeywords(file, kwfiles
[file])
1190 fixed_rcs_keywords
= True
1192 if fixed_rcs_keywords
:
1193 print "Retrying the patch with RCS keywords cleaned up"
1194 if os
.system(tryPatchCmd
) == 0:
1195 patch_succeeded
= True
1197 if not patch_succeeded
:
1198 print "What do you want to do?"
1200 while response
!= "s" and response
!= "a" and response
!= "w":
1201 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1202 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1204 print "Skipping! Good luck with the next patches..."
1205 for f
in editedFiles
:
1207 for f
in filesToAdd
:
1210 elif response
== "a":
1211 os
.system(applyPatchCmd
)
1212 if len(filesToAdd
) > 0:
1213 print "You may also want to call p4 add on the following files:"
1214 print " ".join(filesToAdd
)
1215 if len(filesToDelete
):
1216 print "The following files should be scheduled for deletion with p4 delete:"
1217 print " ".join(filesToDelete
)
1218 die("Please resolve and submit the conflict manually and "
1219 + "continue afterwards with git p4 submit --continue")
1220 elif response
== "w":
1221 system(diffcmd
+ " > patch.txt")
1222 print "Patch saved to patch.txt in %s !" % self
.clientPath
1223 die("Please resolve and submit the conflict manually and "
1224 "continue afterwards with git p4 submit --continue")
1226 system(applyPatchCmd
)
1228 for f
in filesToAdd
:
1230 for f
in filesToDelete
:
1234 # Set/clear executable bits
1235 for f
in filesToChangeExecBit
.keys():
1236 mode
= filesToChangeExecBit
[f
]
1237 setP4ExecBit(f
, mode
)
1239 logMessage
= extractLogMessageFromGitCommit(id)
1240 logMessage
= logMessage
.strip()
1241 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1243 template
= self
.prepareSubmitTemplate()
1244 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1246 if self
.preserveUser
:
1247 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1249 if os
.environ
.has_key("P4DIFF"):
1250 del(os
.environ
["P4DIFF"])
1252 for editedFile
in editedFiles
:
1253 diff
+= p4_read_pipe(['diff', '-du',
1254 wildcard_encode(editedFile
)])
1257 for newFile
in filesToAdd
:
1258 newdiff
+= "==== new file ====\n"
1259 newdiff
+= "--- /dev/null\n"
1260 newdiff
+= "+++ %s\n" % newFile
1261 f
= open(newFile
, "r")
1262 for line
in f
.readlines():
1263 newdiff
+= "+" + line
1266 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1267 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1268 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1269 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1271 separatorLine
= "######## everything below this line is just the diff #######\n"
1273 (handle
, fileName
) = tempfile
.mkstemp()
1274 tmpFile
= os
.fdopen(handle
, "w+")
1276 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1277 separatorLine
= separatorLine
.replace("\n", "\r\n")
1278 newdiff
= newdiff
.replace("\n", "\r\n")
1279 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1282 if self
.edit_template(fileName
):
1283 # read the edited message and submit
1284 tmpFile
= open(fileName
, "rb")
1285 message
= tmpFile
.read()
1287 submitTemplate
= message
[:message
.index(separatorLine
)]
1289 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1290 p4_write_pipe(['submit', '-i'], submitTemplate
)
1292 if self
.preserveUser
:
1294 # Get last changelist number. Cannot easily get it from
1295 # the submit command output as the output is
1297 changelist
= self
.lastP4Changelist()
1298 self
.modifyChangelistUser(changelist
, p4User
)
1300 # The rename/copy happened by applying a patch that created a
1301 # new file. This leaves it writable, which confuses p4.
1302 for f
in pureRenameCopy
:
1307 print "Submission cancelled, undoing p4 changes."
1308 for f
in editedFiles
:
1310 for f
in filesToAdd
:
1316 # Export git tags as p4 labels. Create a p4 label and then tag
1318 def exportGitTags(self
, gitTags
):
1319 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1320 if len(validLabelRegexp
) == 0:
1321 validLabelRegexp
= defaultLabelRegexp
1322 m
= re
.compile(validLabelRegexp
)
1324 for name
in gitTags
:
1326 if not m
.match(name
):
1328 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1331 # Get the p4 commit this corresponds to
1332 logMessage
= extractLogMessageFromGitCommit(name
)
1333 values
= extractSettingsGitLog(logMessage
)
1335 if not values
.has_key('change'):
1336 # a tag pointing to something not sent to p4; ignore
1338 print "git tag %s does not give a p4 commit" % name
1341 changelist
= values
['change']
1343 # Get the tag details.
1347 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1350 if re
.match(r
'tag\s+', l
):
1352 elif re
.match(r
'\s*$', l
):
1359 body
= ["lightweight tag imported by git p4\n"]
1361 # Create the label - use the same view as the client spec we are using
1362 clientSpec
= getClientSpec()
1364 labelTemplate
= "Label: %s\n" % name
1365 labelTemplate
+= "Description:\n"
1367 labelTemplate
+= "\t" + b
+ "\n"
1368 labelTemplate
+= "View:\n"
1369 for mapping
in clientSpec
.mappings
:
1370 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1372 p4_write_pipe(["label", "-i"], labelTemplate
)
1375 p4_system(["tag", "-l", name
] +
1376 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1379 print "created p4 label for tag %s" % name
1381 def run(self
, args
):
1383 self
.master
= currentGitBranch()
1384 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1385 die("Detecting current git branch failed!")
1386 elif len(args
) == 1:
1387 self
.master
= args
[0]
1388 if not branchExists(self
.master
):
1389 die("Branch %s does not exist" % self
.master
)
1393 allowSubmit
= gitConfig("git-p4.allowSubmit")
1394 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1395 die("%s is not in git-p4.allowSubmit" % self
.master
)
1397 [upstream
, settings
] = findUpstreamBranchPoint()
1398 self
.depotPath
= settings
['depot-paths'][0]
1399 if len(self
.origin
) == 0:
1400 self
.origin
= upstream
1402 if self
.preserveUser
:
1403 if not self
.canChangeChangelists():
1404 die("Cannot preserve user names without p4 super-user or admin permissions")
1407 print "Origin branch is " + self
.origin
1409 if len(self
.depotPath
) == 0:
1410 print "Internal error: cannot locate perforce depot path from existing branches"
1413 self
.useClientSpec
= False
1414 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1415 self
.useClientSpec
= True
1416 if self
.useClientSpec
:
1417 self
.clientSpecDirs
= getClientSpec()
1419 if self
.useClientSpec
:
1420 # all files are relative to the client spec
1421 self
.clientPath
= getClientRoot()
1423 self
.clientPath
= p4Where(self
.depotPath
)
1425 if self
.clientPath
== "":
1426 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1428 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1429 self
.oldWorkingDirectory
= os
.getcwd()
1431 # ensure the clientPath exists
1432 new_client_dir
= False
1433 if not os
.path
.exists(self
.clientPath
):
1434 new_client_dir
= True
1435 os
.makedirs(self
.clientPath
)
1437 chdir(self
.clientPath
)
1438 print "Synchronizing p4 checkout..."
1440 # old one was destroyed, and maybe nobody told p4
1441 p4_sync("...", "-f")
1447 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1448 commits
.append(line
.strip())
1451 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1452 self
.checkAuthorship
= False
1454 self
.checkAuthorship
= True
1456 if self
.preserveUser
:
1457 self
.checkValidP4Users(commits
)
1460 # Build up a set of options to be passed to diff when
1461 # submitting each commit to p4.
1463 if self
.detectRenames
:
1464 # command-line -M arg
1465 self
.diffOpts
= "-M"
1467 # If not explicitly set check the config variable
1468 detectRenames
= gitConfig("git-p4.detectRenames")
1470 if detectRenames
.lower() == "false" or detectRenames
== "":
1472 elif detectRenames
.lower() == "true":
1473 self
.diffOpts
= "-M"
1475 self
.diffOpts
= "-M%s" % detectRenames
1477 # no command-line arg for -C or --find-copies-harder, just
1479 detectCopies
= gitConfig("git-p4.detectCopies")
1480 if detectCopies
.lower() == "false" or detectCopies
== "":
1482 elif detectCopies
.lower() == "true":
1483 self
.diffOpts
+= " -C"
1485 self
.diffOpts
+= " -C%s" % detectCopies
1487 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1488 self
.diffOpts
+= " --find-copies-harder"
1490 while len(commits
) > 0:
1492 commits
= commits
[1:]
1493 self
.applyCommit(commit
)
1495 if len(commits
) == 0:
1496 print "All changes applied!"
1497 chdir(self
.oldWorkingDirectory
)
1505 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1506 self
.exportLabels
= True
1508 if self
.exportLabels
:
1509 p4Labels
= getP4Labels(self
.depotPath
)
1510 gitTags
= getGitTags()
1512 missingGitTags
= gitTags
- p4Labels
1513 self
.exportGitTags(missingGitTags
)
1518 """Represent a p4 view ("p4 help views"), and map files in a
1519 repo according to the view."""
1522 """A depot or client path, possibly containing wildcards.
1523 The only one supported is ... at the end, currently.
1524 Initialize with the full path, with //depot or //client."""
1526 def __init__(self
, path
, is_depot
):
1528 self
.is_depot
= is_depot
1529 self
.find_wildcards()
1530 # remember the prefix bit, useful for relative mappings
1531 m
= re
.match("(//[^/]+/)", self
.path
)
1533 die("Path %s does not start with //prefix/" % self
.path
)
1535 if not self
.is_depot
:
1536 # strip //client/ on client paths
1537 self
.path
= self
.path
[len(prefix
):]
1539 def find_wildcards(self
):
1540 """Make sure wildcards are valid, and set up internal
1543 self
.ends_triple_dot
= False
1544 # There are three wildcards allowed in p4 views
1545 # (see "p4 help views"). This code knows how to
1546 # handle "..." (only at the end), but cannot deal with
1547 # "%%n" or "*". Only check the depot_side, as p4 should
1548 # validate that the client_side matches too.
1549 if re
.search(r
'%%[1-9]', self
.path
):
1550 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1551 if self
.path
.find("*") >= 0:
1552 die("Can't handle * wildcards in view: %s" % self
.path
)
1553 triple_dot_index
= self
.path
.find("...")
1554 if triple_dot_index
>= 0:
1555 if triple_dot_index
!= len(self
.path
) - 3:
1556 die("Can handle only single ... wildcard, at end: %s" %
1558 self
.ends_triple_dot
= True
1560 def ensure_compatible(self
, other_path
):
1561 """Make sure the wildcards agree."""
1562 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1563 die("Both paths must end with ... if either does;\n" +
1564 "paths: %s %s" % (self
.path
, other_path
.path
))
1566 def match_wildcards(self
, test_path
):
1567 """See if this test_path matches us, and fill in the value
1568 of the wildcards if so. Returns a tuple of
1569 (True|False, wildcards[]). For now, only the ... at end
1570 is supported, so at most one wildcard."""
1571 if self
.ends_triple_dot
:
1572 dotless
= self
.path
[:-3]
1573 if test_path
.startswith(dotless
):
1574 wildcard
= test_path
[len(dotless
):]
1575 return (True, [ wildcard
])
1577 if test_path
== self
.path
:
1581 def match(self
, test_path
):
1582 """Just return if it matches; don't bother with the wildcards."""
1583 b
, _
= self
.match_wildcards(test_path
)
1586 def fill_in_wildcards(self
, wildcards
):
1587 """Return the relative path, with the wildcards filled in
1588 if there are any."""
1589 if self
.ends_triple_dot
:
1590 return self
.path
[:-3] + wildcards
[0]
1594 class Mapping(object):
1595 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1596 # depot_side is without the trailing /... if it had one
1597 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1598 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1599 self
.overlay
= overlay
# started with "+"
1600 self
.exclude
= exclude
# started with "-"
1601 assert not (self
.overlay
and self
.exclude
)
1602 self
.depot_side
.ensure_compatible(self
.client_side
)
1610 return "View.Mapping: %s%s -> %s" % \
1611 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1613 def map_depot_to_client(self
, depot_path
):
1614 """Calculate the client path if using this mapping on the
1615 given depot path; does not consider the effect of other
1616 mappings in a view. Even excluded mappings are returned."""
1617 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1620 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1629 def append(self
, view_line
):
1630 """Parse a view line, splitting it into depot and client
1631 sides. Append to self.mappings, preserving order."""
1633 # Split the view line into exactly two words. P4 enforces
1634 # structure on these lines that simplifies this quite a bit.
1636 # Either or both words may be double-quoted.
1637 # Single quotes do not matter.
1638 # Double-quote marks cannot occur inside the words.
1639 # A + or - prefix is also inside the quotes.
1640 # There are no quotes unless they contain a space.
1641 # The line is already white-space stripped.
1642 # The two words are separated by a single space.
1644 if view_line
[0] == '"':
1645 # First word is double quoted. Find its end.
1646 close_quote_index
= view_line
.find('"', 1)
1647 if close_quote_index
<= 0:
1648 die("No first-word closing quote found: %s" % view_line
)
1649 depot_side
= view_line
[1:close_quote_index
]
1650 # skip closing quote and space
1651 rhs_index
= close_quote_index
+ 1 + 1
1653 space_index
= view_line
.find(" ")
1654 if space_index
<= 0:
1655 die("No word-splitting space found: %s" % view_line
)
1656 depot_side
= view_line
[0:space_index
]
1657 rhs_index
= space_index
+ 1
1659 if view_line
[rhs_index
] == '"':
1660 # Second word is double quoted. Make sure there is a
1661 # double quote at the end too.
1662 if not view_line
.endswith('"'):
1663 die("View line with rhs quote should end with one: %s" %
1666 client_side
= view_line
[rhs_index
+1:-1]
1668 client_side
= view_line
[rhs_index
:]
1670 # prefix + means overlay on previous mapping
1672 if depot_side
.startswith("+"):
1674 depot_side
= depot_side
[1:]
1676 # prefix - means exclude this path
1678 if depot_side
.startswith("-"):
1680 depot_side
= depot_side
[1:]
1682 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1683 self
.mappings
.append(m
)
1685 def map_in_client(self
, depot_path
):
1686 """Return the relative location in the client where this
1687 depot file should live. Returns "" if the file should
1688 not be mapped in the client."""
1693 # look at later entries first
1694 for m
in self
.mappings
[::-1]:
1696 # see where will this path end up in the client
1697 p
= m
.map_depot_to_client(depot_path
)
1700 # Depot path does not belong in client. Must remember
1701 # this, as previous items should not cause files to
1702 # exist in this path either. Remember that the list is
1703 # being walked from the end, which has higher precedence.
1704 # Overlap mappings do not exclude previous mappings.
1706 paths_filled
.append(m
.client_side
)
1709 # This mapping matched; no need to search any further.
1710 # But, the mapping could be rejected if the client path
1711 # has already been claimed by an earlier mapping (i.e.
1712 # one later in the list, which we are walking backwards).
1713 already_mapped_in_client
= False
1714 for f
in paths_filled
:
1715 # this is View.Path.match
1717 already_mapped_in_client
= True
1719 if not already_mapped_in_client
:
1720 # Include this file, unless it is from a line that
1721 # explicitly said to exclude it.
1725 # a match, even if rejected, always stops the search
1730 class P4Sync(Command
, P4UserMap
):
1731 delete_actions
= ( "delete", "move/delete", "purge" )
1734 Command
.__init
__(self
)
1735 P4UserMap
.__init
__(self
)
1737 optparse
.make_option("--branch", dest
="branch"),
1738 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1739 optparse
.make_option("--changesfile", dest
="changesFile"),
1740 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1741 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1742 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1743 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1744 help="Import into refs/heads/ , not refs/remotes"),
1745 optparse
.make_option("--max-changes", dest
="maxChanges"),
1746 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1747 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1748 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1749 help="Only sync files that are included in the Perforce Client Spec")
1751 self
.description
= """Imports from Perforce into a git repository.\n
1753 //depot/my/project/ -- to import the current head
1754 //depot/my/project/@all -- to import everything
1755 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1757 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1759 self
.usage
+= " //depot/path[@revRange]"
1761 self
.createdBranches
= set()
1762 self
.committedChanges
= set()
1764 self
.detectBranches
= False
1765 self
.detectLabels
= False
1766 self
.importLabels
= False
1767 self
.changesFile
= ""
1768 self
.syncWithOrigin
= True
1769 self
.importIntoRemotes
= True
1770 self
.maxChanges
= ""
1771 self
.isWindows
= (platform
.system() == "Windows")
1772 self
.keepRepoPath
= False
1773 self
.depotPaths
= None
1774 self
.p4BranchesInGit
= []
1775 self
.cloneExclude
= []
1776 self
.useClientSpec
= False
1777 self
.useClientSpec_from_options
= False
1778 self
.clientSpecDirs
= None
1779 self
.tempBranches
= []
1780 self
.tempBranchLocation
= "git-p4-tmp"
1782 if gitConfig("git-p4.syncFromOrigin") == "false":
1783 self
.syncWithOrigin
= False
1785 # Force a checkpoint in fast-import and wait for it to finish
1786 def checkpoint(self
):
1787 self
.gitStream
.write("checkpoint\n\n")
1788 self
.gitStream
.write("progress checkpoint\n\n")
1789 out
= self
.gitOutput
.readline()
1791 print "checkpoint finished: " + out
1793 def extractFilesFromCommit(self
, commit
):
1794 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1795 for path
in self
.cloneExclude
]
1798 while commit
.has_key("depotFile%s" % fnum
):
1799 path
= commit
["depotFile%s" % fnum
]
1801 if [p
for p
in self
.cloneExclude
1802 if p4PathStartsWith(path
, p
)]:
1805 found
= [p
for p
in self
.depotPaths
1806 if p4PathStartsWith(path
, p
)]
1813 file["rev"] = commit
["rev%s" % fnum
]
1814 file["action"] = commit
["action%s" % fnum
]
1815 file["type"] = commit
["type%s" % fnum
]
1820 def stripRepoPath(self
, path
, prefixes
):
1821 if self
.useClientSpec
:
1822 path
= self
.clientSpecDirs
.map_in_client(path
)
1825 if self
.keepRepoPath
:
1826 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1829 if p4PathStartsWith(path
, p
):
1830 path
= path
[len(p
):]
1832 path
= wildcard_decode(path
)
1835 def splitFilesIntoBranches(self
, commit
):
1838 while commit
.has_key("depotFile%s" % fnum
):
1839 path
= commit
["depotFile%s" % fnum
]
1840 found
= [p
for p
in self
.depotPaths
1841 if p4PathStartsWith(path
, p
)]
1848 file["rev"] = commit
["rev%s" % fnum
]
1849 file["action"] = commit
["action%s" % fnum
]
1850 file["type"] = commit
["type%s" % fnum
]
1853 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1855 for branch
in self
.knownBranches
.keys():
1857 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1858 if relPath
.startswith(branch
+ "/"):
1859 if branch
not in branches
:
1860 branches
[branch
] = []
1861 branches
[branch
].append(file)
1866 # output one file from the P4 stream
1867 # - helper for streamP4Files
1869 def streamOneP4File(self
, file, contents
):
1870 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1872 sys
.stderr
.write("%s\n" % relPath
)
1874 (type_base
, type_mods
) = split_p4_type(file["type"])
1877 if "x" in type_mods
:
1879 if type_base
== "symlink":
1881 # p4 print on a symlink contains "target\n"; remove the newline
1882 data
= ''.join(contents
)
1883 contents
= [data
[:-1]]
1885 if type_base
== "utf16":
1886 # p4 delivers different text in the python output to -G
1887 # than it does when using "print -o", or normal p4 client
1888 # operations. utf16 is converted to ascii or utf8, perhaps.
1889 # But ascii text saved as -t utf16 is completely mangled.
1890 # Invoke print -o to get the real contents.
1891 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1894 if type_base
== "apple":
1895 # Apple filetype files will be streamed as a concatenation of
1896 # its appledouble header and the contents. This is useless
1897 # on both macs and non-macs. If using "print -q -o xx", it
1898 # will create "xx" with the data, and "%xx" with the header.
1899 # This is also not very useful.
1901 # Ideally, someday, this script can learn how to generate
1902 # appledouble files directly and import those to git, but
1903 # non-mac machines can never find a use for apple filetype.
1904 print "\nIgnoring apple filetype file %s" % file['depotFile']
1907 # Perhaps windows wants unicode, utf16 newlines translated too;
1908 # but this is not doing it.
1909 if self
.isWindows
and type_base
== "text":
1911 for data
in contents
:
1912 data
= data
.replace("\r\n", "\n")
1913 mangled
.append(data
)
1916 # Note that we do not try to de-mangle keywords on utf16 files,
1917 # even though in theory somebody may want that.
1918 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1920 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1921 text
= ''.join(contents
)
1922 text
= regexp
.sub(r
'$\1$', text
)
1925 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1930 length
= length
+ len(d
)
1932 self
.gitStream
.write("data %d\n" % length
)
1934 self
.gitStream
.write(d
)
1935 self
.gitStream
.write("\n")
1937 def streamOneP4Deletion(self
, file):
1938 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1940 sys
.stderr
.write("delete %s\n" % relPath
)
1941 self
.gitStream
.write("D %s\n" % relPath
)
1943 # handle another chunk of streaming data
1944 def streamP4FilesCb(self
, marshalled
):
1946 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1947 # start of a new file - output the old one first
1948 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1949 self
.stream_file
= {}
1950 self
.stream_contents
= []
1951 self
.stream_have_file_info
= False
1953 # pick up the new file information... for the
1954 # 'data' field we need to append to our array
1955 for k
in marshalled
.keys():
1957 self
.stream_contents
.append(marshalled
['data'])
1959 self
.stream_file
[k
] = marshalled
[k
]
1961 self
.stream_have_file_info
= True
1963 # Stream directly from "p4 files" into "git fast-import"
1964 def streamP4Files(self
, files
):
1970 # if using a client spec, only add the files that have
1971 # a path in the client
1972 if self
.clientSpecDirs
:
1973 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
1976 filesForCommit
.append(f
)
1977 if f
['action'] in self
.delete_actions
:
1978 filesToDelete
.append(f
)
1980 filesToRead
.append(f
)
1983 for f
in filesToDelete
:
1984 self
.streamOneP4Deletion(f
)
1986 if len(filesToRead
) > 0:
1987 self
.stream_file
= {}
1988 self
.stream_contents
= []
1989 self
.stream_have_file_info
= False
1991 # curry self argument
1992 def streamP4FilesCbSelf(entry
):
1993 self
.streamP4FilesCb(entry
)
1995 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
1997 p4CmdList(["-x", "-", "print"],
1999 cb
=streamP4FilesCbSelf
)
2002 if self
.stream_file
.has_key('depotFile'):
2003 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2005 def make_email(self
, userid
):
2006 if userid
in self
.users
:
2007 return self
.users
[userid
]
2009 return "%s <a@b>" % userid
2012 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2014 print "writing tag %s for commit %s" % (labelName
, commit
)
2015 gitStream
.write("tag %s\n" % labelName
)
2016 gitStream
.write("from %s\n" % commit
)
2018 if labelDetails
.has_key('Owner'):
2019 owner
= labelDetails
["Owner"]
2023 # Try to use the owner of the p4 label, or failing that,
2024 # the current p4 user id.
2026 email
= self
.make_email(owner
)
2028 email
= self
.make_email(self
.p4UserId())
2029 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2031 gitStream
.write("tagger %s\n" % tagger
)
2033 print "labelDetails=",labelDetails
2034 if labelDetails
.has_key('Description'):
2035 description
= labelDetails
['Description']
2037 description
= 'Label from git p4'
2039 gitStream
.write("data %d\n" % len(description
))
2040 gitStream
.write(description
)
2041 gitStream
.write("\n")
2043 def commit(self
, details
, files
, branch
, parent
= ""):
2044 epoch
= details
["time"]
2045 author
= details
["user"]
2048 print "commit into %s" % branch
2050 # start with reading files; if that fails, we should not
2054 if [p
for p
in self
.branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2055 new_files
.append (f
)
2057 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2059 self
.gitStream
.write("commit %s\n" % branch
)
2060 # gitStream.write("mark :%s\n" % details["change"])
2061 self
.committedChanges
.add(int(details
["change"]))
2063 if author
not in self
.users
:
2064 self
.getUserMapFromPerforceServer()
2065 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2067 self
.gitStream
.write("committer %s\n" % committer
)
2069 self
.gitStream
.write("data <<EOT\n")
2070 self
.gitStream
.write(details
["desc"])
2071 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2072 (','.join(self
.branchPrefixes
), details
["change"]))
2073 if len(details
['options']) > 0:
2074 self
.gitStream
.write(": options = %s" % details
['options'])
2075 self
.gitStream
.write("]\nEOT\n\n")
2079 print "parent %s" % parent
2080 self
.gitStream
.write("from %s\n" % parent
)
2082 self
.streamP4Files(new_files
)
2083 self
.gitStream
.write("\n")
2085 change
= int(details
["change"])
2087 if self
.labels
.has_key(change
):
2088 label
= self
.labels
[change
]
2089 labelDetails
= label
[0]
2090 labelRevisions
= label
[1]
2092 print "Change %s is labelled %s" % (change
, labelDetails
)
2094 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2095 for p
in self
.branchPrefixes
])
2097 if len(files
) == len(labelRevisions
):
2101 if info
["action"] in self
.delete_actions
:
2103 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2105 if cleanedFiles
== labelRevisions
:
2106 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2110 print ("Tag %s does not match with change %s: files do not match."
2111 % (labelDetails
["label"], change
))
2115 print ("Tag %s does not match with change %s: file count is different."
2116 % (labelDetails
["label"], change
))
2118 # Build a dictionary of changelists and labels, for "detect-labels" option.
2119 def getLabels(self
):
2122 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2123 if len(l
) > 0 and not self
.silent
:
2124 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2127 label
= output
["label"]
2131 print "Querying files for label %s" % label
2132 for file in p4CmdList(["files"] +
2133 ["%s...@%s" % (p
, label
)
2134 for p
in self
.depotPaths
]):
2135 revisions
[file["depotFile"]] = file["rev"]
2136 change
= int(file["change"])
2137 if change
> newestChange
:
2138 newestChange
= change
2140 self
.labels
[newestChange
] = [output
, revisions
]
2143 print "Label changes: %s" % self
.labels
.keys()
2145 # Import p4 labels as git tags. A direct mapping does not
2146 # exist, so assume that if all the files are at the same revision
2147 # then we can use that, or it's something more complicated we should
2149 def importP4Labels(self
, stream
, p4Labels
):
2151 print "import p4 labels: " + ' '.join(p4Labels
)
2153 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2154 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2155 if len(validLabelRegexp
) == 0:
2156 validLabelRegexp
= defaultLabelRegexp
2157 m
= re
.compile(validLabelRegexp
)
2159 for name
in p4Labels
:
2162 if not m
.match(name
):
2164 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2167 if name
in ignoredP4Labels
:
2170 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2172 # get the most recent changelist for each file in this label
2173 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2174 for p
in self
.depotPaths
])
2176 if change
.has_key('change'):
2177 # find the corresponding git commit; take the oldest commit
2178 changelist
= int(change
['change'])
2179 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2180 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2181 if len(gitCommit
) == 0:
2182 print "could not find git commit for changelist %d" % changelist
2184 gitCommit
= gitCommit
.strip()
2186 # Convert from p4 time format
2188 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2190 print "Could not convert label time %s" % labelDetail
['Update']
2193 when
= int(time
.mktime(tmwhen
))
2194 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2196 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2199 print "Label %s has no changelists - possibly deleted?" % name
2202 # We can't import this label; don't try again as it will get very
2203 # expensive repeatedly fetching all the files for labels that will
2204 # never be imported. If the label is moved in the future, the
2205 # ignore will need to be removed manually.
2206 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2208 def guessProjectName(self
):
2209 for p
in self
.depotPaths
:
2212 p
= p
[p
.strip().rfind("/") + 1:]
2213 if not p
.endswith("/"):
2217 def getBranchMapping(self
):
2218 lostAndFoundBranches
= set()
2220 user
= gitConfig("git-p4.branchUser")
2222 command
= "branches -u %s" % user
2224 command
= "branches"
2226 for info
in p4CmdList(command
):
2227 details
= p4Cmd(["branch", "-o", info
["branch"]])
2229 while details
.has_key("View%s" % viewIdx
):
2230 paths
= details
["View%s" % viewIdx
].split(" ")
2231 viewIdx
= viewIdx
+ 1
2232 # require standard //depot/foo/... //depot/bar/... mapping
2233 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2236 destination
= paths
[1]
2238 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2239 source
= source
[len(self
.depotPaths
[0]):-4]
2240 destination
= destination
[len(self
.depotPaths
[0]):-4]
2242 if destination
in self
.knownBranches
:
2244 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2245 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2248 self
.knownBranches
[destination
] = source
2250 lostAndFoundBranches
.discard(destination
)
2252 if source
not in self
.knownBranches
:
2253 lostAndFoundBranches
.add(source
)
2255 # Perforce does not strictly require branches to be defined, so we also
2256 # check git config for a branch list.
2258 # Example of branch definition in git config file:
2260 # branchList=main:branchA
2261 # branchList=main:branchB
2262 # branchList=branchA:branchC
2263 configBranches
= gitConfigList("git-p4.branchList")
2264 for branch
in configBranches
:
2266 (source
, destination
) = branch
.split(":")
2267 self
.knownBranches
[destination
] = source
2269 lostAndFoundBranches
.discard(destination
)
2271 if source
not in self
.knownBranches
:
2272 lostAndFoundBranches
.add(source
)
2275 for branch
in lostAndFoundBranches
:
2276 self
.knownBranches
[branch
] = branch
2278 def getBranchMappingFromGitBranches(self
):
2279 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2280 for branch
in branches
.keys():
2281 if branch
== "master":
2284 branch
= branch
[len(self
.projectName
):]
2285 self
.knownBranches
[branch
] = branch
2287 def listExistingP4GitBranches(self
):
2288 # branches holds mapping from name to commit
2289 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2290 self
.p4BranchesInGit
= branches
.keys()
2291 for branch
in branches
.keys():
2292 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2294 def updateOptionDict(self
, d
):
2296 if self
.keepRepoPath
:
2297 option_keys
['keepRepoPath'] = 1
2299 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2301 def readOptions(self
, d
):
2302 self
.keepRepoPath
= (d
.has_key('options')
2303 and ('keepRepoPath' in d
['options']))
2305 def gitRefForBranch(self
, branch
):
2306 if branch
== "main":
2307 return self
.refPrefix
+ "master"
2309 if len(branch
) <= 0:
2312 return self
.refPrefix
+ self
.projectName
+ branch
2314 def gitCommitByP4Change(self
, ref
, change
):
2316 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2319 latestCommit
= parseRevision(ref
)
2323 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2324 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2329 log
= extractLogMessageFromGitCommit(next
)
2330 settings
= extractSettingsGitLog(log
)
2331 currentChange
= int(settings
['change'])
2333 print "current change %s" % currentChange
2335 if currentChange
== change
:
2337 print "found %s" % next
2340 if currentChange
< change
:
2341 earliestCommit
= "^%s" % next
2343 latestCommit
= "%s" % next
2347 def importNewBranch(self
, branch
, maxChange
):
2348 # make fast-import flush all changes to disk and update the refs using the checkpoint
2349 # command so that we can try to find the branch parent in the git history
2350 self
.gitStream
.write("checkpoint\n\n");
2351 self
.gitStream
.flush();
2352 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2353 range = "@1,%s" % maxChange
2354 #print "prefix" + branchPrefix
2355 changes
= p4ChangesForPaths([branchPrefix
], range)
2356 if len(changes
) <= 0:
2358 firstChange
= changes
[0]
2359 #print "first change in branch: %s" % firstChange
2360 sourceBranch
= self
.knownBranches
[branch
]
2361 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2362 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2363 #print "source " + sourceBranch
2365 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2366 #print "branch parent: %s" % branchParentChange
2367 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2368 if len(gitParent
) > 0:
2369 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2370 #print "parent git commit: %s" % gitParent
2372 self
.importChanges(changes
)
2375 def searchParent(self
, parent
, branch
, target
):
2377 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2379 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2382 print "Found parent of %s in commit %s" % (branch
, blob
)
2389 def importChanges(self
, changes
):
2391 for change
in changes
:
2392 description
= p4Cmd(["describe", str(change
)])
2393 self
.updateOptionDict(description
)
2396 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2401 if self
.detectBranches
:
2402 branches
= self
.splitFilesIntoBranches(description
)
2403 for branch
in branches
.keys():
2405 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2406 self
.branchPrefixes
= [ branchPrefix
]
2410 filesForCommit
= branches
[branch
]
2413 print "branch is %s" % branch
2415 self
.updatedBranches
.add(branch
)
2417 if branch
not in self
.createdBranches
:
2418 self
.createdBranches
.add(branch
)
2419 parent
= self
.knownBranches
[branch
]
2420 if parent
== branch
:
2423 fullBranch
= self
.projectName
+ branch
2424 if fullBranch
not in self
.p4BranchesInGit
:
2426 print("\n Importing new branch %s" % fullBranch
);
2427 if self
.importNewBranch(branch
, change
- 1):
2429 self
.p4BranchesInGit
.append(fullBranch
)
2431 print("\n Resuming with change %s" % change
);
2434 print "parent determined through known branches: %s" % parent
2436 branch
= self
.gitRefForBranch(branch
)
2437 parent
= self
.gitRefForBranch(parent
)
2440 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2442 if len(parent
) == 0 and branch
in self
.initialParents
:
2443 parent
= self
.initialParents
[branch
]
2444 del self
.initialParents
[branch
]
2448 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2450 print "Creating temporary branch: " + tempBranch
2451 self
.commit(description
, filesForCommit
, tempBranch
)
2452 self
.tempBranches
.append(tempBranch
)
2454 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2456 self
.commit(description
, filesForCommit
, branch
, blob
)
2459 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2460 self
.commit(description
, filesForCommit
, branch
, parent
)
2462 files
= self
.extractFilesFromCommit(description
)
2463 self
.commit(description
, files
, self
.branch
,
2465 self
.initialParent
= ""
2467 print self
.gitError
.read()
2470 def importHeadRevision(self
, revision
):
2471 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2474 details
["user"] = "git perforce import user"
2475 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2476 % (' '.join(self
.depotPaths
), revision
))
2477 details
["change"] = revision
2481 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2483 for info
in p4CmdList(["files"] + fileArgs
):
2485 if 'code' in info
and info
['code'] == 'error':
2486 sys
.stderr
.write("p4 returned an error: %s\n"
2488 if info
['data'].find("must refer to client") >= 0:
2489 sys
.stderr
.write("This particular p4 error is misleading.\n")
2490 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2491 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2493 if 'p4ExitCode' in info
:
2494 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2498 change
= int(info
["change"])
2499 if change
> newestRevision
:
2500 newestRevision
= change
2502 if info
["action"] in self
.delete_actions
:
2503 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2504 #fileCnt = fileCnt + 1
2507 for prop
in ["depotFile", "rev", "action", "type" ]:
2508 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2510 fileCnt
= fileCnt
+ 1
2512 details
["change"] = newestRevision
2514 # Use time from top-most change so that all git p4 clones of
2515 # the same p4 repo have the same commit SHA1s.
2516 res
= p4CmdList("describe -s %d" % newestRevision
)
2519 if r
.has_key('time'):
2520 newestTime
= int(r
['time'])
2521 if newestTime
is None:
2522 die("\"describe -s\" on newest change %d did not give a time")
2523 details
["time"] = newestTime
2525 self
.updateOptionDict(details
)
2527 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
)
2529 print "IO error with git fast-import. Is your git version recent enough?"
2530 print self
.gitError
.read()
2533 def run(self
, args
):
2534 self
.depotPaths
= []
2535 self
.changeRange
= ""
2536 self
.initialParent
= ""
2537 self
.previousDepotPaths
= []
2539 # map from branch depot path to parent branch
2540 self
.knownBranches
= {}
2541 self
.initialParents
= {}
2542 self
.hasOrigin
= originP4BranchesExist()
2543 if not self
.syncWithOrigin
:
2544 self
.hasOrigin
= False
2546 if self
.importIntoRemotes
:
2547 self
.refPrefix
= "refs/remotes/p4/"
2549 self
.refPrefix
= "refs/heads/p4/"
2551 if self
.syncWithOrigin
and self
.hasOrigin
:
2553 print "Syncing with origin first by calling git fetch origin"
2554 system("git fetch origin")
2556 if len(self
.branch
) == 0:
2557 self
.branch
= self
.refPrefix
+ "master"
2558 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2559 system("git update-ref %s refs/heads/p4" % self
.branch
)
2560 system("git branch -D p4");
2561 # create it /after/ importing, when master exists
2562 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2563 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2565 # accept either the command-line option, or the configuration variable
2566 if self
.useClientSpec
:
2567 # will use this after clone to set the variable
2568 self
.useClientSpec_from_options
= True
2570 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2571 self
.useClientSpec
= True
2572 if self
.useClientSpec
:
2573 self
.clientSpecDirs
= getClientSpec()
2575 # TODO: should always look at previous commits,
2576 # merge with previous imports, if possible.
2579 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2580 self
.listExistingP4GitBranches()
2582 if len(self
.p4BranchesInGit
) > 1:
2584 print "Importing from/into multiple branches"
2585 self
.detectBranches
= True
2588 print "branches: %s" % self
.p4BranchesInGit
2591 for branch
in self
.p4BranchesInGit
:
2592 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2594 settings
= extractSettingsGitLog(logMsg
)
2596 self
.readOptions(settings
)
2597 if (settings
.has_key('depot-paths')
2598 and settings
.has_key ('change')):
2599 change
= int(settings
['change']) + 1
2600 p4Change
= max(p4Change
, change
)
2602 depotPaths
= sorted(settings
['depot-paths'])
2603 if self
.previousDepotPaths
== []:
2604 self
.previousDepotPaths
= depotPaths
2607 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2608 prev_list
= prev
.split("/")
2609 cur_list
= cur
.split("/")
2610 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2611 if cur_list
[i
] <> prev_list
[i
]:
2615 paths
.append ("/".join(cur_list
[:i
+ 1]))
2617 self
.previousDepotPaths
= paths
2620 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2621 self
.changeRange
= "@%s,#head" % p4Change
2622 if not self
.detectBranches
:
2623 self
.initialParent
= parseRevision(self
.branch
)
2624 if not self
.silent
and not self
.detectBranches
:
2625 print "Performing incremental import into %s git branch" % self
.branch
2627 if not self
.branch
.startswith("refs/"):
2628 self
.branch
= "refs/heads/" + self
.branch
2630 if len(args
) == 0 and self
.depotPaths
:
2632 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2634 if self
.depotPaths
and self
.depotPaths
!= args
:
2635 print ("previous import used depot path %s and now %s was specified. "
2636 "This doesn't work!" % (' '.join (self
.depotPaths
),
2640 self
.depotPaths
= sorted(args
)
2645 # Make sure no revision specifiers are used when --changesfile
2647 bad_changesfile
= False
2648 if len(self
.changesFile
) > 0:
2649 for p
in self
.depotPaths
:
2650 if p
.find("@") >= 0 or p
.find("#") >= 0:
2651 bad_changesfile
= True
2654 die("Option --changesfile is incompatible with revision specifiers")
2657 for p
in self
.depotPaths
:
2658 if p
.find("@") != -1:
2659 atIdx
= p
.index("@")
2660 self
.changeRange
= p
[atIdx
:]
2661 if self
.changeRange
== "@all":
2662 self
.changeRange
= ""
2663 elif ',' not in self
.changeRange
:
2664 revision
= self
.changeRange
2665 self
.changeRange
= ""
2667 elif p
.find("#") != -1:
2668 hashIdx
= p
.index("#")
2669 revision
= p
[hashIdx
:]
2671 elif self
.previousDepotPaths
== []:
2672 # pay attention to changesfile, if given, else import
2673 # the entire p4 tree at the head revision
2674 if len(self
.changesFile
) == 0:
2677 p
= re
.sub ("\.\.\.$", "", p
)
2678 if not p
.endswith("/"):
2683 self
.depotPaths
= newPaths
2685 # --detect-branches may change this for each branch
2686 self
.branchPrefixes
= self
.depotPaths
2688 self
.loadUserMapFromCache()
2690 if self
.detectLabels
:
2693 if self
.detectBranches
:
2694 ## FIXME - what's a P4 projectName ?
2695 self
.projectName
= self
.guessProjectName()
2698 self
.getBranchMappingFromGitBranches()
2700 self
.getBranchMapping()
2702 print "p4-git branches: %s" % self
.p4BranchesInGit
2703 print "initial parents: %s" % self
.initialParents
2704 for b
in self
.p4BranchesInGit
:
2708 b
= b
[len(self
.projectName
):]
2709 self
.createdBranches
.add(b
)
2711 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2713 importProcess
= subprocess
.Popen(["git", "fast-import"],
2714 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2715 stderr
=subprocess
.PIPE
);
2716 self
.gitOutput
= importProcess
.stdout
2717 self
.gitStream
= importProcess
.stdin
2718 self
.gitError
= importProcess
.stderr
2721 self
.importHeadRevision(revision
)
2725 if len(self
.changesFile
) > 0:
2726 output
= open(self
.changesFile
).readlines()
2729 changeSet
.add(int(line
))
2731 for change
in changeSet
:
2732 changes
.append(change
)
2736 # catch "git p4 sync" with no new branches, in a repo that
2737 # does not have any existing p4 branches
2738 if len(args
) == 0 and not self
.p4BranchesInGit
:
2739 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2741 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2743 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2745 if len(self
.maxChanges
) > 0:
2746 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2748 if len(changes
) == 0:
2750 print "No changes to import!"
2752 if not self
.silent
and not self
.detectBranches
:
2753 print "Import destination: %s" % self
.branch
2755 self
.updatedBranches
= set()
2757 self
.importChanges(changes
)
2761 if len(self
.updatedBranches
) > 0:
2762 sys
.stdout
.write("Updated branches: ")
2763 for b
in self
.updatedBranches
:
2764 sys
.stdout
.write("%s " % b
)
2765 sys
.stdout
.write("\n")
2767 if gitConfig("git-p4.importLabels", "--bool") == "true":
2768 self
.importLabels
= True
2770 if self
.importLabels
:
2771 p4Labels
= getP4Labels(self
.depotPaths
)
2772 gitTags
= getGitTags()
2774 missingP4Labels
= p4Labels
- gitTags
2775 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2777 self
.gitStream
.close()
2778 if importProcess
.wait() != 0:
2779 die("fast-import failed: %s" % self
.gitError
.read())
2780 self
.gitOutput
.close()
2781 self
.gitError
.close()
2783 # Cleanup temporary branches created during import
2784 if self
.tempBranches
!= []:
2785 for branch
in self
.tempBranches
:
2786 read_pipe("git update-ref -d %s" % branch
)
2787 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2791 class P4Rebase(Command
):
2793 Command
.__init
__(self
)
2795 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2797 self
.importLabels
= False
2798 self
.description
= ("Fetches the latest revision from perforce and "
2799 + "rebases the current work (branch) against it")
2801 def run(self
, args
):
2803 sync
.importLabels
= self
.importLabels
2806 return self
.rebase()
2809 if os
.system("git update-index --refresh") != 0:
2810 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2811 if len(read_pipe("git diff-index HEAD --")) > 0:
2812 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2814 [upstream
, settings
] = findUpstreamBranchPoint()
2815 if len(upstream
) == 0:
2816 die("Cannot find upstream branchpoint for rebase")
2818 # the branchpoint may be p4/foo~3, so strip off the parent
2819 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2821 print "Rebasing the current branch onto %s" % upstream
2822 oldHead
= read_pipe("git rev-parse HEAD").strip()
2823 system("git rebase %s" % upstream
)
2824 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2827 class P4Clone(P4Sync
):
2829 P4Sync
.__init
__(self
)
2830 self
.description
= "Creates a new git repository and imports from Perforce into it"
2831 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2833 optparse
.make_option("--destination", dest
="cloneDestination",
2834 action
='store', default
=None,
2835 help="where to leave result of the clone"),
2836 optparse
.make_option("-/", dest
="cloneExclude",
2837 action
="append", type="string",
2838 help="exclude depot path"),
2839 optparse
.make_option("--bare", dest
="cloneBare",
2840 action
="store_true", default
=False),
2842 self
.cloneDestination
= None
2843 self
.needsGit
= False
2844 self
.cloneBare
= False
2846 # This is required for the "append" cloneExclude action
2847 def ensure_value(self
, attr
, value
):
2848 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2849 setattr(self
, attr
, value
)
2850 return getattr(self
, attr
)
2852 def defaultDestination(self
, args
):
2853 ## TODO: use common prefix of args?
2855 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2856 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2857 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2858 depotDir
= re
.sub(r
"/$", "", depotDir
)
2859 return os
.path
.split(depotDir
)[1]
2861 def run(self
, args
):
2865 if self
.keepRepoPath
and not self
.cloneDestination
:
2866 sys
.stderr
.write("Must specify destination for --keep-path\n")
2871 if not self
.cloneDestination
and len(depotPaths
) > 1:
2872 self
.cloneDestination
= depotPaths
[-1]
2873 depotPaths
= depotPaths
[:-1]
2875 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2876 for p
in depotPaths
:
2877 if not p
.startswith("//"):
2880 if not self
.cloneDestination
:
2881 self
.cloneDestination
= self
.defaultDestination(args
)
2883 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2885 if not os
.path
.exists(self
.cloneDestination
):
2886 os
.makedirs(self
.cloneDestination
)
2887 chdir(self
.cloneDestination
)
2889 init_cmd
= [ "git", "init" ]
2891 init_cmd
.append("--bare")
2892 subprocess
.check_call(init_cmd
)
2894 if not P4Sync
.run(self
, depotPaths
):
2896 if self
.branch
!= "master":
2897 if self
.importIntoRemotes
:
2898 masterbranch
= "refs/remotes/p4/master"
2900 masterbranch
= "refs/heads/p4/master"
2901 if gitBranchExists(masterbranch
):
2902 system("git branch master %s" % masterbranch
)
2903 if not self
.cloneBare
:
2904 system("git checkout -f")
2906 print "Could not detect main branch. No checkout/master branch created."
2908 # auto-set this variable if invoked with --use-client-spec
2909 if self
.useClientSpec_from_options
:
2910 system("git config --bool git-p4.useclientspec true")
2914 class P4Branches(Command
):
2916 Command
.__init
__(self
)
2918 self
.description
= ("Shows the git branches that hold imports and their "
2919 + "corresponding perforce depot paths")
2920 self
.verbose
= False
2922 def run(self
, args
):
2923 if originP4BranchesExist():
2924 createOrUpdateBranchesFromOrigin()
2926 cmdline
= "git rev-parse --symbolic "
2927 cmdline
+= " --remotes"
2929 for line
in read_pipe_lines(cmdline
):
2932 if not line
.startswith('p4/') or line
== "p4/HEAD":
2936 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2937 settings
= extractSettingsGitLog(log
)
2939 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2942 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2944 optparse
.IndentedHelpFormatter
.__init
__(self
)
2946 def format_description(self
, description
):
2948 return description
+ "\n"
2952 def printUsage(commands
):
2953 print "usage: %s <command> [options]" % sys
.argv
[0]
2955 print "valid commands: %s" % ", ".join(commands
)
2957 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2962 "submit" : P4Submit
,
2963 "commit" : P4Submit
,
2965 "rebase" : P4Rebase
,
2967 "rollback" : P4RollBack
,
2968 "branches" : P4Branches
2973 if len(sys
.argv
[1:]) == 0:
2974 printUsage(commands
.keys())
2978 cmdName
= sys
.argv
[1]
2980 klass
= commands
[cmdName
]
2983 print "unknown command %s" % cmdName
2985 printUsage(commands
.keys())
2988 options
= cmd
.options
2989 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2993 options
.append(optparse
.make_option("--verbose", dest
="verbose", action
="store_true"))
2995 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2997 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2999 description
= cmd
.description
,
3000 formatter
= HelpFormatter())
3002 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3004 verbose
= cmd
.verbose
3006 if cmd
.gitdir
== None:
3007 cmd
.gitdir
= os
.path
.abspath(".git")
3008 if not isValidGitDir(cmd
.gitdir
):
3009 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3010 if os
.path
.exists(cmd
.gitdir
):
3011 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3015 if not isValidGitDir(cmd
.gitdir
):
3016 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3017 cmd
.gitdir
+= "/.git"
3019 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3021 os
.environ
["GIT_DIR"] = cmd
.gitdir
3023 if not cmd
.run(args
):
3028 if __name__
== '__main__':