3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
123 def p4_has_command(cmd
):
124 """Ask p4 for help on this command. If it returns an error, the
125 command does not exist in this version of p4."""
126 real_cmd
= p4_build_cmd(["help", cmd
])
127 p
= subprocess
.Popen(real_cmd
, stdout
=subprocess
.PIPE
,
128 stderr
=subprocess
.PIPE
)
130 return p
.returncode
== 0
133 expand
= isinstance(cmd
,basestring
)
135 sys
.stderr
.write("executing %s\n" % str(cmd
))
136 subprocess
.check_call(cmd
, shell
=expand
)
139 """Specifically invoke p4 as the system command. """
140 real_cmd
= p4_build_cmd(cmd
)
141 expand
= isinstance(real_cmd
, basestring
)
142 subprocess
.check_call(real_cmd
, shell
=expand
)
144 def p4_integrate(src
, dest
):
145 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
147 def p4_sync(f
, *options
):
148 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
151 # forcibly add file names with wildcards
152 if wildcard_present(f
):
153 p4_system(["add", "-f", f
])
155 p4_system(["add", f
])
158 p4_system(["delete", wildcard_encode(f
)])
161 p4_system(["edit", wildcard_encode(f
)])
164 p4_system(["revert", wildcard_encode(f
)])
166 def p4_reopen(type, f
):
167 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
169 def p4_move(src
, dest
):
170 p4_system(["move", "-k", wildcard_encode(src
), wildcard_encode(dest
)])
173 # Canonicalize the p4 type and return a tuple of the
174 # base type, plus any modifiers. See "p4 help filetypes"
175 # for a list and explanation.
177 def split_p4_type(p4type
):
179 p4_filetypes_historical
= {
180 "ctempobj": "binary+Sw",
186 "tempobj": "binary+FSw",
187 "ubinary": "binary+F",
188 "uresource": "resource+F",
189 "uxbinary": "binary+Fx",
190 "xbinary": "binary+x",
192 "xtempobj": "binary+Swx",
194 "xunicode": "unicode+x",
197 if p4type
in p4_filetypes_historical
:
198 p4type
= p4_filetypes_historical
[p4type
]
200 s
= p4type
.split("+")
208 # return the raw p4 type of a file (text, text+ko, etc)
211 results
= p4CmdList(["fstat", "-T", "headType", file])
212 return results
[0]['headType']
215 # Given a type base and modifier, return a regexp matching
216 # the keywords that can be expanded in the file
218 def p4_keywords_regexp_for_type(base
, type_mods
):
219 if base
in ("text", "unicode", "binary"):
221 if "ko" in type_mods
:
223 elif "k" in type_mods
:
224 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
228 \$ # Starts with a dollar, followed by...
229 (%s) # one of the keywords, followed by...
230 (:[^$]+)? # possibly an old expansion, followed by...
238 # Given a file, return a regexp matching the possible
239 # RCS keywords that will be expanded, or None for files
240 # with kw expansion turned off.
242 def p4_keywords_regexp_for_file(file):
243 if not os
.path
.exists(file):
246 (type_base
, type_mods
) = split_p4_type(p4_type(file))
247 return p4_keywords_regexp_for_type(type_base
, type_mods
)
249 def setP4ExecBit(file, mode
):
250 # Reopens an already open file and changes the execute bit to match
251 # the execute bit setting in the passed in mode.
255 if not isModeExec(mode
):
256 p4Type
= getP4OpenedType(file)
257 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
258 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
259 if p4Type
[-1] == "+":
260 p4Type
= p4Type
[0:-1]
262 p4_reopen(p4Type
, file)
264 def getP4OpenedType(file):
265 # Returns the perforce file type for the given file.
267 result
= p4_read_pipe(["opened", wildcard_encode(file)])
268 match
= re
.match(".*\((.+)\)\r?$", result
)
270 return match
.group(1)
272 die("Could not determine file type for %s (result: '%s')" % (file, result
))
274 # Return the set of all p4 labels
275 def getP4Labels(depotPaths
):
277 if isinstance(depotPaths
,basestring
):
278 depotPaths
= [depotPaths
]
280 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
286 # Return the set of all git tags
289 for line
in read_pipe_lines(["git", "tag"]):
294 def diffTreePattern():
295 # This is a simple generator for the diff tree regex pattern. This could be
296 # a class variable if this and parseDiffTreeEntry were a part of a class.
297 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
301 def parseDiffTreeEntry(entry
):
302 """Parses a single diff tree entry into its component elements.
304 See git-diff-tree(1) manpage for details about the format of the diff
305 output. This method returns a dictionary with the following elements:
307 src_mode - The mode of the source file
308 dst_mode - The mode of the destination file
309 src_sha1 - The sha1 for the source file
310 dst_sha1 - The sha1 fr the destination file
311 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
312 status_score - The score for the status (applicable for 'C' and 'R'
313 statuses). This is None if there is no score.
314 src - The path for the source file.
315 dst - The path for the destination file. This is only present for
316 copy or renames. If it is not present, this is None.
318 If the pattern is not matched, None is returned."""
320 match
= diffTreePattern().next().match(entry
)
323 'src_mode': match
.group(1),
324 'dst_mode': match
.group(2),
325 'src_sha1': match
.group(3),
326 'dst_sha1': match
.group(4),
327 'status': match
.group(5),
328 'status_score': match
.group(6),
329 'src': match
.group(7),
330 'dst': match
.group(10)
334 def isModeExec(mode
):
335 # Returns True if the given git mode represents an executable file,
337 return mode
[-3:] == "755"
339 def isModeExecChanged(src_mode
, dst_mode
):
340 return isModeExec(src_mode
) != isModeExec(dst_mode
)
342 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
344 if isinstance(cmd
,basestring
):
351 cmd
= p4_build_cmd(cmd
)
353 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
355 # Use a temporary file to avoid deadlocks without
356 # subprocess.communicate(), which would put another copy
357 # of stdout into memory.
359 if stdin
is not None:
360 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
361 if isinstance(stdin
,basestring
):
362 stdin_file
.write(stdin
)
365 stdin_file
.write(i
+ '\n')
369 p4
= subprocess
.Popen(cmd
,
372 stdout
=subprocess
.PIPE
)
377 entry
= marshal
.load(p4
.stdout
)
387 entry
["p4ExitCode"] = exitCode
393 list = p4CmdList(cmd
)
399 def p4Where(depotPath
):
400 if not depotPath
.endswith("/"):
402 depotPath
= depotPath
+ "..."
403 outputList
= p4CmdList(["where", depotPath
])
405 for entry
in outputList
:
406 if "depotFile" in entry
:
407 if entry
["depotFile"] == depotPath
:
410 elif "data" in entry
:
411 data
= entry
.get("data")
412 space
= data
.find(" ")
413 if data
[:space
] == depotPath
:
418 if output
["code"] == "error":
422 clientPath
= output
.get("path")
423 elif "data" in output
:
424 data
= output
.get("data")
425 lastSpace
= data
.rfind(" ")
426 clientPath
= data
[lastSpace
+ 1:]
428 if clientPath
.endswith("..."):
429 clientPath
= clientPath
[:-3]
432 def currentGitBranch():
433 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
435 def isValidGitDir(path
):
436 if (os
.path
.exists(path
+ "/HEAD")
437 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
441 def parseRevision(ref
):
442 return read_pipe("git rev-parse %s" % ref
).strip()
444 def branchExists(ref
):
445 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
449 def extractLogMessageFromGitCommit(commit
):
452 ## fixme: title is first line of commit, not 1st paragraph.
454 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
463 def extractSettingsGitLog(log
):
465 for line
in log
.split("\n"):
467 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
471 assignments
= m
.group(1).split (':')
472 for a
in assignments
:
474 key
= vals
[0].strip()
475 val
= ('='.join (vals
[1:])).strip()
476 if val
.endswith ('\"') and val
.startswith('"'):
481 paths
= values
.get("depot-paths")
483 paths
= values
.get("depot-path")
485 values
['depot-paths'] = paths
.split(',')
488 def gitBranchExists(branch
):
489 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
490 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
491 return proc
.wait() == 0;
494 def gitConfig(key
, args
= None): # set args to "--bool", for instance
495 if not _gitConfig
.has_key(key
):
498 argsFilter
= "%s " % args
499 cmd
= "git config %s%s" % (argsFilter
, key
)
500 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
501 return _gitConfig
[key
]
503 def gitConfigList(key
):
504 if not _gitConfig
.has_key(key
):
505 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
506 return _gitConfig
[key
]
508 def p4BranchesInGit(branchesAreInRemotes
= True):
511 cmdline
= "git rev-parse --symbolic "
512 if branchesAreInRemotes
:
513 cmdline
+= " --remotes"
515 cmdline
+= " --branches"
517 for line
in read_pipe_lines(cmdline
):
520 ## only import to p4/
521 if not line
.startswith('p4/') or line
== "p4/HEAD":
526 branch
= re
.sub ("^p4/", "", line
)
528 branches
[branch
] = parseRevision(line
)
531 def findUpstreamBranchPoint(head
= "HEAD"):
532 branches
= p4BranchesInGit()
533 # map from depot-path to branch name
534 branchByDepotPath
= {}
535 for branch
in branches
.keys():
536 tip
= branches
[branch
]
537 log
= extractLogMessageFromGitCommit(tip
)
538 settings
= extractSettingsGitLog(log
)
539 if settings
.has_key("depot-paths"):
540 paths
= ",".join(settings
["depot-paths"])
541 branchByDepotPath
[paths
] = "remotes/p4/" + branch
545 while parent
< 65535:
546 commit
= head
+ "~%s" % parent
547 log
= extractLogMessageFromGitCommit(commit
)
548 settings
= extractSettingsGitLog(log
)
549 if settings
.has_key("depot-paths"):
550 paths
= ",".join(settings
["depot-paths"])
551 if branchByDepotPath
.has_key(paths
):
552 return [branchByDepotPath
[paths
], settings
]
556 return ["", settings
]
558 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
560 print ("Creating/updating branch(es) in %s based on origin branch(es)"
563 originPrefix
= "origin/p4/"
565 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
567 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
570 headName
= line
[len(originPrefix
):]
571 remoteHead
= localRefPrefix
+ headName
574 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
575 if (not original
.has_key('depot-paths')
576 or not original
.has_key('change')):
580 if not gitBranchExists(remoteHead
):
582 print "creating %s" % remoteHead
585 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
586 if settings
.has_key('change') > 0:
587 if settings
['depot-paths'] == original
['depot-paths']:
588 originP4Change
= int(original
['change'])
589 p4Change
= int(settings
['change'])
590 if originP4Change
> p4Change
:
591 print ("%s (%s) is newer than %s (%s). "
592 "Updating p4 branch from origin."
593 % (originHead
, originP4Change
,
594 remoteHead
, p4Change
))
597 print ("Ignoring: %s was imported from %s while "
598 "%s was imported from %s"
599 % (originHead
, ','.join(original
['depot-paths']),
600 remoteHead
, ','.join(settings
['depot-paths'])))
603 system("git update-ref %s %s" % (remoteHead
, originHead
))
605 def originP4BranchesExist():
606 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
608 def p4ChangesForPaths(depotPaths
, changeRange
):
612 cmd
+= ["%s...%s" % (p
, changeRange
)]
613 output
= p4_read_pipe_lines(cmd
)
617 changeNum
= int(line
.split(" ")[1])
618 changes
[changeNum
] = True
620 changelist
= changes
.keys()
624 def p4PathStartsWith(path
, prefix
):
625 # This method tries to remedy a potential mixed-case issue:
627 # If UserA adds //depot/DirA/file1
628 # and UserB adds //depot/dira/file2
630 # we may or may not have a problem. If you have core.ignorecase=true,
631 # we treat DirA and dira as the same directory
632 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
634 return path
.lower().startswith(prefix
.lower())
635 return path
.startswith(prefix
)
638 """Look at the p4 client spec, create a View() object that contains
639 all the mappings, and return it."""
641 specList
= p4CmdList("client -o")
642 if len(specList
) != 1:
643 die('Output from "client -o" is %d lines, expecting 1' %
646 # dictionary of all client parameters
649 # just the keys that start with "View"
650 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
655 # append the lines, in order, to the view
656 for view_num
in range(len(view_keys
)):
657 k
= "View%d" % view_num
658 if k
not in view_keys
:
659 die("Expected view key %s missing" % k
)
660 view
.append(entry
[k
])
665 """Grab the client directory."""
667 output
= p4CmdList("client -o")
669 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
672 if "Root" not in entry
:
673 die('Client has no "Root"')
678 # P4 wildcards are not allowed in filenames. P4 complains
679 # if you simply add them, but you can force it with "-f", in
680 # which case it translates them into %xx encoding internally.
682 def wildcard_decode(path
):
683 # Search for and fix just these four characters. Do % last so
684 # that fixing it does not inadvertently create new %-escapes.
685 # Cannot have * in a filename in windows; untested as to
686 # what p4 would do in such a case.
687 if not platform
.system() == "Windows":
688 path
= path
.replace("%2A", "*")
689 path
= path
.replace("%23", "#") \
690 .replace("%40", "@") \
694 def wildcard_encode(path
):
695 # do % first to avoid double-encoding the %s introduced here
696 path
= path
.replace("%", "%25") \
697 .replace("*", "%2A") \
698 .replace("#", "%23") \
702 def wildcard_present(path
):
703 return path
.translate(None, "*#@%") != path
707 self
.usage
= "usage: %prog [options]"
713 self
.userMapFromPerforceServer
= False
714 self
.myP4UserId
= None
718 return self
.myP4UserId
720 results
= p4CmdList("user -o")
722 if r
.has_key('User'):
723 self
.myP4UserId
= r
['User']
725 die("Could not find your p4 user id")
727 def p4UserIsMe(self
, p4User
):
728 # return True if the given p4 user is actually me
730 if not p4User
or p4User
!= me
:
735 def getUserCacheFilename(self
):
736 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
737 return home
+ "/.gitp4-usercache.txt"
739 def getUserMapFromPerforceServer(self
):
740 if self
.userMapFromPerforceServer
:
745 for output
in p4CmdList("users"):
746 if not output
.has_key("User"):
748 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
749 self
.emails
[output
["Email"]] = output
["User"]
753 for (key
, val
) in self
.users
.items():
754 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
756 open(self
.getUserCacheFilename(), "wb").write(s
)
757 self
.userMapFromPerforceServer
= True
759 def loadUserMapFromCache(self
):
761 self
.userMapFromPerforceServer
= False
763 cache
= open(self
.getUserCacheFilename(), "rb")
764 lines
= cache
.readlines()
767 entry
= line
.strip().split("\t")
768 self
.users
[entry
[0]] = entry
[1]
770 self
.getUserMapFromPerforceServer()
772 class P4Debug(Command
):
774 Command
.__init
__(self
)
776 self
.description
= "A tool to debug the output of p4 -G."
777 self
.needsGit
= False
781 for output
in p4CmdList(args
):
782 print 'Element: %d' % j
787 class P4RollBack(Command
):
789 Command
.__init
__(self
)
791 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
793 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
794 self
.rollbackLocalBranches
= False
799 maxChange
= int(args
[0])
801 if "p4ExitCode" in p4Cmd("changes -m 1"):
802 die("Problems executing p4");
804 if self
.rollbackLocalBranches
:
805 refPrefix
= "refs/heads/"
806 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
808 refPrefix
= "refs/remotes/"
809 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
812 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
814 ref
= refPrefix
+ line
815 log
= extractLogMessageFromGitCommit(ref
)
816 settings
= extractSettingsGitLog(log
)
818 depotPaths
= settings
['depot-paths']
819 change
= settings
['change']
823 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
824 for p
in depotPaths
]))) == 0:
825 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
826 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
829 while change
and int(change
) > maxChange
:
832 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
833 system("git update-ref %s \"%s^\"" % (ref
, ref
))
834 log
= extractLogMessageFromGitCommit(ref
)
835 settings
= extractSettingsGitLog(log
)
838 depotPaths
= settings
['depot-paths']
839 change
= settings
['change']
842 print "%s rewound to %s" % (ref
, change
)
846 class P4Submit(Command
, P4UserMap
):
848 Command
.__init
__(self
)
849 P4UserMap
.__init
__(self
)
851 optparse
.make_option("--origin", dest
="origin"),
852 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
853 # preserve the user, requires relevant p4 permissions
854 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
855 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
856 optparse
.make_option("--dry-run", "-n", dest
="dry_run", action
="store_true"),
858 self
.description
= "Submit changes from git to the perforce depot."
859 self
.usage
+= " [name of git branch to submit into perforce depot]"
861 self
.detectRenames
= False
862 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
864 self
.isWindows
= (platform
.system() == "Windows")
865 self
.exportLabels
= False
866 self
.p4HasMoveCommand
= p4_has_command("move")
869 if len(p4CmdList("opened ...")) > 0:
870 die("You have files opened with perforce! Close them before starting the sync.")
872 def separate_jobs_from_description(self
, message
):
873 """Extract and return a possible Jobs field in the commit
874 message. It goes into a separate section in the p4 change
877 A jobs line starts with "Jobs:" and looks like a new field
878 in a form. Values are white-space separated on the same
879 line or on following lines that start with a tab.
881 This does not parse and extract the full git commit message
882 like a p4 form. It just sees the Jobs: line as a marker
883 to pass everything from then on directly into the p4 form,
884 but outside the description section.
886 Return a tuple (stripped log message, jobs string)."""
888 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
890 return (message
, None)
892 jobtext
= message
[m
.start():]
893 stripped_message
= message
[:m
.start()].rstrip()
894 return (stripped_message
, jobtext
)
896 def prepareLogMessage(self
, template
, message
, jobs
):
897 """Edits the template returned from "p4 change -o" to insert
898 the message in the Description field, and the jobs text in
902 inDescriptionSection
= False
904 for line
in template
.split("\n"):
905 if line
.startswith("#"):
906 result
+= line
+ "\n"
909 if inDescriptionSection
:
910 if line
.startswith("Files:") or line
.startswith("Jobs:"):
911 inDescriptionSection
= False
912 # insert Jobs section
914 result
+= jobs
+ "\n"
918 if line
.startswith("Description:"):
919 inDescriptionSection
= True
921 for messageLine
in message
.split("\n"):
922 line
+= "\t" + messageLine
+ "\n"
924 result
+= line
+ "\n"
928 def patchRCSKeywords(self
, file, pattern
):
929 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
930 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
932 outFile
= os
.fdopen(handle
, "w+")
933 inFile
= open(file, "r")
934 regexp
= re
.compile(pattern
, re
.VERBOSE
)
935 for line
in inFile
.readlines():
936 line
= regexp
.sub(r
'$\1$', line
)
940 # Forcibly overwrite the original file
942 shutil
.move(outFileName
, file)
944 # cleanup our temporary file
945 os
.unlink(outFileName
)
946 print "Failed to strip RCS keywords in %s" % file
949 print "Patched up RCS keywords in %s" % file
951 def p4UserForCommit(self
,id):
952 # Return the tuple (perforce user,git email) for a given git commit id
953 self
.getUserMapFromPerforceServer()
954 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
955 gitEmail
= gitEmail
.strip()
956 if not self
.emails
.has_key(gitEmail
):
957 return (None,gitEmail
)
959 return (self
.emails
[gitEmail
],gitEmail
)
961 def checkValidP4Users(self
,commits
):
962 # check if any git authors cannot be mapped to p4 users
964 (user
,email
) = self
.p4UserForCommit(id)
966 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
967 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
970 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
972 def lastP4Changelist(self
):
973 # Get back the last changelist number submitted in this client spec. This
974 # then gets used to patch up the username in the change. If the same
975 # client spec is being used by multiple processes then this might go
977 results
= p4CmdList("client -o") # find the current client
980 if r
.has_key('Client'):
984 die("could not get client spec")
985 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
987 if r
.has_key('change'):
989 die("Could not get changelist number for last submit - cannot patch up user details")
991 def modifyChangelistUser(self
, changelist
, newUser
):
992 # fixup the user field of a changelist after it has been submitted.
993 changes
= p4CmdList("change -o %s" % changelist
)
994 if len(changes
) != 1:
995 die("Bad output from p4 change modifying %s to user %s" %
996 (changelist
, newUser
))
999 if c
['User'] == newUser
: return # nothing to do
1001 input = marshal
.dumps(c
)
1003 result
= p4CmdList("change -f -i", stdin
=input)
1005 if r
.has_key('code'):
1006 if r
['code'] == 'error':
1007 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
1008 if r
.has_key('data'):
1009 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
1011 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
1013 def canChangeChangelists(self
):
1014 # check to see if we have p4 admin or super-user permissions, either of
1015 # which are required to modify changelists.
1016 results
= p4CmdList(["protects", self
.depotPath
])
1018 if r
.has_key('perm'):
1019 if r
['perm'] == 'admin':
1021 if r
['perm'] == 'super':
1025 def prepareSubmitTemplate(self
):
1026 """Run "p4 change -o" to grab a change specification template.
1027 This does not use "p4 -G", as it is nice to keep the submission
1028 template in original order, since a human might edit it.
1030 Remove lines in the Files section that show changes to files
1031 outside the depot path we're committing into."""
1034 inFilesSection
= False
1035 for line
in p4_read_pipe_lines(['change', '-o']):
1036 if line
.endswith("\r\n"):
1037 line
= line
[:-2] + "\n"
1039 if line
.startswith("\t"):
1040 # path starts and ends with a tab
1042 lastTab
= path
.rfind("\t")
1044 path
= path
[:lastTab
]
1045 if not p4PathStartsWith(path
, self
.depotPath
):
1048 inFilesSection
= False
1050 if line
.startswith("Files:"):
1051 inFilesSection
= True
1057 def edit_template(self
, template_file
):
1058 """Invoke the editor to let the user change the submission
1059 message. Return true if okay to continue with the submit."""
1061 # if configured to skip the editing part, just submit
1062 if gitConfig("git-p4.skipSubmitEdit") == "true":
1065 # look at the modification time, to check later if the user saved
1067 mtime
= os
.stat(template_file
).st_mtime
1070 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1071 editor
= os
.environ
.get("P4EDITOR")
1073 editor
= read_pipe("git var GIT_EDITOR").strip()
1074 system(editor
+ " " + template_file
)
1076 # If the file was not saved, prompt to see if this patch should
1077 # be skipped. But skip this verification step if configured so.
1078 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1081 # modification time updated means user saved the file
1082 if os
.stat(template_file
).st_mtime
> mtime
:
1086 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1092 def applyCommit(self
, id):
1093 """Apply one commit, return True if it succeeded."""
1095 print "Applying", read_pipe(["git", "show", "-s",
1096 "--format=format:%h %s", id])
1098 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1100 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self
.diffOpts
, id, id))
1102 filesToDelete
= set()
1104 pureRenameCopy
= set()
1105 filesToChangeExecBit
= {}
1108 diff
= parseDiffTreeEntry(line
)
1109 modifier
= diff
['status']
1113 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1114 filesToChangeExecBit
[path
] = diff
['dst_mode']
1115 editedFiles
.add(path
)
1116 elif modifier
== "A":
1117 filesToAdd
.add(path
)
1118 filesToChangeExecBit
[path
] = diff
['dst_mode']
1119 if path
in filesToDelete
:
1120 filesToDelete
.remove(path
)
1121 elif modifier
== "D":
1122 filesToDelete
.add(path
)
1123 if path
in filesToAdd
:
1124 filesToAdd
.remove(path
)
1125 elif modifier
== "C":
1126 src
, dest
= diff
['src'], diff
['dst']
1127 p4_integrate(src
, dest
)
1128 pureRenameCopy
.add(dest
)
1129 if diff
['src_sha1'] != diff
['dst_sha1']:
1131 pureRenameCopy
.discard(dest
)
1132 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1134 pureRenameCopy
.discard(dest
)
1135 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1137 editedFiles
.add(dest
)
1138 elif modifier
== "R":
1139 src
, dest
= diff
['src'], diff
['dst']
1140 if self
.p4HasMoveCommand
:
1141 p4_edit(src
) # src must be open before move
1142 p4_move(src
, dest
) # opens for (move/delete, move/add)
1144 p4_integrate(src
, dest
)
1145 if diff
['src_sha1'] != diff
['dst_sha1']:
1148 pureRenameCopy
.add(dest
)
1149 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1150 if not self
.p4HasMoveCommand
:
1151 p4_edit(dest
) # with move: already open, writable
1152 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1153 if not self
.p4HasMoveCommand
:
1155 filesToDelete
.add(src
)
1156 editedFiles
.add(dest
)
1158 die("unknown modifier %s for %s" % (modifier
, path
))
1160 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1161 patchcmd
= diffcmd
+ " | git apply "
1162 tryPatchCmd
= patchcmd
+ "--check -"
1163 applyPatchCmd
= patchcmd
+ "--check --apply -"
1164 patch_succeeded
= True
1166 if os
.system(tryPatchCmd
) != 0:
1167 fixed_rcs_keywords
= False
1168 patch_succeeded
= False
1169 print "Unfortunately applying the change failed!"
1171 # Patch failed, maybe it's just RCS keyword woes. Look through
1172 # the patch to see if that's possible.
1173 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1177 for file in editedFiles | filesToDelete
:
1178 # did this file's delta contain RCS keywords?
1179 pattern
= p4_keywords_regexp_for_file(file)
1182 # this file is a possibility...look for RCS keywords.
1183 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1184 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1185 if regexp
.search(line
):
1187 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1188 kwfiles
[file] = pattern
1191 for file in kwfiles
:
1193 print "zapping %s with %s" % (line
,pattern
)
1194 self
.patchRCSKeywords(file, kwfiles
[file])
1195 fixed_rcs_keywords
= True
1197 if fixed_rcs_keywords
:
1198 print "Retrying the patch with RCS keywords cleaned up"
1199 if os
.system(tryPatchCmd
) == 0:
1200 patch_succeeded
= True
1202 if not patch_succeeded
:
1203 for f
in editedFiles
:
1208 # Apply the patch for real, and do add/delete/+x handling.
1210 system(applyPatchCmd
)
1212 for f
in filesToAdd
:
1214 for f
in filesToDelete
:
1218 # Set/clear executable bits
1219 for f
in filesToChangeExecBit
.keys():
1220 mode
= filesToChangeExecBit
[f
]
1221 setP4ExecBit(f
, mode
)
1224 # Build p4 change description, starting with the contents
1225 # of the git commit message.
1227 logMessage
= extractLogMessageFromGitCommit(id)
1228 logMessage
= logMessage
.strip()
1229 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1231 template
= self
.prepareSubmitTemplate()
1232 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1234 if self
.preserveUser
:
1235 submitTemplate
+= "\n######## Actual user %s, modified after commit\n" % p4User
1237 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1238 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1239 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1240 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1242 separatorLine
= "######## everything below this line is just the diff #######\n"
1245 if os
.environ
.has_key("P4DIFF"):
1246 del(os
.environ
["P4DIFF"])
1248 for editedFile
in editedFiles
:
1249 diff
+= p4_read_pipe(['diff', '-du',
1250 wildcard_encode(editedFile
)])
1254 for newFile
in filesToAdd
:
1255 newdiff
+= "==== new file ====\n"
1256 newdiff
+= "--- /dev/null\n"
1257 newdiff
+= "+++ %s\n" % newFile
1258 f
= open(newFile
, "r")
1259 for line
in f
.readlines():
1260 newdiff
+= "+" + line
1263 # change description file: submitTemplate, separatorLine, diff, newdiff
1264 (handle
, fileName
) = tempfile
.mkstemp()
1265 tmpFile
= os
.fdopen(handle
, "w+")
1267 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1268 separatorLine
= separatorLine
.replace("\n", "\r\n")
1269 newdiff
= newdiff
.replace("\n", "\r\n")
1270 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1274 # Let the user edit the change description, then submit it.
1276 if self
.edit_template(fileName
):
1277 # read the edited message and submit
1279 tmpFile
= open(fileName
, "rb")
1280 message
= tmpFile
.read()
1282 submitTemplate
= message
[:message
.index(separatorLine
)]
1284 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1285 p4_write_pipe(['submit', '-i'], submitTemplate
)
1287 if self
.preserveUser
:
1289 # Get last changelist number. Cannot easily get it from
1290 # the submit command output as the output is
1292 changelist
= self
.lastP4Changelist()
1293 self
.modifyChangelistUser(changelist
, p4User
)
1295 # The rename/copy happened by applying a patch that created a
1296 # new file. This leaves it writable, which confuses p4.
1297 for f
in pureRenameCopy
:
1303 print "Submission cancelled, undoing p4 changes."
1304 for f
in editedFiles
:
1306 for f
in filesToAdd
:
1309 for f
in filesToDelete
:
1315 # Export git tags as p4 labels. Create a p4 label and then tag
1317 def exportGitTags(self
, gitTags
):
1318 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1319 if len(validLabelRegexp
) == 0:
1320 validLabelRegexp
= defaultLabelRegexp
1321 m
= re
.compile(validLabelRegexp
)
1323 for name
in gitTags
:
1325 if not m
.match(name
):
1327 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1330 # Get the p4 commit this corresponds to
1331 logMessage
= extractLogMessageFromGitCommit(name
)
1332 values
= extractSettingsGitLog(logMessage
)
1334 if not values
.has_key('change'):
1335 # a tag pointing to something not sent to p4; ignore
1337 print "git tag %s does not give a p4 commit" % name
1340 changelist
= values
['change']
1342 # Get the tag details.
1346 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1349 if re
.match(r
'tag\s+', l
):
1351 elif re
.match(r
'\s*$', l
):
1358 body
= ["lightweight tag imported by git p4\n"]
1360 # Create the label - use the same view as the client spec we are using
1361 clientSpec
= getClientSpec()
1363 labelTemplate
= "Label: %s\n" % name
1364 labelTemplate
+= "Description:\n"
1366 labelTemplate
+= "\t" + b
+ "\n"
1367 labelTemplate
+= "View:\n"
1368 for mapping
in clientSpec
.mappings
:
1369 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1372 print "Would create p4 label %s for tag" % name
1374 p4_write_pipe(["label", "-i"], labelTemplate
)
1377 p4_system(["tag", "-l", name
] +
1378 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1381 print "created p4 label for tag %s" % name
1383 def run(self
, args
):
1385 self
.master
= currentGitBranch()
1386 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1387 die("Detecting current git branch failed!")
1388 elif len(args
) == 1:
1389 self
.master
= args
[0]
1390 if not branchExists(self
.master
):
1391 die("Branch %s does not exist" % self
.master
)
1395 allowSubmit
= gitConfig("git-p4.allowSubmit")
1396 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1397 die("%s is not in git-p4.allowSubmit" % self
.master
)
1399 [upstream
, settings
] = findUpstreamBranchPoint()
1400 self
.depotPath
= settings
['depot-paths'][0]
1401 if len(self
.origin
) == 0:
1402 self
.origin
= upstream
1404 if self
.preserveUser
:
1405 if not self
.canChangeChangelists():
1406 die("Cannot preserve user names without p4 super-user or admin permissions")
1409 print "Origin branch is " + self
.origin
1411 if len(self
.depotPath
) == 0:
1412 print "Internal error: cannot locate perforce depot path from existing branches"
1415 self
.useClientSpec
= False
1416 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1417 self
.useClientSpec
= True
1418 if self
.useClientSpec
:
1419 self
.clientSpecDirs
= getClientSpec()
1421 if self
.useClientSpec
:
1422 # all files are relative to the client spec
1423 self
.clientPath
= getClientRoot()
1425 self
.clientPath
= p4Where(self
.depotPath
)
1427 if self
.clientPath
== "":
1428 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1430 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1431 self
.oldWorkingDirectory
= os
.getcwd()
1433 # ensure the clientPath exists
1434 new_client_dir
= False
1435 if not os
.path
.exists(self
.clientPath
):
1436 new_client_dir
= True
1437 os
.makedirs(self
.clientPath
)
1439 chdir(self
.clientPath
)
1441 print "Would synchronize p4 checkout in %s" % self
.clientPath
1443 print "Synchronizing p4 checkout..."
1445 # old one was destroyed, and maybe nobody told p4
1446 p4_sync("...", "-f")
1452 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1453 commits
.append(line
.strip())
1456 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1457 self
.checkAuthorship
= False
1459 self
.checkAuthorship
= True
1461 if self
.preserveUser
:
1462 self
.checkValidP4Users(commits
)
1465 # Build up a set of options to be passed to diff when
1466 # submitting each commit to p4.
1468 if self
.detectRenames
:
1469 # command-line -M arg
1470 self
.diffOpts
= "-M"
1472 # If not explicitly set check the config variable
1473 detectRenames
= gitConfig("git-p4.detectRenames")
1475 if detectRenames
.lower() == "false" or detectRenames
== "":
1477 elif detectRenames
.lower() == "true":
1478 self
.diffOpts
= "-M"
1480 self
.diffOpts
= "-M%s" % detectRenames
1482 # no command-line arg for -C or --find-copies-harder, just
1484 detectCopies
= gitConfig("git-p4.detectCopies")
1485 if detectCopies
.lower() == "false" or detectCopies
== "":
1487 elif detectCopies
.lower() == "true":
1488 self
.diffOpts
+= " -C"
1490 self
.diffOpts
+= " -C%s" % detectCopies
1492 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1493 self
.diffOpts
+= " --find-copies-harder"
1496 # Apply the commits, one at a time. On failure, ask if should
1497 # continue to try the rest of the patches, or quit.
1502 last
= len(commits
) - 1
1503 for i
, commit
in enumerate(commits
):
1505 print " ", read_pipe(["git", "show", "-s",
1506 "--format=format:%h %s", commit
])
1509 ok
= self
.applyCommit(commit
)
1511 applied
.append(commit
)
1516 print "What do you want to do?"
1517 response
= raw_input("[s]kip this commit but apply"
1518 " the rest, or [q]uit? ")
1521 if response
[0] == "s":
1522 print "Skipping this commit, but applying the rest"
1524 if response
[0] == "q":
1531 chdir(self
.oldWorkingDirectory
)
1535 elif len(commits
) == len(applied
):
1536 print "All commits applied!"
1545 if len(applied
) == 0:
1546 print "No commits applied."
1548 print "Applied only the commits marked with '*':"
1554 print star
, read_pipe(["git", "show", "-s",
1555 "--format=format:%h %s", c
])
1556 print "You will have to do 'git p4 sync' and rebase."
1558 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1559 self
.exportLabels
= True
1561 if self
.exportLabels
:
1562 p4Labels
= getP4Labels(self
.depotPath
)
1563 gitTags
= getGitTags()
1565 missingGitTags
= gitTags
- p4Labels
1566 self
.exportGitTags(missingGitTags
)
1568 # exit with error unless everything applied perfecly
1569 if len(commits
) != len(applied
):
1575 """Represent a p4 view ("p4 help views"), and map files in a
1576 repo according to the view."""
1579 """A depot or client path, possibly containing wildcards.
1580 The only one supported is ... at the end, currently.
1581 Initialize with the full path, with //depot or //client."""
1583 def __init__(self
, path
, is_depot
):
1585 self
.is_depot
= is_depot
1586 self
.find_wildcards()
1587 # remember the prefix bit, useful for relative mappings
1588 m
= re
.match("(//[^/]+/)", self
.path
)
1590 die("Path %s does not start with //prefix/" % self
.path
)
1592 if not self
.is_depot
:
1593 # strip //client/ on client paths
1594 self
.path
= self
.path
[len(prefix
):]
1596 def find_wildcards(self
):
1597 """Make sure wildcards are valid, and set up internal
1600 self
.ends_triple_dot
= False
1601 # There are three wildcards allowed in p4 views
1602 # (see "p4 help views"). This code knows how to
1603 # handle "..." (only at the end), but cannot deal with
1604 # "%%n" or "*". Only check the depot_side, as p4 should
1605 # validate that the client_side matches too.
1606 if re
.search(r
'%%[1-9]', self
.path
):
1607 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1608 if self
.path
.find("*") >= 0:
1609 die("Can't handle * wildcards in view: %s" % self
.path
)
1610 triple_dot_index
= self
.path
.find("...")
1611 if triple_dot_index
>= 0:
1612 if triple_dot_index
!= len(self
.path
) - 3:
1613 die("Can handle only single ... wildcard, at end: %s" %
1615 self
.ends_triple_dot
= True
1617 def ensure_compatible(self
, other_path
):
1618 """Make sure the wildcards agree."""
1619 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1620 die("Both paths must end with ... if either does;\n" +
1621 "paths: %s %s" % (self
.path
, other_path
.path
))
1623 def match_wildcards(self
, test_path
):
1624 """See if this test_path matches us, and fill in the value
1625 of the wildcards if so. Returns a tuple of
1626 (True|False, wildcards[]). For now, only the ... at end
1627 is supported, so at most one wildcard."""
1628 if self
.ends_triple_dot
:
1629 dotless
= self
.path
[:-3]
1630 if test_path
.startswith(dotless
):
1631 wildcard
= test_path
[len(dotless
):]
1632 return (True, [ wildcard
])
1634 if test_path
== self
.path
:
1638 def match(self
, test_path
):
1639 """Just return if it matches; don't bother with the wildcards."""
1640 b
, _
= self
.match_wildcards(test_path
)
1643 def fill_in_wildcards(self
, wildcards
):
1644 """Return the relative path, with the wildcards filled in
1645 if there are any."""
1646 if self
.ends_triple_dot
:
1647 return self
.path
[:-3] + wildcards
[0]
1651 class Mapping(object):
1652 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1653 # depot_side is without the trailing /... if it had one
1654 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1655 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1656 self
.overlay
= overlay
# started with "+"
1657 self
.exclude
= exclude
# started with "-"
1658 assert not (self
.overlay
and self
.exclude
)
1659 self
.depot_side
.ensure_compatible(self
.client_side
)
1667 return "View.Mapping: %s%s -> %s" % \
1668 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1670 def map_depot_to_client(self
, depot_path
):
1671 """Calculate the client path if using this mapping on the
1672 given depot path; does not consider the effect of other
1673 mappings in a view. Even excluded mappings are returned."""
1674 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1677 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1686 def append(self
, view_line
):
1687 """Parse a view line, splitting it into depot and client
1688 sides. Append to self.mappings, preserving order."""
1690 # Split the view line into exactly two words. P4 enforces
1691 # structure on these lines that simplifies this quite a bit.
1693 # Either or both words may be double-quoted.
1694 # Single quotes do not matter.
1695 # Double-quote marks cannot occur inside the words.
1696 # A + or - prefix is also inside the quotes.
1697 # There are no quotes unless they contain a space.
1698 # The line is already white-space stripped.
1699 # The two words are separated by a single space.
1701 if view_line
[0] == '"':
1702 # First word is double quoted. Find its end.
1703 close_quote_index
= view_line
.find('"', 1)
1704 if close_quote_index
<= 0:
1705 die("No first-word closing quote found: %s" % view_line
)
1706 depot_side
= view_line
[1:close_quote_index
]
1707 # skip closing quote and space
1708 rhs_index
= close_quote_index
+ 1 + 1
1710 space_index
= view_line
.find(" ")
1711 if space_index
<= 0:
1712 die("No word-splitting space found: %s" % view_line
)
1713 depot_side
= view_line
[0:space_index
]
1714 rhs_index
= space_index
+ 1
1716 if view_line
[rhs_index
] == '"':
1717 # Second word is double quoted. Make sure there is a
1718 # double quote at the end too.
1719 if not view_line
.endswith('"'):
1720 die("View line with rhs quote should end with one: %s" %
1723 client_side
= view_line
[rhs_index
+1:-1]
1725 client_side
= view_line
[rhs_index
:]
1727 # prefix + means overlay on previous mapping
1729 if depot_side
.startswith("+"):
1731 depot_side
= depot_side
[1:]
1733 # prefix - means exclude this path
1735 if depot_side
.startswith("-"):
1737 depot_side
= depot_side
[1:]
1739 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1740 self
.mappings
.append(m
)
1742 def map_in_client(self
, depot_path
):
1743 """Return the relative location in the client where this
1744 depot file should live. Returns "" if the file should
1745 not be mapped in the client."""
1750 # look at later entries first
1751 for m
in self
.mappings
[::-1]:
1753 # see where will this path end up in the client
1754 p
= m
.map_depot_to_client(depot_path
)
1757 # Depot path does not belong in client. Must remember
1758 # this, as previous items should not cause files to
1759 # exist in this path either. Remember that the list is
1760 # being walked from the end, which has higher precedence.
1761 # Overlap mappings do not exclude previous mappings.
1763 paths_filled
.append(m
.client_side
)
1766 # This mapping matched; no need to search any further.
1767 # But, the mapping could be rejected if the client path
1768 # has already been claimed by an earlier mapping (i.e.
1769 # one later in the list, which we are walking backwards).
1770 already_mapped_in_client
= False
1771 for f
in paths_filled
:
1772 # this is View.Path.match
1774 already_mapped_in_client
= True
1776 if not already_mapped_in_client
:
1777 # Include this file, unless it is from a line that
1778 # explicitly said to exclude it.
1782 # a match, even if rejected, always stops the search
1787 class P4Sync(Command
, P4UserMap
):
1788 delete_actions
= ( "delete", "move/delete", "purge" )
1791 Command
.__init
__(self
)
1792 P4UserMap
.__init
__(self
)
1794 optparse
.make_option("--branch", dest
="branch"),
1795 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1796 optparse
.make_option("--changesfile", dest
="changesFile"),
1797 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1798 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1799 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1800 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1801 help="Import into refs/heads/ , not refs/remotes"),
1802 optparse
.make_option("--max-changes", dest
="maxChanges"),
1803 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1804 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1805 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1806 help="Only sync files that are included in the Perforce Client Spec")
1808 self
.description
= """Imports from Perforce into a git repository.\n
1810 //depot/my/project/ -- to import the current head
1811 //depot/my/project/@all -- to import everything
1812 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1814 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1816 self
.usage
+= " //depot/path[@revRange]"
1818 self
.createdBranches
= set()
1819 self
.committedChanges
= set()
1821 self
.detectBranches
= False
1822 self
.detectLabels
= False
1823 self
.importLabels
= False
1824 self
.changesFile
= ""
1825 self
.syncWithOrigin
= True
1826 self
.importIntoRemotes
= True
1827 self
.maxChanges
= ""
1828 self
.isWindows
= (platform
.system() == "Windows")
1829 self
.keepRepoPath
= False
1830 self
.depotPaths
= None
1831 self
.p4BranchesInGit
= []
1832 self
.cloneExclude
= []
1833 self
.useClientSpec
= False
1834 self
.useClientSpec_from_options
= False
1835 self
.clientSpecDirs
= None
1836 self
.tempBranches
= []
1837 self
.tempBranchLocation
= "git-p4-tmp"
1839 if gitConfig("git-p4.syncFromOrigin") == "false":
1840 self
.syncWithOrigin
= False
1842 # Force a checkpoint in fast-import and wait for it to finish
1843 def checkpoint(self
):
1844 self
.gitStream
.write("checkpoint\n\n")
1845 self
.gitStream
.write("progress checkpoint\n\n")
1846 out
= self
.gitOutput
.readline()
1848 print "checkpoint finished: " + out
1850 def extractFilesFromCommit(self
, commit
):
1851 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1852 for path
in self
.cloneExclude
]
1855 while commit
.has_key("depotFile%s" % fnum
):
1856 path
= commit
["depotFile%s" % fnum
]
1858 if [p
for p
in self
.cloneExclude
1859 if p4PathStartsWith(path
, p
)]:
1862 found
= [p
for p
in self
.depotPaths
1863 if p4PathStartsWith(path
, p
)]
1870 file["rev"] = commit
["rev%s" % fnum
]
1871 file["action"] = commit
["action%s" % fnum
]
1872 file["type"] = commit
["type%s" % fnum
]
1877 def stripRepoPath(self
, path
, prefixes
):
1878 if self
.useClientSpec
:
1879 return self
.clientSpecDirs
.map_in_client(path
)
1881 if self
.keepRepoPath
:
1882 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1885 if p4PathStartsWith(path
, p
):
1886 path
= path
[len(p
):]
1890 def splitFilesIntoBranches(self
, commit
):
1893 while commit
.has_key("depotFile%s" % fnum
):
1894 path
= commit
["depotFile%s" % fnum
]
1895 found
= [p
for p
in self
.depotPaths
1896 if p4PathStartsWith(path
, p
)]
1903 file["rev"] = commit
["rev%s" % fnum
]
1904 file["action"] = commit
["action%s" % fnum
]
1905 file["type"] = commit
["type%s" % fnum
]
1908 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1909 relPath
= wildcard_decode(relPath
)
1911 for branch
in self
.knownBranches
.keys():
1913 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1914 if relPath
.startswith(branch
+ "/"):
1915 if branch
not in branches
:
1916 branches
[branch
] = []
1917 branches
[branch
].append(file)
1922 # output one file from the P4 stream
1923 # - helper for streamP4Files
1925 def streamOneP4File(self
, file, contents
):
1926 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1927 relPath
= wildcard_decode(relPath
)
1929 sys
.stderr
.write("%s\n" % relPath
)
1931 (type_base
, type_mods
) = split_p4_type(file["type"])
1934 if "x" in type_mods
:
1936 if type_base
== "symlink":
1938 # p4 print on a symlink contains "target\n"; remove the newline
1939 data
= ''.join(contents
)
1940 contents
= [data
[:-1]]
1942 if type_base
== "utf16":
1943 # p4 delivers different text in the python output to -G
1944 # than it does when using "print -o", or normal p4 client
1945 # operations. utf16 is converted to ascii or utf8, perhaps.
1946 # But ascii text saved as -t utf16 is completely mangled.
1947 # Invoke print -o to get the real contents.
1948 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1951 if type_base
== "apple":
1952 # Apple filetype files will be streamed as a concatenation of
1953 # its appledouble header and the contents. This is useless
1954 # on both macs and non-macs. If using "print -q -o xx", it
1955 # will create "xx" with the data, and "%xx" with the header.
1956 # This is also not very useful.
1958 # Ideally, someday, this script can learn how to generate
1959 # appledouble files directly and import those to git, but
1960 # non-mac machines can never find a use for apple filetype.
1961 print "\nIgnoring apple filetype file %s" % file['depotFile']
1964 # Perhaps windows wants unicode, utf16 newlines translated too;
1965 # but this is not doing it.
1966 if self
.isWindows
and type_base
== "text":
1968 for data
in contents
:
1969 data
= data
.replace("\r\n", "\n")
1970 mangled
.append(data
)
1973 # Note that we do not try to de-mangle keywords on utf16 files,
1974 # even though in theory somebody may want that.
1975 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1977 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1978 text
= ''.join(contents
)
1979 text
= regexp
.sub(r
'$\1$', text
)
1982 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1987 length
= length
+ len(d
)
1989 self
.gitStream
.write("data %d\n" % length
)
1991 self
.gitStream
.write(d
)
1992 self
.gitStream
.write("\n")
1994 def streamOneP4Deletion(self
, file):
1995 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1996 relPath
= wildcard_decode(relPath
)
1998 sys
.stderr
.write("delete %s\n" % relPath
)
1999 self
.gitStream
.write("D %s\n" % relPath
)
2001 # handle another chunk of streaming data
2002 def streamP4FilesCb(self
, marshalled
):
2004 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
2005 # start of a new file - output the old one first
2006 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2007 self
.stream_file
= {}
2008 self
.stream_contents
= []
2009 self
.stream_have_file_info
= False
2011 # pick up the new file information... for the
2012 # 'data' field we need to append to our array
2013 for k
in marshalled
.keys():
2015 self
.stream_contents
.append(marshalled
['data'])
2017 self
.stream_file
[k
] = marshalled
[k
]
2019 self
.stream_have_file_info
= True
2021 # Stream directly from "p4 files" into "git fast-import"
2022 def streamP4Files(self
, files
):
2028 # if using a client spec, only add the files that have
2029 # a path in the client
2030 if self
.clientSpecDirs
:
2031 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
2034 filesForCommit
.append(f
)
2035 if f
['action'] in self
.delete_actions
:
2036 filesToDelete
.append(f
)
2038 filesToRead
.append(f
)
2041 for f
in filesToDelete
:
2042 self
.streamOneP4Deletion(f
)
2044 if len(filesToRead
) > 0:
2045 self
.stream_file
= {}
2046 self
.stream_contents
= []
2047 self
.stream_have_file_info
= False
2049 # curry self argument
2050 def streamP4FilesCbSelf(entry
):
2051 self
.streamP4FilesCb(entry
)
2053 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
2055 p4CmdList(["-x", "-", "print"],
2057 cb
=streamP4FilesCbSelf
)
2060 if self
.stream_file
.has_key('depotFile'):
2061 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
2063 def make_email(self
, userid
):
2064 if userid
in self
.users
:
2065 return self
.users
[userid
]
2067 return "%s <a@b>" % userid
2070 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
2072 print "writing tag %s for commit %s" % (labelName
, commit
)
2073 gitStream
.write("tag %s\n" % labelName
)
2074 gitStream
.write("from %s\n" % commit
)
2076 if labelDetails
.has_key('Owner'):
2077 owner
= labelDetails
["Owner"]
2081 # Try to use the owner of the p4 label, or failing that,
2082 # the current p4 user id.
2084 email
= self
.make_email(owner
)
2086 email
= self
.make_email(self
.p4UserId())
2087 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2089 gitStream
.write("tagger %s\n" % tagger
)
2091 print "labelDetails=",labelDetails
2092 if labelDetails
.has_key('Description'):
2093 description
= labelDetails
['Description']
2095 description
= 'Label from git p4'
2097 gitStream
.write("data %d\n" % len(description
))
2098 gitStream
.write(description
)
2099 gitStream
.write("\n")
2101 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
2102 epoch
= details
["time"]
2103 author
= details
["user"]
2104 self
.branchPrefixes
= branchPrefixes
2107 print "commit into %s" % branch
2109 # start with reading files; if that fails, we should not
2113 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2114 new_files
.append (f
)
2116 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2118 self
.gitStream
.write("commit %s\n" % branch
)
2119 # gitStream.write("mark :%s\n" % details["change"])
2120 self
.committedChanges
.add(int(details
["change"]))
2122 if author
not in self
.users
:
2123 self
.getUserMapFromPerforceServer()
2124 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2126 self
.gitStream
.write("committer %s\n" % committer
)
2128 self
.gitStream
.write("data <<EOT\n")
2129 self
.gitStream
.write(details
["desc"])
2130 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2131 % (','.join (branchPrefixes
), details
["change"]))
2132 if len(details
['options']) > 0:
2133 self
.gitStream
.write(": options = %s" % details
['options'])
2134 self
.gitStream
.write("]\nEOT\n\n")
2138 print "parent %s" % parent
2139 self
.gitStream
.write("from %s\n" % parent
)
2141 self
.streamP4Files(new_files
)
2142 self
.gitStream
.write("\n")
2144 change
= int(details
["change"])
2146 if self
.labels
.has_key(change
):
2147 label
= self
.labels
[change
]
2148 labelDetails
= label
[0]
2149 labelRevisions
= label
[1]
2151 print "Change %s is labelled %s" % (change
, labelDetails
)
2153 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2154 for p
in branchPrefixes
])
2156 if len(files
) == len(labelRevisions
):
2160 if info
["action"] in self
.delete_actions
:
2162 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2164 if cleanedFiles
== labelRevisions
:
2165 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2169 print ("Tag %s does not match with change %s: files do not match."
2170 % (labelDetails
["label"], change
))
2174 print ("Tag %s does not match with change %s: file count is different."
2175 % (labelDetails
["label"], change
))
2177 # Build a dictionary of changelists and labels, for "detect-labels" option.
2178 def getLabels(self
):
2181 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2182 if len(l
) > 0 and not self
.silent
:
2183 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2186 label
= output
["label"]
2190 print "Querying files for label %s" % label
2191 for file in p4CmdList(["files"] +
2192 ["%s...@%s" % (p
, label
)
2193 for p
in self
.depotPaths
]):
2194 revisions
[file["depotFile"]] = file["rev"]
2195 change
= int(file["change"])
2196 if change
> newestChange
:
2197 newestChange
= change
2199 self
.labels
[newestChange
] = [output
, revisions
]
2202 print "Label changes: %s" % self
.labels
.keys()
2204 # Import p4 labels as git tags. A direct mapping does not
2205 # exist, so assume that if all the files are at the same revision
2206 # then we can use that, or it's something more complicated we should
2208 def importP4Labels(self
, stream
, p4Labels
):
2210 print "import p4 labels: " + ' '.join(p4Labels
)
2212 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2213 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2214 if len(validLabelRegexp
) == 0:
2215 validLabelRegexp
= defaultLabelRegexp
2216 m
= re
.compile(validLabelRegexp
)
2218 for name
in p4Labels
:
2221 if not m
.match(name
):
2223 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2226 if name
in ignoredP4Labels
:
2229 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2231 # get the most recent changelist for each file in this label
2232 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2233 for p
in self
.depotPaths
])
2235 if change
.has_key('change'):
2236 # find the corresponding git commit; take the oldest commit
2237 changelist
= int(change
['change'])
2238 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2239 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2240 if len(gitCommit
) == 0:
2241 print "could not find git commit for changelist %d" % changelist
2243 gitCommit
= gitCommit
.strip()
2245 # Convert from p4 time format
2247 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2249 print "Could not convert label time %s" % labelDetail
['Update']
2252 when
= int(time
.mktime(tmwhen
))
2253 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2255 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2258 print "Label %s has no changelists - possibly deleted?" % name
2261 # We can't import this label; don't try again as it will get very
2262 # expensive repeatedly fetching all the files for labels that will
2263 # never be imported. If the label is moved in the future, the
2264 # ignore will need to be removed manually.
2265 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2267 def guessProjectName(self
):
2268 for p
in self
.depotPaths
:
2271 p
= p
[p
.strip().rfind("/") + 1:]
2272 if not p
.endswith("/"):
2276 def getBranchMapping(self
):
2277 lostAndFoundBranches
= set()
2279 user
= gitConfig("git-p4.branchUser")
2281 command
= "branches -u %s" % user
2283 command
= "branches"
2285 for info
in p4CmdList(command
):
2286 details
= p4Cmd(["branch", "-o", info
["branch"]])
2288 while details
.has_key("View%s" % viewIdx
):
2289 paths
= details
["View%s" % viewIdx
].split(" ")
2290 viewIdx
= viewIdx
+ 1
2291 # require standard //depot/foo/... //depot/bar/... mapping
2292 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2295 destination
= paths
[1]
2297 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2298 source
= source
[len(self
.depotPaths
[0]):-4]
2299 destination
= destination
[len(self
.depotPaths
[0]):-4]
2301 if destination
in self
.knownBranches
:
2303 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2304 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2307 self
.knownBranches
[destination
] = source
2309 lostAndFoundBranches
.discard(destination
)
2311 if source
not in self
.knownBranches
:
2312 lostAndFoundBranches
.add(source
)
2314 # Perforce does not strictly require branches to be defined, so we also
2315 # check git config for a branch list.
2317 # Example of branch definition in git config file:
2319 # branchList=main:branchA
2320 # branchList=main:branchB
2321 # branchList=branchA:branchC
2322 configBranches
= gitConfigList("git-p4.branchList")
2323 for branch
in configBranches
:
2325 (source
, destination
) = branch
.split(":")
2326 self
.knownBranches
[destination
] = source
2328 lostAndFoundBranches
.discard(destination
)
2330 if source
not in self
.knownBranches
:
2331 lostAndFoundBranches
.add(source
)
2334 for branch
in lostAndFoundBranches
:
2335 self
.knownBranches
[branch
] = branch
2337 def getBranchMappingFromGitBranches(self
):
2338 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2339 for branch
in branches
.keys():
2340 if branch
== "master":
2343 branch
= branch
[len(self
.projectName
):]
2344 self
.knownBranches
[branch
] = branch
2346 def listExistingP4GitBranches(self
):
2347 # branches holds mapping from name to commit
2348 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2349 self
.p4BranchesInGit
= branches
.keys()
2350 for branch
in branches
.keys():
2351 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2353 def updateOptionDict(self
, d
):
2355 if self
.keepRepoPath
:
2356 option_keys
['keepRepoPath'] = 1
2358 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2360 def readOptions(self
, d
):
2361 self
.keepRepoPath
= (d
.has_key('options')
2362 and ('keepRepoPath' in d
['options']))
2364 def gitRefForBranch(self
, branch
):
2365 if branch
== "main":
2366 return self
.refPrefix
+ "master"
2368 if len(branch
) <= 0:
2371 return self
.refPrefix
+ self
.projectName
+ branch
2373 def gitCommitByP4Change(self
, ref
, change
):
2375 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2378 latestCommit
= parseRevision(ref
)
2382 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2383 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2388 log
= extractLogMessageFromGitCommit(next
)
2389 settings
= extractSettingsGitLog(log
)
2390 currentChange
= int(settings
['change'])
2392 print "current change %s" % currentChange
2394 if currentChange
== change
:
2396 print "found %s" % next
2399 if currentChange
< change
:
2400 earliestCommit
= "^%s" % next
2402 latestCommit
= "%s" % next
2406 def importNewBranch(self
, branch
, maxChange
):
2407 # make fast-import flush all changes to disk and update the refs using the checkpoint
2408 # command so that we can try to find the branch parent in the git history
2409 self
.gitStream
.write("checkpoint\n\n");
2410 self
.gitStream
.flush();
2411 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2412 range = "@1,%s" % maxChange
2413 #print "prefix" + branchPrefix
2414 changes
= p4ChangesForPaths([branchPrefix
], range)
2415 if len(changes
) <= 0:
2417 firstChange
= changes
[0]
2418 #print "first change in branch: %s" % firstChange
2419 sourceBranch
= self
.knownBranches
[branch
]
2420 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2421 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2422 #print "source " + sourceBranch
2424 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2425 #print "branch parent: %s" % branchParentChange
2426 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2427 if len(gitParent
) > 0:
2428 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2429 #print "parent git commit: %s" % gitParent
2431 self
.importChanges(changes
)
2434 def searchParent(self
, parent
, branch
, target
):
2436 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2438 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2441 print "Found parent of %s in commit %s" % (branch
, blob
)
2448 def importChanges(self
, changes
):
2450 for change
in changes
:
2451 description
= p4Cmd(["describe", str(change
)])
2452 self
.updateOptionDict(description
)
2455 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2460 if self
.detectBranches
:
2461 branches
= self
.splitFilesIntoBranches(description
)
2462 for branch
in branches
.keys():
2464 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2468 filesForCommit
= branches
[branch
]
2471 print "branch is %s" % branch
2473 self
.updatedBranches
.add(branch
)
2475 if branch
not in self
.createdBranches
:
2476 self
.createdBranches
.add(branch
)
2477 parent
= self
.knownBranches
[branch
]
2478 if parent
== branch
:
2481 fullBranch
= self
.projectName
+ branch
2482 if fullBranch
not in self
.p4BranchesInGit
:
2484 print("\n Importing new branch %s" % fullBranch
);
2485 if self
.importNewBranch(branch
, change
- 1):
2487 self
.p4BranchesInGit
.append(fullBranch
)
2489 print("\n Resuming with change %s" % change
);
2492 print "parent determined through known branches: %s" % parent
2494 branch
= self
.gitRefForBranch(branch
)
2495 parent
= self
.gitRefForBranch(parent
)
2498 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2500 if len(parent
) == 0 and branch
in self
.initialParents
:
2501 parent
= self
.initialParents
[branch
]
2502 del self
.initialParents
[branch
]
2506 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2508 print "Creating temporary branch: " + tempBranch
2509 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2510 self
.tempBranches
.append(tempBranch
)
2512 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2514 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2517 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2518 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2520 files
= self
.extractFilesFromCommit(description
)
2521 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2523 self
.initialParent
= ""
2525 print self
.gitError
.read()
2528 def importHeadRevision(self
, revision
):
2529 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2532 details
["user"] = "git perforce import user"
2533 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2534 % (' '.join(self
.depotPaths
), revision
))
2535 details
["change"] = revision
2539 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2541 for info
in p4CmdList(["files"] + fileArgs
):
2543 if 'code' in info
and info
['code'] == 'error':
2544 sys
.stderr
.write("p4 returned an error: %s\n"
2546 if info
['data'].find("must refer to client") >= 0:
2547 sys
.stderr
.write("This particular p4 error is misleading.\n")
2548 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2549 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2551 if 'p4ExitCode' in info
:
2552 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2556 change
= int(info
["change"])
2557 if change
> newestRevision
:
2558 newestRevision
= change
2560 if info
["action"] in self
.delete_actions
:
2561 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2562 #fileCnt = fileCnt + 1
2565 for prop
in ["depotFile", "rev", "action", "type" ]:
2566 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2568 fileCnt
= fileCnt
+ 1
2570 details
["change"] = newestRevision
2572 # Use time from top-most change so that all git p4 clones of
2573 # the same p4 repo have the same commit SHA1s.
2574 res
= p4CmdList("describe -s %d" % newestRevision
)
2577 if r
.has_key('time'):
2578 newestTime
= int(r
['time'])
2579 if newestTime
is None:
2580 die("\"describe -s\" on newest change %d did not give a time")
2581 details
["time"] = newestTime
2583 self
.updateOptionDict(details
)
2585 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2587 print "IO error with git fast-import. Is your git version recent enough?"
2588 print self
.gitError
.read()
2591 def run(self
, args
):
2592 self
.depotPaths
= []
2593 self
.changeRange
= ""
2594 self
.initialParent
= ""
2595 self
.previousDepotPaths
= []
2597 # map from branch depot path to parent branch
2598 self
.knownBranches
= {}
2599 self
.initialParents
= {}
2600 self
.hasOrigin
= originP4BranchesExist()
2601 if not self
.syncWithOrigin
:
2602 self
.hasOrigin
= False
2604 if self
.importIntoRemotes
:
2605 self
.refPrefix
= "refs/remotes/p4/"
2607 self
.refPrefix
= "refs/heads/p4/"
2609 if self
.syncWithOrigin
and self
.hasOrigin
:
2611 print "Syncing with origin first by calling git fetch origin"
2612 system("git fetch origin")
2614 if len(self
.branch
) == 0:
2615 self
.branch
= self
.refPrefix
+ "master"
2616 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2617 system("git update-ref %s refs/heads/p4" % self
.branch
)
2618 system("git branch -D p4");
2619 # create it /after/ importing, when master exists
2620 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2621 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2623 # accept either the command-line option, or the configuration variable
2624 if self
.useClientSpec
:
2625 # will use this after clone to set the variable
2626 self
.useClientSpec_from_options
= True
2628 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2629 self
.useClientSpec
= True
2630 if self
.useClientSpec
:
2631 self
.clientSpecDirs
= getClientSpec()
2633 # TODO: should always look at previous commits,
2634 # merge with previous imports, if possible.
2637 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2638 self
.listExistingP4GitBranches()
2640 if len(self
.p4BranchesInGit
) > 1:
2642 print "Importing from/into multiple branches"
2643 self
.detectBranches
= True
2646 print "branches: %s" % self
.p4BranchesInGit
2649 for branch
in self
.p4BranchesInGit
:
2650 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2652 settings
= extractSettingsGitLog(logMsg
)
2654 self
.readOptions(settings
)
2655 if (settings
.has_key('depot-paths')
2656 and settings
.has_key ('change')):
2657 change
= int(settings
['change']) + 1
2658 p4Change
= max(p4Change
, change
)
2660 depotPaths
= sorted(settings
['depot-paths'])
2661 if self
.previousDepotPaths
== []:
2662 self
.previousDepotPaths
= depotPaths
2665 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2666 prev_list
= prev
.split("/")
2667 cur_list
= cur
.split("/")
2668 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2669 if cur_list
[i
] <> prev_list
[i
]:
2673 paths
.append ("/".join(cur_list
[:i
+ 1]))
2675 self
.previousDepotPaths
= paths
2678 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2679 self
.changeRange
= "@%s,#head" % p4Change
2680 if not self
.detectBranches
:
2681 self
.initialParent
= parseRevision(self
.branch
)
2682 if not self
.silent
and not self
.detectBranches
:
2683 print "Performing incremental import into %s git branch" % self
.branch
2685 if not self
.branch
.startswith("refs/"):
2686 self
.branch
= "refs/heads/" + self
.branch
2688 if len(args
) == 0 and self
.depotPaths
:
2690 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2692 if self
.depotPaths
and self
.depotPaths
!= args
:
2693 print ("previous import used depot path %s and now %s was specified. "
2694 "This doesn't work!" % (' '.join (self
.depotPaths
),
2698 self
.depotPaths
= sorted(args
)
2703 # Make sure no revision specifiers are used when --changesfile
2705 bad_changesfile
= False
2706 if len(self
.changesFile
) > 0:
2707 for p
in self
.depotPaths
:
2708 if p
.find("@") >= 0 or p
.find("#") >= 0:
2709 bad_changesfile
= True
2712 die("Option --changesfile is incompatible with revision specifiers")
2715 for p
in self
.depotPaths
:
2716 if p
.find("@") != -1:
2717 atIdx
= p
.index("@")
2718 self
.changeRange
= p
[atIdx
:]
2719 if self
.changeRange
== "@all":
2720 self
.changeRange
= ""
2721 elif ',' not in self
.changeRange
:
2722 revision
= self
.changeRange
2723 self
.changeRange
= ""
2725 elif p
.find("#") != -1:
2726 hashIdx
= p
.index("#")
2727 revision
= p
[hashIdx
:]
2729 elif self
.previousDepotPaths
== []:
2730 # pay attention to changesfile, if given, else import
2731 # the entire p4 tree at the head revision
2732 if len(self
.changesFile
) == 0:
2735 p
= re
.sub ("\.\.\.$", "", p
)
2736 if not p
.endswith("/"):
2741 self
.depotPaths
= newPaths
2743 self
.loadUserMapFromCache()
2745 if self
.detectLabels
:
2748 if self
.detectBranches
:
2749 ## FIXME - what's a P4 projectName ?
2750 self
.projectName
= self
.guessProjectName()
2753 self
.getBranchMappingFromGitBranches()
2755 self
.getBranchMapping()
2757 print "p4-git branches: %s" % self
.p4BranchesInGit
2758 print "initial parents: %s" % self
.initialParents
2759 for b
in self
.p4BranchesInGit
:
2763 b
= b
[len(self
.projectName
):]
2764 self
.createdBranches
.add(b
)
2766 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2768 importProcess
= subprocess
.Popen(["git", "fast-import"],
2769 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2770 stderr
=subprocess
.PIPE
);
2771 self
.gitOutput
= importProcess
.stdout
2772 self
.gitStream
= importProcess
.stdin
2773 self
.gitError
= importProcess
.stderr
2776 self
.importHeadRevision(revision
)
2780 if len(self
.changesFile
) > 0:
2781 output
= open(self
.changesFile
).readlines()
2784 changeSet
.add(int(line
))
2786 for change
in changeSet
:
2787 changes
.append(change
)
2791 # catch "git p4 sync" with no new branches, in a repo that
2792 # does not have any existing p4 branches
2793 if len(args
) == 0 and not self
.p4BranchesInGit
:
2794 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2796 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2798 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2800 if len(self
.maxChanges
) > 0:
2801 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2803 if len(changes
) == 0:
2805 print "No changes to import!"
2807 if not self
.silent
and not self
.detectBranches
:
2808 print "Import destination: %s" % self
.branch
2810 self
.updatedBranches
= set()
2812 self
.importChanges(changes
)
2816 if len(self
.updatedBranches
) > 0:
2817 sys
.stdout
.write("Updated branches: ")
2818 for b
in self
.updatedBranches
:
2819 sys
.stdout
.write("%s " % b
)
2820 sys
.stdout
.write("\n")
2822 if gitConfig("git-p4.importLabels", "--bool") == "true":
2823 self
.importLabels
= True
2825 if self
.importLabels
:
2826 p4Labels
= getP4Labels(self
.depotPaths
)
2827 gitTags
= getGitTags()
2829 missingP4Labels
= p4Labels
- gitTags
2830 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2832 self
.gitStream
.close()
2833 if importProcess
.wait() != 0:
2834 die("fast-import failed: %s" % self
.gitError
.read())
2835 self
.gitOutput
.close()
2836 self
.gitError
.close()
2838 # Cleanup temporary branches created during import
2839 if self
.tempBranches
!= []:
2840 for branch
in self
.tempBranches
:
2841 read_pipe("git update-ref -d %s" % branch
)
2842 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2846 class P4Rebase(Command
):
2848 Command
.__init
__(self
)
2850 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2852 self
.importLabels
= False
2853 self
.description
= ("Fetches the latest revision from perforce and "
2854 + "rebases the current work (branch) against it")
2856 def run(self
, args
):
2858 sync
.importLabels
= self
.importLabels
2861 return self
.rebase()
2864 if os
.system("git update-index --refresh") != 0:
2865 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2866 if len(read_pipe("git diff-index HEAD --")) > 0:
2867 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2869 [upstream
, settings
] = findUpstreamBranchPoint()
2870 if len(upstream
) == 0:
2871 die("Cannot find upstream branchpoint for rebase")
2873 # the branchpoint may be p4/foo~3, so strip off the parent
2874 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2876 print "Rebasing the current branch onto %s" % upstream
2877 oldHead
= read_pipe("git rev-parse HEAD").strip()
2878 system("git rebase %s" % upstream
)
2879 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2882 class P4Clone(P4Sync
):
2884 P4Sync
.__init
__(self
)
2885 self
.description
= "Creates a new git repository and imports from Perforce into it"
2886 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2888 optparse
.make_option("--destination", dest
="cloneDestination",
2889 action
='store', default
=None,
2890 help="where to leave result of the clone"),
2891 optparse
.make_option("-/", dest
="cloneExclude",
2892 action
="append", type="string",
2893 help="exclude depot path"),
2894 optparse
.make_option("--bare", dest
="cloneBare",
2895 action
="store_true", default
=False),
2897 self
.cloneDestination
= None
2898 self
.needsGit
= False
2899 self
.cloneBare
= False
2901 # This is required for the "append" cloneExclude action
2902 def ensure_value(self
, attr
, value
):
2903 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2904 setattr(self
, attr
, value
)
2905 return getattr(self
, attr
)
2907 def defaultDestination(self
, args
):
2908 ## TODO: use common prefix of args?
2910 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2911 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2912 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2913 depotDir
= re
.sub(r
"/$", "", depotDir
)
2914 return os
.path
.split(depotDir
)[1]
2916 def run(self
, args
):
2920 if self
.keepRepoPath
and not self
.cloneDestination
:
2921 sys
.stderr
.write("Must specify destination for --keep-path\n")
2926 if not self
.cloneDestination
and len(depotPaths
) > 1:
2927 self
.cloneDestination
= depotPaths
[-1]
2928 depotPaths
= depotPaths
[:-1]
2930 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2931 for p
in depotPaths
:
2932 if not p
.startswith("//"):
2935 if not self
.cloneDestination
:
2936 self
.cloneDestination
= self
.defaultDestination(args
)
2938 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2940 if not os
.path
.exists(self
.cloneDestination
):
2941 os
.makedirs(self
.cloneDestination
)
2942 chdir(self
.cloneDestination
)
2944 init_cmd
= [ "git", "init" ]
2946 init_cmd
.append("--bare")
2947 subprocess
.check_call(init_cmd
)
2949 if not P4Sync
.run(self
, depotPaths
):
2951 if self
.branch
!= "master":
2952 if self
.importIntoRemotes
:
2953 masterbranch
= "refs/remotes/p4/master"
2955 masterbranch
= "refs/heads/p4/master"
2956 if gitBranchExists(masterbranch
):
2957 system("git branch master %s" % masterbranch
)
2958 if not self
.cloneBare
:
2959 system("git checkout -f")
2961 print "Could not detect main branch. No checkout/master branch created."
2963 # auto-set this variable if invoked with --use-client-spec
2964 if self
.useClientSpec_from_options
:
2965 system("git config --bool git-p4.useclientspec true")
2969 class P4Branches(Command
):
2971 Command
.__init
__(self
)
2973 self
.description
= ("Shows the git branches that hold imports and their "
2974 + "corresponding perforce depot paths")
2975 self
.verbose
= False
2977 def run(self
, args
):
2978 if originP4BranchesExist():
2979 createOrUpdateBranchesFromOrigin()
2981 cmdline
= "git rev-parse --symbolic "
2982 cmdline
+= " --remotes"
2984 for line
in read_pipe_lines(cmdline
):
2987 if not line
.startswith('p4/') or line
== "p4/HEAD":
2991 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2992 settings
= extractSettingsGitLog(log
)
2994 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2997 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2999 optparse
.IndentedHelpFormatter
.__init
__(self
)
3001 def format_description(self
, description
):
3003 return description
+ "\n"
3007 def printUsage(commands
):
3008 print "usage: %s <command> [options]" % sys
.argv
[0]
3010 print "valid commands: %s" % ", ".join(commands
)
3012 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
3017 "submit" : P4Submit
,
3018 "commit" : P4Submit
,
3020 "rebase" : P4Rebase
,
3022 "rollback" : P4RollBack
,
3023 "branches" : P4Branches
3028 if len(sys
.argv
[1:]) == 0:
3029 printUsage(commands
.keys())
3033 cmdName
= sys
.argv
[1]
3035 klass
= commands
[cmdName
]
3038 print "unknown command %s" % cmdName
3040 printUsage(commands
.keys())
3043 options
= cmd
.options
3044 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
3048 options
.append(optparse
.make_option("--verbose", "-v", dest
="verbose", action
="store_true"))
3050 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
3052 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
3054 description
= cmd
.description
,
3055 formatter
= HelpFormatter())
3057 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
3059 verbose
= cmd
.verbose
3061 if cmd
.gitdir
== None:
3062 cmd
.gitdir
= os
.path
.abspath(".git")
3063 if not isValidGitDir(cmd
.gitdir
):
3064 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
3065 if os
.path
.exists(cmd
.gitdir
):
3066 cdup
= read_pipe("git rev-parse --show-cdup").strip()
3070 if not isValidGitDir(cmd
.gitdir
):
3071 if isValidGitDir(cmd
.gitdir
+ "/.git"):
3072 cmd
.gitdir
+= "/.git"
3074 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
3076 os
.environ
["GIT_DIR"] = cmd
.gitdir
3078 if not cmd
.run(args
):
3083 if __name__
== '__main__':