3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
11 import optparse
, sys
, os
, marshal
, subprocess
, shelve
12 import tempfile
, getopt
, os
.path
, time
, platform
17 # Only labels/tags matching this will be imported/exported
18 defaultLabelRegexp
= r
'[a-zA-Z0-9_\-.]+$'
20 def p4_build_cmd(cmd
):
21 """Build a suitable p4 command line.
23 This consolidates building and returning a p4 command line into one
24 location. It means that hooking into the environment, or other configuration
25 can be done more easily.
29 user
= gitConfig("git-p4.user")
31 real_cmd
+= ["-u",user
]
33 password
= gitConfig("git-p4.password")
35 real_cmd
+= ["-P", password
]
37 port
= gitConfig("git-p4.port")
39 real_cmd
+= ["-p", port
]
41 host
= gitConfig("git-p4.host")
43 real_cmd
+= ["-H", host
]
45 client
= gitConfig("git-p4.client")
47 real_cmd
+= ["-c", client
]
50 if isinstance(cmd
,basestring
):
51 real_cmd
= ' '.join(real_cmd
) + ' ' + cmd
57 # P4 uses the PWD environment variable rather than getcwd(). Since we're
58 # not using the shell, we have to set it ourselves. This path could
59 # be relative, so go there first, then figure out where we ended up.
61 os
.environ
['PWD'] = os
.getcwd()
67 sys
.stderr
.write(msg
+ "\n")
70 def write_pipe(c
, stdin
):
72 sys
.stderr
.write('Writing pipe: %s\n' % str(c
))
74 expand
= isinstance(c
,basestring
)
75 p
= subprocess
.Popen(c
, stdin
=subprocess
.PIPE
, shell
=expand
)
77 val
= pipe
.write(stdin
)
80 die('Command failed: %s' % str(c
))
84 def p4_write_pipe(c
, stdin
):
85 real_cmd
= p4_build_cmd(c
)
86 return write_pipe(real_cmd
, stdin
)
88 def read_pipe(c
, ignore_error
=False):
90 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
92 expand
= isinstance(c
,basestring
)
93 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
96 if p
.wait() and not ignore_error
:
97 die('Command failed: %s' % str(c
))
101 def p4_read_pipe(c
, ignore_error
=False):
102 real_cmd
= p4_build_cmd(c
)
103 return read_pipe(real_cmd
, ignore_error
)
105 def read_pipe_lines(c
):
107 sys
.stderr
.write('Reading pipe: %s\n' % str(c
))
109 expand
= isinstance(c
, basestring
)
110 p
= subprocess
.Popen(c
, stdout
=subprocess
.PIPE
, shell
=expand
)
112 val
= pipe
.readlines()
113 if pipe
.close() or p
.wait():
114 die('Command failed: %s' % str(c
))
118 def p4_read_pipe_lines(c
):
119 """Specifically invoke p4 on the command supplied. """
120 real_cmd
= p4_build_cmd(c
)
121 return read_pipe_lines(real_cmd
)
124 expand
= isinstance(cmd
,basestring
)
126 sys
.stderr
.write("executing %s\n" % str(cmd
))
127 subprocess
.check_call(cmd
, shell
=expand
)
130 """Specifically invoke p4 as the system command. """
131 real_cmd
= p4_build_cmd(cmd
)
132 expand
= isinstance(real_cmd
, basestring
)
133 subprocess
.check_call(real_cmd
, shell
=expand
)
135 def p4_integrate(src
, dest
):
136 p4_system(["integrate", "-Dt", wildcard_encode(src
), wildcard_encode(dest
)])
138 def p4_sync(f
, *options
):
139 p4_system(["sync"] + list(options
) + [wildcard_encode(f
)])
142 # forcibly add file names with wildcards
143 if wildcard_present(f
):
144 p4_system(["add", "-f", f
])
146 p4_system(["add", f
])
149 p4_system(["delete", wildcard_encode(f
)])
152 p4_system(["edit", wildcard_encode(f
)])
155 p4_system(["revert", wildcard_encode(f
)])
157 def p4_reopen(type, f
):
158 p4_system(["reopen", "-t", type, wildcard_encode(f
)])
161 # Canonicalize the p4 type and return a tuple of the
162 # base type, plus any modifiers. See "p4 help filetypes"
163 # for a list and explanation.
165 def split_p4_type(p4type
):
167 p4_filetypes_historical
= {
168 "ctempobj": "binary+Sw",
174 "tempobj": "binary+FSw",
175 "ubinary": "binary+F",
176 "uresource": "resource+F",
177 "uxbinary": "binary+Fx",
178 "xbinary": "binary+x",
180 "xtempobj": "binary+Swx",
182 "xunicode": "unicode+x",
185 if p4type
in p4_filetypes_historical
:
186 p4type
= p4_filetypes_historical
[p4type
]
188 s
= p4type
.split("+")
196 # return the raw p4 type of a file (text, text+ko, etc)
199 results
= p4CmdList(["fstat", "-T", "headType", file])
200 return results
[0]['headType']
203 # Given a type base and modifier, return a regexp matching
204 # the keywords that can be expanded in the file
206 def p4_keywords_regexp_for_type(base
, type_mods
):
207 if base
in ("text", "unicode", "binary"):
209 if "ko" in type_mods
:
211 elif "k" in type_mods
:
212 kwords
= 'Id|Header|Author|Date|DateTime|Change|File|Revision'
216 \$ # Starts with a dollar, followed by...
217 (%s) # one of the keywords, followed by...
218 (:[^$]+)? # possibly an old expansion, followed by...
226 # Given a file, return a regexp matching the possible
227 # RCS keywords that will be expanded, or None for files
228 # with kw expansion turned off.
230 def p4_keywords_regexp_for_file(file):
231 if not os
.path
.exists(file):
234 (type_base
, type_mods
) = split_p4_type(p4_type(file))
235 return p4_keywords_regexp_for_type(type_base
, type_mods
)
237 def setP4ExecBit(file, mode
):
238 # Reopens an already open file and changes the execute bit to match
239 # the execute bit setting in the passed in mode.
243 if not isModeExec(mode
):
244 p4Type
= getP4OpenedType(file)
245 p4Type
= re
.sub('^([cku]?)x(.*)', '\\1\\2', p4Type
)
246 p4Type
= re
.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type
)
247 if p4Type
[-1] == "+":
248 p4Type
= p4Type
[0:-1]
250 p4_reopen(p4Type
, file)
252 def getP4OpenedType(file):
253 # Returns the perforce file type for the given file.
255 result
= p4_read_pipe(["opened", wildcard_encode(file)])
256 match
= re
.match(".*\((.+)\)\r?$", result
)
258 return match
.group(1)
260 die("Could not determine file type for %s (result: '%s')" % (file, result
))
262 # Return the set of all p4 labels
263 def getP4Labels(depotPaths
):
265 if isinstance(depotPaths
,basestring
):
266 depotPaths
= [depotPaths
]
268 for l
in p4CmdList(["labels"] + ["%s..." % p
for p
in depotPaths
]):
274 # Return the set of all git tags
277 for line
in read_pipe_lines(["git", "tag"]):
282 def diffTreePattern():
283 # This is a simple generator for the diff tree regex pattern. This could be
284 # a class variable if this and parseDiffTreeEntry were a part of a class.
285 pattern
= re
.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
289 def parseDiffTreeEntry(entry
):
290 """Parses a single diff tree entry into its component elements.
292 See git-diff-tree(1) manpage for details about the format of the diff
293 output. This method returns a dictionary with the following elements:
295 src_mode - The mode of the source file
296 dst_mode - The mode of the destination file
297 src_sha1 - The sha1 for the source file
298 dst_sha1 - The sha1 fr the destination file
299 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
300 status_score - The score for the status (applicable for 'C' and 'R'
301 statuses). This is None if there is no score.
302 src - The path for the source file.
303 dst - The path for the destination file. This is only present for
304 copy or renames. If it is not present, this is None.
306 If the pattern is not matched, None is returned."""
308 match
= diffTreePattern().next().match(entry
)
311 'src_mode': match
.group(1),
312 'dst_mode': match
.group(2),
313 'src_sha1': match
.group(3),
314 'dst_sha1': match
.group(4),
315 'status': match
.group(5),
316 'status_score': match
.group(6),
317 'src': match
.group(7),
318 'dst': match
.group(10)
322 def isModeExec(mode
):
323 # Returns True if the given git mode represents an executable file,
325 return mode
[-3:] == "755"
327 def isModeExecChanged(src_mode
, dst_mode
):
328 return isModeExec(src_mode
) != isModeExec(dst_mode
)
330 def p4CmdList(cmd
, stdin
=None, stdin_mode
='w+b', cb
=None):
332 if isinstance(cmd
,basestring
):
339 cmd
= p4_build_cmd(cmd
)
341 sys
.stderr
.write("Opening pipe: %s\n" % str(cmd
))
343 # Use a temporary file to avoid deadlocks without
344 # subprocess.communicate(), which would put another copy
345 # of stdout into memory.
347 if stdin
is not None:
348 stdin_file
= tempfile
.TemporaryFile(prefix
='p4-stdin', mode
=stdin_mode
)
349 if isinstance(stdin
,basestring
):
350 stdin_file
.write(stdin
)
353 stdin_file
.write(i
+ '\n')
357 p4
= subprocess
.Popen(cmd
,
360 stdout
=subprocess
.PIPE
)
365 entry
= marshal
.load(p4
.stdout
)
375 entry
["p4ExitCode"] = exitCode
381 list = p4CmdList(cmd
)
387 def p4Where(depotPath
):
388 if not depotPath
.endswith("/"):
390 depotPath
= depotPath
+ "..."
391 outputList
= p4CmdList(["where", depotPath
])
393 for entry
in outputList
:
394 if "depotFile" in entry
:
395 if entry
["depotFile"] == depotPath
:
398 elif "data" in entry
:
399 data
= entry
.get("data")
400 space
= data
.find(" ")
401 if data
[:space
] == depotPath
:
406 if output
["code"] == "error":
410 clientPath
= output
.get("path")
411 elif "data" in output
:
412 data
= output
.get("data")
413 lastSpace
= data
.rfind(" ")
414 clientPath
= data
[lastSpace
+ 1:]
416 if clientPath
.endswith("..."):
417 clientPath
= clientPath
[:-3]
420 def currentGitBranch():
421 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
423 def isValidGitDir(path
):
424 if (os
.path
.exists(path
+ "/HEAD")
425 and os
.path
.exists(path
+ "/refs") and os
.path
.exists(path
+ "/objects")):
429 def parseRevision(ref
):
430 return read_pipe("git rev-parse %s" % ref
).strip()
432 def branchExists(ref
):
433 rev
= read_pipe(["git", "rev-parse", "-q", "--verify", ref
],
437 def extractLogMessageFromGitCommit(commit
):
440 ## fixme: title is first line of commit, not 1st paragraph.
442 for log
in read_pipe_lines("git cat-file commit %s" % commit
):
451 def extractSettingsGitLog(log
):
453 for line
in log
.split("\n"):
455 m
= re
.search (r
"^ *\[git-p4: (.*)\]$", line
)
459 assignments
= m
.group(1).split (':')
460 for a
in assignments
:
462 key
= vals
[0].strip()
463 val
= ('='.join (vals
[1:])).strip()
464 if val
.endswith ('\"') and val
.startswith('"'):
469 paths
= values
.get("depot-paths")
471 paths
= values
.get("depot-path")
473 values
['depot-paths'] = paths
.split(',')
476 def gitBranchExists(branch
):
477 proc
= subprocess
.Popen(["git", "rev-parse", branch
],
478 stderr
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
);
479 return proc
.wait() == 0;
482 def gitConfig(key
, args
= None): # set args to "--bool", for instance
483 if not _gitConfig
.has_key(key
):
486 argsFilter
= "%s " % args
487 cmd
= "git config %s%s" % (argsFilter
, key
)
488 _gitConfig
[key
] = read_pipe(cmd
, ignore_error
=True).strip()
489 return _gitConfig
[key
]
491 def gitConfigList(key
):
492 if not _gitConfig
.has_key(key
):
493 _gitConfig
[key
] = read_pipe("git config --get-all %s" % key
, ignore_error
=True).strip().split(os
.linesep
)
494 return _gitConfig
[key
]
496 def p4BranchesInGit(branchesAreInRemotes
= True):
499 cmdline
= "git rev-parse --symbolic "
500 if branchesAreInRemotes
:
501 cmdline
+= " --remotes"
503 cmdline
+= " --branches"
505 for line
in read_pipe_lines(cmdline
):
508 ## only import to p4/
509 if not line
.startswith('p4/') or line
== "p4/HEAD":
514 branch
= re
.sub ("^p4/", "", line
)
516 branches
[branch
] = parseRevision(line
)
519 def findUpstreamBranchPoint(head
= "HEAD"):
520 branches
= p4BranchesInGit()
521 # map from depot-path to branch name
522 branchByDepotPath
= {}
523 for branch
in branches
.keys():
524 tip
= branches
[branch
]
525 log
= extractLogMessageFromGitCommit(tip
)
526 settings
= extractSettingsGitLog(log
)
527 if settings
.has_key("depot-paths"):
528 paths
= ",".join(settings
["depot-paths"])
529 branchByDepotPath
[paths
] = "remotes/p4/" + branch
533 while parent
< 65535:
534 commit
= head
+ "~%s" % parent
535 log
= extractLogMessageFromGitCommit(commit
)
536 settings
= extractSettingsGitLog(log
)
537 if settings
.has_key("depot-paths"):
538 paths
= ",".join(settings
["depot-paths"])
539 if branchByDepotPath
.has_key(paths
):
540 return [branchByDepotPath
[paths
], settings
]
544 return ["", settings
]
546 def createOrUpdateBranchesFromOrigin(localRefPrefix
= "refs/remotes/p4/", silent
=True):
548 print ("Creating/updating branch(es) in %s based on origin branch(es)"
551 originPrefix
= "origin/p4/"
553 for line
in read_pipe_lines("git rev-parse --symbolic --remotes"):
555 if (not line
.startswith(originPrefix
)) or line
.endswith("HEAD"):
558 headName
= line
[len(originPrefix
):]
559 remoteHead
= localRefPrefix
+ headName
562 original
= extractSettingsGitLog(extractLogMessageFromGitCommit(originHead
))
563 if (not original
.has_key('depot-paths')
564 or not original
.has_key('change')):
568 if not gitBranchExists(remoteHead
):
570 print "creating %s" % remoteHead
573 settings
= extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead
))
574 if settings
.has_key('change') > 0:
575 if settings
['depot-paths'] == original
['depot-paths']:
576 originP4Change
= int(original
['change'])
577 p4Change
= int(settings
['change'])
578 if originP4Change
> p4Change
:
579 print ("%s (%s) is newer than %s (%s). "
580 "Updating p4 branch from origin."
581 % (originHead
, originP4Change
,
582 remoteHead
, p4Change
))
585 print ("Ignoring: %s was imported from %s while "
586 "%s was imported from %s"
587 % (originHead
, ','.join(original
['depot-paths']),
588 remoteHead
, ','.join(settings
['depot-paths'])))
591 system("git update-ref %s %s" % (remoteHead
, originHead
))
593 def originP4BranchesExist():
594 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
596 def p4ChangesForPaths(depotPaths
, changeRange
):
600 cmd
+= ["%s...%s" % (p
, changeRange
)]
601 output
= p4_read_pipe_lines(cmd
)
605 changeNum
= int(line
.split(" ")[1])
606 changes
[changeNum
] = True
608 changelist
= changes
.keys()
612 def p4PathStartsWith(path
, prefix
):
613 # This method tries to remedy a potential mixed-case issue:
615 # If UserA adds //depot/DirA/file1
616 # and UserB adds //depot/dira/file2
618 # we may or may not have a problem. If you have core.ignorecase=true,
619 # we treat DirA and dira as the same directory
620 ignorecase
= gitConfig("core.ignorecase", "--bool") == "true"
622 return path
.lower().startswith(prefix
.lower())
623 return path
.startswith(prefix
)
626 """Look at the p4 client spec, create a View() object that contains
627 all the mappings, and return it."""
629 specList
= p4CmdList("client -o")
630 if len(specList
) != 1:
631 die('Output from "client -o" is %d lines, expecting 1' %
634 # dictionary of all client parameters
637 # just the keys that start with "View"
638 view_keys
= [ k
for k
in entry
.keys() if k
.startswith("View") ]
643 # append the lines, in order, to the view
644 for view_num
in range(len(view_keys
)):
645 k
= "View%d" % view_num
646 if k
not in view_keys
:
647 die("Expected view key %s missing" % k
)
648 view
.append(entry
[k
])
653 """Grab the client directory."""
655 output
= p4CmdList("client -o")
657 die('Output from "client -o" is %d lines, expecting 1' % len(output
))
660 if "Root" not in entry
:
661 die('Client has no "Root"')
666 # P4 wildcards are not allowed in filenames. P4 complains
667 # if you simply add them, but you can force it with "-f", in
668 # which case it translates them into %xx encoding internally.
670 def wildcard_decode(path
):
671 # Search for and fix just these four characters. Do % last so
672 # that fixing it does not inadvertently create new %-escapes.
673 # Cannot have * in a filename in windows; untested as to
674 # what p4 would do in such a case.
675 if not platform
.system() == "Windows":
676 path
= path
.replace("%2A", "*")
677 path
= path
.replace("%23", "#") \
678 .replace("%40", "@") \
682 def wildcard_encode(path
):
683 # do % first to avoid double-encoding the %s introduced here
684 path
= path
.replace("%", "%25") \
685 .replace("*", "%2A") \
686 .replace("#", "%23") \
690 def wildcard_present(path
):
691 return path
.translate(None, "*#@%") != path
695 self
.usage
= "usage: %prog [options]"
701 self
.userMapFromPerforceServer
= False
702 self
.myP4UserId
= None
706 return self
.myP4UserId
708 results
= p4CmdList("user -o")
710 if r
.has_key('User'):
711 self
.myP4UserId
= r
['User']
713 die("Could not find your p4 user id")
715 def p4UserIsMe(self
, p4User
):
716 # return True if the given p4 user is actually me
718 if not p4User
or p4User
!= me
:
723 def getUserCacheFilename(self
):
724 home
= os
.environ
.get("HOME", os
.environ
.get("USERPROFILE"))
725 return home
+ "/.gitp4-usercache.txt"
727 def getUserMapFromPerforceServer(self
):
728 if self
.userMapFromPerforceServer
:
733 for output
in p4CmdList("users"):
734 if not output
.has_key("User"):
736 self
.users
[output
["User"]] = output
["FullName"] + " <" + output
["Email"] + ">"
737 self
.emails
[output
["Email"]] = output
["User"]
741 for (key
, val
) in self
.users
.items():
742 s
+= "%s\t%s\n" % (key
.expandtabs(1), val
.expandtabs(1))
744 open(self
.getUserCacheFilename(), "wb").write(s
)
745 self
.userMapFromPerforceServer
= True
747 def loadUserMapFromCache(self
):
749 self
.userMapFromPerforceServer
= False
751 cache
= open(self
.getUserCacheFilename(), "rb")
752 lines
= cache
.readlines()
755 entry
= line
.strip().split("\t")
756 self
.users
[entry
[0]] = entry
[1]
758 self
.getUserMapFromPerforceServer()
760 class P4Debug(Command
):
762 Command
.__init
__(self
)
764 self
.description
= "A tool to debug the output of p4 -G."
765 self
.needsGit
= False
769 for output
in p4CmdList(args
):
770 print 'Element: %d' % j
775 class P4RollBack(Command
):
777 Command
.__init
__(self
)
779 optparse
.make_option("--local", dest
="rollbackLocalBranches", action
="store_true")
781 self
.description
= "A tool to debug the multi-branch import. Don't use :)"
782 self
.rollbackLocalBranches
= False
787 maxChange
= int(args
[0])
789 if "p4ExitCode" in p4Cmd("changes -m 1"):
790 die("Problems executing p4");
792 if self
.rollbackLocalBranches
:
793 refPrefix
= "refs/heads/"
794 lines
= read_pipe_lines("git rev-parse --symbolic --branches")
796 refPrefix
= "refs/remotes/"
797 lines
= read_pipe_lines("git rev-parse --symbolic --remotes")
800 if self
.rollbackLocalBranches
or (line
.startswith("p4/") and line
!= "p4/HEAD\n"):
802 ref
= refPrefix
+ line
803 log
= extractLogMessageFromGitCommit(ref
)
804 settings
= extractSettingsGitLog(log
)
806 depotPaths
= settings
['depot-paths']
807 change
= settings
['change']
811 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p
, maxChange
)
812 for p
in depotPaths
]))) == 0:
813 print "Branch %s did not exist at change %s, deleting." % (ref
, maxChange
)
814 system("git update-ref -d %s `git rev-parse %s`" % (ref
, ref
))
817 while change
and int(change
) > maxChange
:
820 print "%s is at %s ; rewinding towards %s" % (ref
, change
, maxChange
)
821 system("git update-ref %s \"%s^\"" % (ref
, ref
))
822 log
= extractLogMessageFromGitCommit(ref
)
823 settings
= extractSettingsGitLog(log
)
826 depotPaths
= settings
['depot-paths']
827 change
= settings
['change']
830 print "%s rewound to %s" % (ref
, change
)
834 class P4Submit(Command
, P4UserMap
):
836 Command
.__init
__(self
)
837 P4UserMap
.__init
__(self
)
839 optparse
.make_option("--origin", dest
="origin"),
840 optparse
.make_option("-M", dest
="detectRenames", action
="store_true"),
841 # preserve the user, requires relevant p4 permissions
842 optparse
.make_option("--preserve-user", dest
="preserveUser", action
="store_true"),
843 optparse
.make_option("--export-labels", dest
="exportLabels", action
="store_true"),
845 self
.description
= "Submit changes from git to the perforce depot."
846 self
.usage
+= " [name of git branch to submit into perforce depot]"
848 self
.detectRenames
= False
849 self
.preserveUser
= gitConfig("git-p4.preserveUser").lower() == "true"
850 self
.isWindows
= (platform
.system() == "Windows")
851 self
.exportLabels
= False
854 if len(p4CmdList("opened ...")) > 0:
855 die("You have files opened with perforce! Close them before starting the sync.")
857 def separate_jobs_from_description(self
, message
):
858 """Extract and return a possible Jobs field in the commit
859 message. It goes into a separate section in the p4 change
862 A jobs line starts with "Jobs:" and looks like a new field
863 in a form. Values are white-space separated on the same
864 line or on following lines that start with a tab.
866 This does not parse and extract the full git commit message
867 like a p4 form. It just sees the Jobs: line as a marker
868 to pass everything from then on directly into the p4 form,
869 but outside the description section.
871 Return a tuple (stripped log message, jobs string)."""
873 m
= re
.search(r
'^Jobs:', message
, re
.MULTILINE
)
875 return (message
, None)
877 jobtext
= message
[m
.start():]
878 stripped_message
= message
[:m
.start()].rstrip()
879 return (stripped_message
, jobtext
)
881 def prepareLogMessage(self
, template
, message
, jobs
):
882 """Edits the template returned from "p4 change -o" to insert
883 the message in the Description field, and the jobs text in
887 inDescriptionSection
= False
889 for line
in template
.split("\n"):
890 if line
.startswith("#"):
891 result
+= line
+ "\n"
894 if inDescriptionSection
:
895 if line
.startswith("Files:") or line
.startswith("Jobs:"):
896 inDescriptionSection
= False
897 # insert Jobs section
899 result
+= jobs
+ "\n"
903 if line
.startswith("Description:"):
904 inDescriptionSection
= True
906 for messageLine
in message
.split("\n"):
907 line
+= "\t" + messageLine
+ "\n"
909 result
+= line
+ "\n"
913 def patchRCSKeywords(self
, file, pattern
):
914 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
915 (handle
, outFileName
) = tempfile
.mkstemp(dir='.')
917 outFile
= os
.fdopen(handle
, "w+")
918 inFile
= open(file, "r")
919 regexp
= re
.compile(pattern
, re
.VERBOSE
)
920 for line
in inFile
.readlines():
921 line
= regexp
.sub(r
'$\1$', line
)
925 # Forcibly overwrite the original file
927 shutil
.move(outFileName
, file)
929 # cleanup our temporary file
930 os
.unlink(outFileName
)
931 print "Failed to strip RCS keywords in %s" % file
934 print "Patched up RCS keywords in %s" % file
936 def p4UserForCommit(self
,id):
937 # Return the tuple (perforce user,git email) for a given git commit id
938 self
.getUserMapFromPerforceServer()
939 gitEmail
= read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
940 gitEmail
= gitEmail
.strip()
941 if not self
.emails
.has_key(gitEmail
):
942 return (None,gitEmail
)
944 return (self
.emails
[gitEmail
],gitEmail
)
946 def checkValidP4Users(self
,commits
):
947 # check if any git authors cannot be mapped to p4 users
949 (user
,email
) = self
.p4UserForCommit(id)
951 msg
= "Cannot find p4 user for email %s in commit %s." % (email
, id)
952 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
955 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg
)
957 def lastP4Changelist(self
):
958 # Get back the last changelist number submitted in this client spec. This
959 # then gets used to patch up the username in the change. If the same
960 # client spec is being used by multiple processes then this might go
962 results
= p4CmdList("client -o") # find the current client
965 if r
.has_key('Client'):
969 die("could not get client spec")
970 results
= p4CmdList(["changes", "-c", client
, "-m", "1"])
972 if r
.has_key('change'):
974 die("Could not get changelist number for last submit - cannot patch up user details")
976 def modifyChangelistUser(self
, changelist
, newUser
):
977 # fixup the user field of a changelist after it has been submitted.
978 changes
= p4CmdList("change -o %s" % changelist
)
979 if len(changes
) != 1:
980 die("Bad output from p4 change modifying %s to user %s" %
981 (changelist
, newUser
))
984 if c
['User'] == newUser
: return # nothing to do
986 input = marshal
.dumps(c
)
988 result
= p4CmdList("change -f -i", stdin
=input)
990 if r
.has_key('code'):
991 if r
['code'] == 'error':
992 die("Could not modify user field of changelist %s to %s:%s" % (changelist
, newUser
, r
['data']))
993 if r
.has_key('data'):
994 print("Updated user field for changelist %s to %s" % (changelist
, newUser
))
996 die("Could not modify user field of changelist %s to %s" % (changelist
, newUser
))
998 def canChangeChangelists(self
):
999 # check to see if we have p4 admin or super-user permissions, either of
1000 # which are required to modify changelists.
1001 results
= p4CmdList(["protects", self
.depotPath
])
1003 if r
.has_key('perm'):
1004 if r
['perm'] == 'admin':
1006 if r
['perm'] == 'super':
1010 def prepareSubmitTemplate(self
):
1011 """Run "p4 change -o" to grab a change specification template.
1012 This does not use "p4 -G", as it is nice to keep the submission
1013 template in original order, since a human might edit it.
1015 Remove lines in the Files section that show changes to files
1016 outside the depot path we're committing into."""
1019 inFilesSection
= False
1020 for line
in p4_read_pipe_lines(['change', '-o']):
1021 if line
.endswith("\r\n"):
1022 line
= line
[:-2] + "\n"
1024 if line
.startswith("\t"):
1025 # path starts and ends with a tab
1027 lastTab
= path
.rfind("\t")
1029 path
= path
[:lastTab
]
1030 if not p4PathStartsWith(path
, self
.depotPath
):
1033 inFilesSection
= False
1035 if line
.startswith("Files:"):
1036 inFilesSection
= True
1042 def edit_template(self
, template_file
):
1043 """Invoke the editor to let the user change the submission
1044 message. Return true if okay to continue with the submit."""
1046 # if configured to skip the editing part, just submit
1047 if gitConfig("git-p4.skipSubmitEdit") == "true":
1050 # look at the modification time, to check later if the user saved
1052 mtime
= os
.stat(template_file
).st_mtime
1055 if os
.environ
.has_key("P4EDITOR") and (os
.environ
.get("P4EDITOR") != ""):
1056 editor
= os
.environ
.get("P4EDITOR")
1058 editor
= read_pipe("git var GIT_EDITOR").strip()
1059 system(editor
+ " " + template_file
)
1061 # If the file was not saved, prompt to see if this patch should
1062 # be skipped. But skip this verification step if configured so.
1063 if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1066 # modification time updated means user saved the file
1067 if os
.stat(template_file
).st_mtime
> mtime
:
1071 response
= raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1077 def applyCommit(self
, id):
1078 print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1080 (p4User
, gitEmail
) = self
.p4UserForCommit(id)
1082 if not self
.detectRenames
:
1083 # If not explicitly set check the config variable
1084 self
.detectRenames
= gitConfig("git-p4.detectRenames")
1086 if self
.detectRenames
.lower() == "false" or self
.detectRenames
== "":
1088 elif self
.detectRenames
.lower() == "true":
1091 diffOpts
= "-M%s" % self
.detectRenames
1093 detectCopies
= gitConfig("git-p4.detectCopies")
1094 if detectCopies
.lower() == "true":
1096 elif detectCopies
!= "" and detectCopies
.lower() != "false":
1097 diffOpts
+= " -C%s" % detectCopies
1099 if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1100 diffOpts
+= " --find-copies-harder"
1102 diff
= read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts
, id, id))
1104 filesToDelete
= set()
1106 pureRenameCopy
= set()
1107 filesToChangeExecBit
= {}
1110 diff
= parseDiffTreeEntry(line
)
1111 modifier
= diff
['status']
1115 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1116 filesToChangeExecBit
[path
] = diff
['dst_mode']
1117 editedFiles
.add(path
)
1118 elif modifier
== "A":
1119 filesToAdd
.add(path
)
1120 filesToChangeExecBit
[path
] = diff
['dst_mode']
1121 if path
in filesToDelete
:
1122 filesToDelete
.remove(path
)
1123 elif modifier
== "D":
1124 filesToDelete
.add(path
)
1125 if path
in filesToAdd
:
1126 filesToAdd
.remove(path
)
1127 elif modifier
== "C":
1128 src
, dest
= diff
['src'], diff
['dst']
1129 p4_integrate(src
, dest
)
1130 pureRenameCopy
.add(dest
)
1131 if diff
['src_sha1'] != diff
['dst_sha1']:
1133 pureRenameCopy
.discard(dest
)
1134 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1136 pureRenameCopy
.discard(dest
)
1137 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1139 editedFiles
.add(dest
)
1140 elif modifier
== "R":
1141 src
, dest
= diff
['src'], diff
['dst']
1142 p4_integrate(src
, dest
)
1143 if diff
['src_sha1'] != diff
['dst_sha1']:
1146 pureRenameCopy
.add(dest
)
1147 if isModeExecChanged(diff
['src_mode'], diff
['dst_mode']):
1149 filesToChangeExecBit
[dest
] = diff
['dst_mode']
1151 editedFiles
.add(dest
)
1152 filesToDelete
.add(src
)
1154 die("unknown modifier %s for %s" % (modifier
, path
))
1156 diffcmd
= "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1157 patchcmd
= diffcmd
+ " | git apply "
1158 tryPatchCmd
= patchcmd
+ "--check -"
1159 applyPatchCmd
= patchcmd
+ "--check --apply -"
1160 patch_succeeded
= True
1162 if os
.system(tryPatchCmd
) != 0:
1163 fixed_rcs_keywords
= False
1164 patch_succeeded
= False
1165 print "Unfortunately applying the change failed!"
1167 # Patch failed, maybe it's just RCS keyword woes. Look through
1168 # the patch to see if that's possible.
1169 if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1173 for file in editedFiles | filesToDelete
:
1174 # did this file's delta contain RCS keywords?
1175 pattern
= p4_keywords_regexp_for_file(file)
1178 # this file is a possibility...look for RCS keywords.
1179 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1180 for line
in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1181 if regexp
.search(line
):
1183 print "got keyword match on %s in %s in %s" % (pattern
, line
, file)
1184 kwfiles
[file] = pattern
1187 for file in kwfiles
:
1189 print "zapping %s with %s" % (line
,pattern
)
1190 self
.patchRCSKeywords(file, kwfiles
[file])
1191 fixed_rcs_keywords
= True
1193 if fixed_rcs_keywords
:
1194 print "Retrying the patch with RCS keywords cleaned up"
1195 if os
.system(tryPatchCmd
) == 0:
1196 patch_succeeded
= True
1198 if not patch_succeeded
:
1199 print "What do you want to do?"
1201 while response
!= "s" and response
!= "a" and response
!= "w":
1202 response
= raw_input("[s]kip this patch / [a]pply the patch forcibly "
1203 "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1205 print "Skipping! Good luck with the next patches..."
1206 for f
in editedFiles
:
1208 for f
in filesToAdd
:
1211 elif response
== "a":
1212 os
.system(applyPatchCmd
)
1213 if len(filesToAdd
) > 0:
1214 print "You may also want to call p4 add on the following files:"
1215 print " ".join(filesToAdd
)
1216 if len(filesToDelete
):
1217 print "The following files should be scheduled for deletion with p4 delete:"
1218 print " ".join(filesToDelete
)
1219 die("Please resolve and submit the conflict manually and "
1220 + "continue afterwards with git p4 submit --continue")
1221 elif response
== "w":
1222 system(diffcmd
+ " > patch.txt")
1223 print "Patch saved to patch.txt in %s !" % self
.clientPath
1224 die("Please resolve and submit the conflict manually and "
1225 "continue afterwards with git p4 submit --continue")
1227 system(applyPatchCmd
)
1229 for f
in filesToAdd
:
1231 for f
in filesToDelete
:
1235 # Set/clear executable bits
1236 for f
in filesToChangeExecBit
.keys():
1237 mode
= filesToChangeExecBit
[f
]
1238 setP4ExecBit(f
, mode
)
1240 logMessage
= extractLogMessageFromGitCommit(id)
1241 logMessage
= logMessage
.strip()
1242 (logMessage
, jobs
) = self
.separate_jobs_from_description(logMessage
)
1244 template
= self
.prepareSubmitTemplate()
1245 submitTemplate
= self
.prepareLogMessage(template
, logMessage
, jobs
)
1247 if self
.preserveUser
:
1248 submitTemplate
= submitTemplate
+ ("\n######## Actual user %s, modified after commit\n" % p4User
)
1250 if os
.environ
.has_key("P4DIFF"):
1251 del(os
.environ
["P4DIFF"])
1253 for editedFile
in editedFiles
:
1254 diff
+= p4_read_pipe(['diff', '-du',
1255 wildcard_encode(editedFile
)])
1258 for newFile
in filesToAdd
:
1259 newdiff
+= "==== new file ====\n"
1260 newdiff
+= "--- /dev/null\n"
1261 newdiff
+= "+++ %s\n" % newFile
1262 f
= open(newFile
, "r")
1263 for line
in f
.readlines():
1264 newdiff
+= "+" + line
1267 if self
.checkAuthorship
and not self
.p4UserIsMe(p4User
):
1268 submitTemplate
+= "######## git author %s does not match your p4 account.\n" % gitEmail
1269 submitTemplate
+= "######## Use option --preserve-user to modify authorship.\n"
1270 submitTemplate
+= "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1272 separatorLine
= "######## everything below this line is just the diff #######\n"
1274 (handle
, fileName
) = tempfile
.mkstemp()
1275 tmpFile
= os
.fdopen(handle
, "w+")
1277 submitTemplate
= submitTemplate
.replace("\n", "\r\n")
1278 separatorLine
= separatorLine
.replace("\n", "\r\n")
1279 newdiff
= newdiff
.replace("\n", "\r\n")
1280 tmpFile
.write(submitTemplate
+ separatorLine
+ diff
+ newdiff
)
1283 if self
.edit_template(fileName
):
1284 # read the edited message and submit
1285 tmpFile
= open(fileName
, "rb")
1286 message
= tmpFile
.read()
1288 submitTemplate
= message
[:message
.index(separatorLine
)]
1290 submitTemplate
= submitTemplate
.replace("\r\n", "\n")
1291 p4_write_pipe(['submit', '-i'], submitTemplate
)
1293 if self
.preserveUser
:
1295 # Get last changelist number. Cannot easily get it from
1296 # the submit command output as the output is
1298 changelist
= self
.lastP4Changelist()
1299 self
.modifyChangelistUser(changelist
, p4User
)
1301 # The rename/copy happened by applying a patch that created a
1302 # new file. This leaves it writable, which confuses p4.
1303 for f
in pureRenameCopy
:
1308 print "Submission cancelled, undoing p4 changes."
1309 for f
in editedFiles
:
1311 for f
in filesToAdd
:
1317 # Export git tags as p4 labels. Create a p4 label and then tag
1319 def exportGitTags(self
, gitTags
):
1320 validLabelRegexp
= gitConfig("git-p4.labelExportRegexp")
1321 if len(validLabelRegexp
) == 0:
1322 validLabelRegexp
= defaultLabelRegexp
1323 m
= re
.compile(validLabelRegexp
)
1325 for name
in gitTags
:
1327 if not m
.match(name
):
1329 print "tag %s does not match regexp %s" % (name
, validLabelRegexp
)
1332 # Get the p4 commit this corresponds to
1333 logMessage
= extractLogMessageFromGitCommit(name
)
1334 values
= extractSettingsGitLog(logMessage
)
1336 if not values
.has_key('change'):
1337 # a tag pointing to something not sent to p4; ignore
1339 print "git tag %s does not give a p4 commit" % name
1342 changelist
= values
['change']
1344 # Get the tag details.
1348 for l
in read_pipe_lines(["git", "cat-file", "-p", name
]):
1351 if re
.match(r
'tag\s+', l
):
1353 elif re
.match(r
'\s*$', l
):
1360 body
= ["lightweight tag imported by git p4\n"]
1362 # Create the label - use the same view as the client spec we are using
1363 clientSpec
= getClientSpec()
1365 labelTemplate
= "Label: %s\n" % name
1366 labelTemplate
+= "Description:\n"
1368 labelTemplate
+= "\t" + b
+ "\n"
1369 labelTemplate
+= "View:\n"
1370 for mapping
in clientSpec
.mappings
:
1371 labelTemplate
+= "\t%s\n" % mapping
.depot_side
.path
1373 p4_write_pipe(["label", "-i"], labelTemplate
)
1376 p4_system(["tag", "-l", name
] +
1377 ["%s@%s" % (mapping
.depot_side
.path
, changelist
) for mapping
in clientSpec
.mappings
])
1380 print "created p4 label for tag %s" % name
1382 def run(self
, args
):
1384 self
.master
= currentGitBranch()
1385 if len(self
.master
) == 0 or not gitBranchExists("refs/heads/%s" % self
.master
):
1386 die("Detecting current git branch failed!")
1387 elif len(args
) == 1:
1388 self
.master
= args
[0]
1389 if not branchExists(self
.master
):
1390 die("Branch %s does not exist" % self
.master
)
1394 allowSubmit
= gitConfig("git-p4.allowSubmit")
1395 if len(allowSubmit
) > 0 and not self
.master
in allowSubmit
.split(","):
1396 die("%s is not in git-p4.allowSubmit" % self
.master
)
1398 [upstream
, settings
] = findUpstreamBranchPoint()
1399 self
.depotPath
= settings
['depot-paths'][0]
1400 if len(self
.origin
) == 0:
1401 self
.origin
= upstream
1403 if self
.preserveUser
:
1404 if not self
.canChangeChangelists():
1405 die("Cannot preserve user names without p4 super-user or admin permissions")
1408 print "Origin branch is " + self
.origin
1410 if len(self
.depotPath
) == 0:
1411 print "Internal error: cannot locate perforce depot path from existing branches"
1414 self
.useClientSpec
= False
1415 if gitConfig("git-p4.useclientspec", "--bool") == "true":
1416 self
.useClientSpec
= True
1417 if self
.useClientSpec
:
1418 self
.clientSpecDirs
= getClientSpec()
1420 if self
.useClientSpec
:
1421 # all files are relative to the client spec
1422 self
.clientPath
= getClientRoot()
1424 self
.clientPath
= p4Where(self
.depotPath
)
1426 if self
.clientPath
== "":
1427 die("Error: Cannot locate perforce checkout of %s in client view" % self
.depotPath
)
1429 print "Perforce checkout for depot path %s located at %s" % (self
.depotPath
, self
.clientPath
)
1430 self
.oldWorkingDirectory
= os
.getcwd()
1432 # ensure the clientPath exists
1433 new_client_dir
= False
1434 if not os
.path
.exists(self
.clientPath
):
1435 new_client_dir
= True
1436 os
.makedirs(self
.clientPath
)
1438 chdir(self
.clientPath
)
1439 print "Synchronizing p4 checkout..."
1441 # old one was destroyed, and maybe nobody told p4
1442 p4_sync("...", "-f")
1448 for line
in read_pipe_lines("git rev-list --no-merges %s..%s" % (self
.origin
, self
.master
)):
1449 commits
.append(line
.strip())
1452 if self
.preserveUser
or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1453 self
.checkAuthorship
= False
1455 self
.checkAuthorship
= True
1457 if self
.preserveUser
:
1458 self
.checkValidP4Users(commits
)
1460 while len(commits
) > 0:
1462 commits
= commits
[1:]
1463 self
.applyCommit(commit
)
1465 if len(commits
) == 0:
1466 print "All changes applied!"
1467 chdir(self
.oldWorkingDirectory
)
1475 if gitConfig("git-p4.exportLabels", "--bool") == "true":
1476 self
.exportLabels
= True
1478 if self
.exportLabels
:
1479 p4Labels
= getP4Labels(self
.depotPath
)
1480 gitTags
= getGitTags()
1482 missingGitTags
= gitTags
- p4Labels
1483 self
.exportGitTags(missingGitTags
)
1488 """Represent a p4 view ("p4 help views"), and map files in a
1489 repo according to the view."""
1492 """A depot or client path, possibly containing wildcards.
1493 The only one supported is ... at the end, currently.
1494 Initialize with the full path, with //depot or //client."""
1496 def __init__(self
, path
, is_depot
):
1498 self
.is_depot
= is_depot
1499 self
.find_wildcards()
1500 # remember the prefix bit, useful for relative mappings
1501 m
= re
.match("(//[^/]+/)", self
.path
)
1503 die("Path %s does not start with //prefix/" % self
.path
)
1505 if not self
.is_depot
:
1506 # strip //client/ on client paths
1507 self
.path
= self
.path
[len(prefix
):]
1509 def find_wildcards(self
):
1510 """Make sure wildcards are valid, and set up internal
1513 self
.ends_triple_dot
= False
1514 # There are three wildcards allowed in p4 views
1515 # (see "p4 help views"). This code knows how to
1516 # handle "..." (only at the end), but cannot deal with
1517 # "%%n" or "*". Only check the depot_side, as p4 should
1518 # validate that the client_side matches too.
1519 if re
.search(r
'%%[1-9]', self
.path
):
1520 die("Can't handle %%n wildcards in view: %s" % self
.path
)
1521 if self
.path
.find("*") >= 0:
1522 die("Can't handle * wildcards in view: %s" % self
.path
)
1523 triple_dot_index
= self
.path
.find("...")
1524 if triple_dot_index
>= 0:
1525 if triple_dot_index
!= len(self
.path
) - 3:
1526 die("Can handle only single ... wildcard, at end: %s" %
1528 self
.ends_triple_dot
= True
1530 def ensure_compatible(self
, other_path
):
1531 """Make sure the wildcards agree."""
1532 if self
.ends_triple_dot
!= other_path
.ends_triple_dot
:
1533 die("Both paths must end with ... if either does;\n" +
1534 "paths: %s %s" % (self
.path
, other_path
.path
))
1536 def match_wildcards(self
, test_path
):
1537 """See if this test_path matches us, and fill in the value
1538 of the wildcards if so. Returns a tuple of
1539 (True|False, wildcards[]). For now, only the ... at end
1540 is supported, so at most one wildcard."""
1541 if self
.ends_triple_dot
:
1542 dotless
= self
.path
[:-3]
1543 if test_path
.startswith(dotless
):
1544 wildcard
= test_path
[len(dotless
):]
1545 return (True, [ wildcard
])
1547 if test_path
== self
.path
:
1551 def match(self
, test_path
):
1552 """Just return if it matches; don't bother with the wildcards."""
1553 b
, _
= self
.match_wildcards(test_path
)
1556 def fill_in_wildcards(self
, wildcards
):
1557 """Return the relative path, with the wildcards filled in
1558 if there are any."""
1559 if self
.ends_triple_dot
:
1560 return self
.path
[:-3] + wildcards
[0]
1564 class Mapping(object):
1565 def __init__(self
, depot_side
, client_side
, overlay
, exclude
):
1566 # depot_side is without the trailing /... if it had one
1567 self
.depot_side
= View
.Path(depot_side
, is_depot
=True)
1568 self
.client_side
= View
.Path(client_side
, is_depot
=False)
1569 self
.overlay
= overlay
# started with "+"
1570 self
.exclude
= exclude
# started with "-"
1571 assert not (self
.overlay
and self
.exclude
)
1572 self
.depot_side
.ensure_compatible(self
.client_side
)
1580 return "View.Mapping: %s%s -> %s" % \
1581 (c
, self
.depot_side
.path
, self
.client_side
.path
)
1583 def map_depot_to_client(self
, depot_path
):
1584 """Calculate the client path if using this mapping on the
1585 given depot path; does not consider the effect of other
1586 mappings in a view. Even excluded mappings are returned."""
1587 matches
, wildcards
= self
.depot_side
.match_wildcards(depot_path
)
1590 client_path
= self
.client_side
.fill_in_wildcards(wildcards
)
1599 def append(self
, view_line
):
1600 """Parse a view line, splitting it into depot and client
1601 sides. Append to self.mappings, preserving order."""
1603 # Split the view line into exactly two words. P4 enforces
1604 # structure on these lines that simplifies this quite a bit.
1606 # Either or both words may be double-quoted.
1607 # Single quotes do not matter.
1608 # Double-quote marks cannot occur inside the words.
1609 # A + or - prefix is also inside the quotes.
1610 # There are no quotes unless they contain a space.
1611 # The line is already white-space stripped.
1612 # The two words are separated by a single space.
1614 if view_line
[0] == '"':
1615 # First word is double quoted. Find its end.
1616 close_quote_index
= view_line
.find('"', 1)
1617 if close_quote_index
<= 0:
1618 die("No first-word closing quote found: %s" % view_line
)
1619 depot_side
= view_line
[1:close_quote_index
]
1620 # skip closing quote and space
1621 rhs_index
= close_quote_index
+ 1 + 1
1623 space_index
= view_line
.find(" ")
1624 if space_index
<= 0:
1625 die("No word-splitting space found: %s" % view_line
)
1626 depot_side
= view_line
[0:space_index
]
1627 rhs_index
= space_index
+ 1
1629 if view_line
[rhs_index
] == '"':
1630 # Second word is double quoted. Make sure there is a
1631 # double quote at the end too.
1632 if not view_line
.endswith('"'):
1633 die("View line with rhs quote should end with one: %s" %
1636 client_side
= view_line
[rhs_index
+1:-1]
1638 client_side
= view_line
[rhs_index
:]
1640 # prefix + means overlay on previous mapping
1642 if depot_side
.startswith("+"):
1644 depot_side
= depot_side
[1:]
1646 # prefix - means exclude this path
1648 if depot_side
.startswith("-"):
1650 depot_side
= depot_side
[1:]
1652 m
= View
.Mapping(depot_side
, client_side
, overlay
, exclude
)
1653 self
.mappings
.append(m
)
1655 def map_in_client(self
, depot_path
):
1656 """Return the relative location in the client where this
1657 depot file should live. Returns "" if the file should
1658 not be mapped in the client."""
1663 # look at later entries first
1664 for m
in self
.mappings
[::-1]:
1666 # see where will this path end up in the client
1667 p
= m
.map_depot_to_client(depot_path
)
1670 # Depot path does not belong in client. Must remember
1671 # this, as previous items should not cause files to
1672 # exist in this path either. Remember that the list is
1673 # being walked from the end, which has higher precedence.
1674 # Overlap mappings do not exclude previous mappings.
1676 paths_filled
.append(m
.client_side
)
1679 # This mapping matched; no need to search any further.
1680 # But, the mapping could be rejected if the client path
1681 # has already been claimed by an earlier mapping (i.e.
1682 # one later in the list, which we are walking backwards).
1683 already_mapped_in_client
= False
1684 for f
in paths_filled
:
1685 # this is View.Path.match
1687 already_mapped_in_client
= True
1689 if not already_mapped_in_client
:
1690 # Include this file, unless it is from a line that
1691 # explicitly said to exclude it.
1695 # a match, even if rejected, always stops the search
1700 class P4Sync(Command
, P4UserMap
):
1701 delete_actions
= ( "delete", "move/delete", "purge" )
1704 Command
.__init
__(self
)
1705 P4UserMap
.__init
__(self
)
1707 optparse
.make_option("--branch", dest
="branch"),
1708 optparse
.make_option("--detect-branches", dest
="detectBranches", action
="store_true"),
1709 optparse
.make_option("--changesfile", dest
="changesFile"),
1710 optparse
.make_option("--silent", dest
="silent", action
="store_true"),
1711 optparse
.make_option("--detect-labels", dest
="detectLabels", action
="store_true"),
1712 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
1713 optparse
.make_option("--import-local", dest
="importIntoRemotes", action
="store_false",
1714 help="Import into refs/heads/ , not refs/remotes"),
1715 optparse
.make_option("--max-changes", dest
="maxChanges"),
1716 optparse
.make_option("--keep-path", dest
="keepRepoPath", action
='store_true',
1717 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1718 optparse
.make_option("--use-client-spec", dest
="useClientSpec", action
='store_true',
1719 help="Only sync files that are included in the Perforce Client Spec")
1721 self
.description
= """Imports from Perforce into a git repository.\n
1723 //depot/my/project/ -- to import the current head
1724 //depot/my/project/@all -- to import everything
1725 //depot/my/project/@1,6 -- to import only from revision 1 to 6
1727 (a ... is not needed in the path p4 specification, it's added implicitly)"""
1729 self
.usage
+= " //depot/path[@revRange]"
1731 self
.createdBranches
= set()
1732 self
.committedChanges
= set()
1734 self
.detectBranches
= False
1735 self
.detectLabels
= False
1736 self
.importLabels
= False
1737 self
.changesFile
= ""
1738 self
.syncWithOrigin
= True
1739 self
.importIntoRemotes
= True
1740 self
.maxChanges
= ""
1741 self
.isWindows
= (platform
.system() == "Windows")
1742 self
.keepRepoPath
= False
1743 self
.depotPaths
= None
1744 self
.p4BranchesInGit
= []
1745 self
.cloneExclude
= []
1746 self
.useClientSpec
= False
1747 self
.useClientSpec_from_options
= False
1748 self
.clientSpecDirs
= None
1749 self
.tempBranches
= []
1750 self
.tempBranchLocation
= "git-p4-tmp"
1752 if gitConfig("git-p4.syncFromOrigin") == "false":
1753 self
.syncWithOrigin
= False
1755 # Force a checkpoint in fast-import and wait for it to finish
1756 def checkpoint(self
):
1757 self
.gitStream
.write("checkpoint\n\n")
1758 self
.gitStream
.write("progress checkpoint\n\n")
1759 out
= self
.gitOutput
.readline()
1761 print "checkpoint finished: " + out
1763 def extractFilesFromCommit(self
, commit
):
1764 self
.cloneExclude
= [re
.sub(r
"\.\.\.$", "", path
)
1765 for path
in self
.cloneExclude
]
1768 while commit
.has_key("depotFile%s" % fnum
):
1769 path
= commit
["depotFile%s" % fnum
]
1771 if [p
for p
in self
.cloneExclude
1772 if p4PathStartsWith(path
, p
)]:
1775 found
= [p
for p
in self
.depotPaths
1776 if p4PathStartsWith(path
, p
)]
1783 file["rev"] = commit
["rev%s" % fnum
]
1784 file["action"] = commit
["action%s" % fnum
]
1785 file["type"] = commit
["type%s" % fnum
]
1790 def stripRepoPath(self
, path
, prefixes
):
1791 if self
.useClientSpec
:
1792 return self
.clientSpecDirs
.map_in_client(path
)
1794 if self
.keepRepoPath
:
1795 prefixes
= [re
.sub("^(//[^/]+/).*", r
'\1', prefixes
[0])]
1798 if p4PathStartsWith(path
, p
):
1799 path
= path
[len(p
):]
1803 def splitFilesIntoBranches(self
, commit
):
1806 while commit
.has_key("depotFile%s" % fnum
):
1807 path
= commit
["depotFile%s" % fnum
]
1808 found
= [p
for p
in self
.depotPaths
1809 if p4PathStartsWith(path
, p
)]
1816 file["rev"] = commit
["rev%s" % fnum
]
1817 file["action"] = commit
["action%s" % fnum
]
1818 file["type"] = commit
["type%s" % fnum
]
1821 relPath
= self
.stripRepoPath(path
, self
.depotPaths
)
1822 relPath
= wildcard_decode(relPath
)
1824 for branch
in self
.knownBranches
.keys():
1826 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1827 if relPath
.startswith(branch
+ "/"):
1828 if branch
not in branches
:
1829 branches
[branch
] = []
1830 branches
[branch
].append(file)
1835 # output one file from the P4 stream
1836 # - helper for streamP4Files
1838 def streamOneP4File(self
, file, contents
):
1839 relPath
= self
.stripRepoPath(file['depotFile'], self
.branchPrefixes
)
1840 relPath
= wildcard_decode(relPath
)
1842 sys
.stderr
.write("%s\n" % relPath
)
1844 (type_base
, type_mods
) = split_p4_type(file["type"])
1847 if "x" in type_mods
:
1849 if type_base
== "symlink":
1851 # p4 print on a symlink contains "target\n"; remove the newline
1852 data
= ''.join(contents
)
1853 contents
= [data
[:-1]]
1855 if type_base
== "utf16":
1856 # p4 delivers different text in the python output to -G
1857 # than it does when using "print -o", or normal p4 client
1858 # operations. utf16 is converted to ascii or utf8, perhaps.
1859 # But ascii text saved as -t utf16 is completely mangled.
1860 # Invoke print -o to get the real contents.
1861 text
= p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1864 if type_base
== "apple":
1865 # Apple filetype files will be streamed as a concatenation of
1866 # its appledouble header and the contents. This is useless
1867 # on both macs and non-macs. If using "print -q -o xx", it
1868 # will create "xx" with the data, and "%xx" with the header.
1869 # This is also not very useful.
1871 # Ideally, someday, this script can learn how to generate
1872 # appledouble files directly and import those to git, but
1873 # non-mac machines can never find a use for apple filetype.
1874 print "\nIgnoring apple filetype file %s" % file['depotFile']
1877 # Perhaps windows wants unicode, utf16 newlines translated too;
1878 # but this is not doing it.
1879 if self
.isWindows
and type_base
== "text":
1881 for data
in contents
:
1882 data
= data
.replace("\r\n", "\n")
1883 mangled
.append(data
)
1886 # Note that we do not try to de-mangle keywords on utf16 files,
1887 # even though in theory somebody may want that.
1888 pattern
= p4_keywords_regexp_for_type(type_base
, type_mods
)
1890 regexp
= re
.compile(pattern
, re
.VERBOSE
)
1891 text
= ''.join(contents
)
1892 text
= regexp
.sub(r
'$\1$', text
)
1895 self
.gitStream
.write("M %s inline %s\n" % (git_mode
, relPath
))
1900 length
= length
+ len(d
)
1902 self
.gitStream
.write("data %d\n" % length
)
1904 self
.gitStream
.write(d
)
1905 self
.gitStream
.write("\n")
1907 def streamOneP4Deletion(self
, file):
1908 relPath
= self
.stripRepoPath(file['path'], self
.branchPrefixes
)
1909 relPath
= wildcard_decode(relPath
)
1911 sys
.stderr
.write("delete %s\n" % relPath
)
1912 self
.gitStream
.write("D %s\n" % relPath
)
1914 # handle another chunk of streaming data
1915 def streamP4FilesCb(self
, marshalled
):
1917 if marshalled
.has_key('depotFile') and self
.stream_have_file_info
:
1918 # start of a new file - output the old one first
1919 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1920 self
.stream_file
= {}
1921 self
.stream_contents
= []
1922 self
.stream_have_file_info
= False
1924 # pick up the new file information... for the
1925 # 'data' field we need to append to our array
1926 for k
in marshalled
.keys():
1928 self
.stream_contents
.append(marshalled
['data'])
1930 self
.stream_file
[k
] = marshalled
[k
]
1932 self
.stream_have_file_info
= True
1934 # Stream directly from "p4 files" into "git fast-import"
1935 def streamP4Files(self
, files
):
1941 # if using a client spec, only add the files that have
1942 # a path in the client
1943 if self
.clientSpecDirs
:
1944 if self
.clientSpecDirs
.map_in_client(f
['path']) == "":
1947 filesForCommit
.append(f
)
1948 if f
['action'] in self
.delete_actions
:
1949 filesToDelete
.append(f
)
1951 filesToRead
.append(f
)
1954 for f
in filesToDelete
:
1955 self
.streamOneP4Deletion(f
)
1957 if len(filesToRead
) > 0:
1958 self
.stream_file
= {}
1959 self
.stream_contents
= []
1960 self
.stream_have_file_info
= False
1962 # curry self argument
1963 def streamP4FilesCbSelf(entry
):
1964 self
.streamP4FilesCb(entry
)
1966 fileArgs
= ['%s#%s' % (f
['path'], f
['rev']) for f
in filesToRead
]
1968 p4CmdList(["-x", "-", "print"],
1970 cb
=streamP4FilesCbSelf
)
1973 if self
.stream_file
.has_key('depotFile'):
1974 self
.streamOneP4File(self
.stream_file
, self
.stream_contents
)
1976 def make_email(self
, userid
):
1977 if userid
in self
.users
:
1978 return self
.users
[userid
]
1980 return "%s <a@b>" % userid
1983 def streamTag(self
, gitStream
, labelName
, labelDetails
, commit
, epoch
):
1985 print "writing tag %s for commit %s" % (labelName
, commit
)
1986 gitStream
.write("tag %s\n" % labelName
)
1987 gitStream
.write("from %s\n" % commit
)
1989 if labelDetails
.has_key('Owner'):
1990 owner
= labelDetails
["Owner"]
1994 # Try to use the owner of the p4 label, or failing that,
1995 # the current p4 user id.
1997 email
= self
.make_email(owner
)
1999 email
= self
.make_email(self
.p4UserId())
2000 tagger
= "%s %s %s" % (email
, epoch
, self
.tz
)
2002 gitStream
.write("tagger %s\n" % tagger
)
2004 print "labelDetails=",labelDetails
2005 if labelDetails
.has_key('Description'):
2006 description
= labelDetails
['Description']
2008 description
= 'Label from git p4'
2010 gitStream
.write("data %d\n" % len(description
))
2011 gitStream
.write(description
)
2012 gitStream
.write("\n")
2014 def commit(self
, details
, files
, branch
, branchPrefixes
, parent
= ""):
2015 epoch
= details
["time"]
2016 author
= details
["user"]
2017 self
.branchPrefixes
= branchPrefixes
2020 print "commit into %s" % branch
2022 # start with reading files; if that fails, we should not
2026 if [p
for p
in branchPrefixes
if p4PathStartsWith(f
['path'], p
)]:
2027 new_files
.append (f
)
2029 sys
.stderr
.write("Ignoring file outside of prefix: %s\n" % f
['path'])
2031 self
.gitStream
.write("commit %s\n" % branch
)
2032 # gitStream.write("mark :%s\n" % details["change"])
2033 self
.committedChanges
.add(int(details
["change"]))
2035 if author
not in self
.users
:
2036 self
.getUserMapFromPerforceServer()
2037 committer
= "%s %s %s" % (self
.make_email(author
), epoch
, self
.tz
)
2039 self
.gitStream
.write("committer %s\n" % committer
)
2041 self
.gitStream
.write("data <<EOT\n")
2042 self
.gitStream
.write(details
["desc"])
2043 self
.gitStream
.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2044 % (','.join (branchPrefixes
), details
["change"]))
2045 if len(details
['options']) > 0:
2046 self
.gitStream
.write(": options = %s" % details
['options'])
2047 self
.gitStream
.write("]\nEOT\n\n")
2051 print "parent %s" % parent
2052 self
.gitStream
.write("from %s\n" % parent
)
2054 self
.streamP4Files(new_files
)
2055 self
.gitStream
.write("\n")
2057 change
= int(details
["change"])
2059 if self
.labels
.has_key(change
):
2060 label
= self
.labels
[change
]
2061 labelDetails
= label
[0]
2062 labelRevisions
= label
[1]
2064 print "Change %s is labelled %s" % (change
, labelDetails
)
2066 files
= p4CmdList(["files"] + ["%s...@%s" % (p
, change
)
2067 for p
in branchPrefixes
])
2069 if len(files
) == len(labelRevisions
):
2073 if info
["action"] in self
.delete_actions
:
2075 cleanedFiles
[info
["depotFile"]] = info
["rev"]
2077 if cleanedFiles
== labelRevisions
:
2078 self
.streamTag(self
.gitStream
, 'tag_%s' % labelDetails
['label'], labelDetails
, branch
, epoch
)
2082 print ("Tag %s does not match with change %s: files do not match."
2083 % (labelDetails
["label"], change
))
2087 print ("Tag %s does not match with change %s: file count is different."
2088 % (labelDetails
["label"], change
))
2090 # Build a dictionary of changelists and labels, for "detect-labels" option.
2091 def getLabels(self
):
2094 l
= p4CmdList(["labels"] + ["%s..." % p
for p
in self
.depotPaths
])
2095 if len(l
) > 0 and not self
.silent
:
2096 print "Finding files belonging to labels in %s" % `self
.depotPaths`
2099 label
= output
["label"]
2103 print "Querying files for label %s" % label
2104 for file in p4CmdList(["files"] +
2105 ["%s...@%s" % (p
, label
)
2106 for p
in self
.depotPaths
]):
2107 revisions
[file["depotFile"]] = file["rev"]
2108 change
= int(file["change"])
2109 if change
> newestChange
:
2110 newestChange
= change
2112 self
.labels
[newestChange
] = [output
, revisions
]
2115 print "Label changes: %s" % self
.labels
.keys()
2117 # Import p4 labels as git tags. A direct mapping does not
2118 # exist, so assume that if all the files are at the same revision
2119 # then we can use that, or it's something more complicated we should
2121 def importP4Labels(self
, stream
, p4Labels
):
2123 print "import p4 labels: " + ' '.join(p4Labels
)
2125 ignoredP4Labels
= gitConfigList("git-p4.ignoredP4Labels")
2126 validLabelRegexp
= gitConfig("git-p4.labelImportRegexp")
2127 if len(validLabelRegexp
) == 0:
2128 validLabelRegexp
= defaultLabelRegexp
2129 m
= re
.compile(validLabelRegexp
)
2131 for name
in p4Labels
:
2134 if not m
.match(name
):
2136 print "label %s does not match regexp %s" % (name
,validLabelRegexp
)
2139 if name
in ignoredP4Labels
:
2142 labelDetails
= p4CmdList(['label', "-o", name
])[0]
2144 # get the most recent changelist for each file in this label
2145 change
= p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p
, name
)
2146 for p
in self
.depotPaths
])
2148 if change
.has_key('change'):
2149 # find the corresponding git commit; take the oldest commit
2150 changelist
= int(change
['change'])
2151 gitCommit
= read_pipe(["git", "rev-list", "--max-count=1",
2152 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist
])
2153 if len(gitCommit
) == 0:
2154 print "could not find git commit for changelist %d" % changelist
2156 gitCommit
= gitCommit
.strip()
2158 # Convert from p4 time format
2160 tmwhen
= time
.strptime(labelDetails
['Update'], "%Y/%m/%d %H:%M:%S")
2162 print "Could not convert label time %s" % labelDetail
['Update']
2165 when
= int(time
.mktime(tmwhen
))
2166 self
.streamTag(stream
, name
, labelDetails
, gitCommit
, when
)
2168 print "p4 label %s mapped to git commit %s" % (name
, gitCommit
)
2171 print "Label %s has no changelists - possibly deleted?" % name
2174 # We can't import this label; don't try again as it will get very
2175 # expensive repeatedly fetching all the files for labels that will
2176 # never be imported. If the label is moved in the future, the
2177 # ignore will need to be removed manually.
2178 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name
])
2180 def guessProjectName(self
):
2181 for p
in self
.depotPaths
:
2184 p
= p
[p
.strip().rfind("/") + 1:]
2185 if not p
.endswith("/"):
2189 def getBranchMapping(self
):
2190 lostAndFoundBranches
= set()
2192 user
= gitConfig("git-p4.branchUser")
2194 command
= "branches -u %s" % user
2196 command
= "branches"
2198 for info
in p4CmdList(command
):
2199 details
= p4Cmd(["branch", "-o", info
["branch"]])
2201 while details
.has_key("View%s" % viewIdx
):
2202 paths
= details
["View%s" % viewIdx
].split(" ")
2203 viewIdx
= viewIdx
+ 1
2204 # require standard //depot/foo/... //depot/bar/... mapping
2205 if len(paths
) != 2 or not paths
[0].endswith("/...") or not paths
[1].endswith("/..."):
2208 destination
= paths
[1]
2210 if p4PathStartsWith(source
, self
.depotPaths
[0]) and p4PathStartsWith(destination
, self
.depotPaths
[0]):
2211 source
= source
[len(self
.depotPaths
[0]):-4]
2212 destination
= destination
[len(self
.depotPaths
[0]):-4]
2214 if destination
in self
.knownBranches
:
2216 print "p4 branch %s defines a mapping from %s to %s" % (info
["branch"], source
, destination
)
2217 print "but there exists another mapping from %s to %s already!" % (self
.knownBranches
[destination
], destination
)
2220 self
.knownBranches
[destination
] = source
2222 lostAndFoundBranches
.discard(destination
)
2224 if source
not in self
.knownBranches
:
2225 lostAndFoundBranches
.add(source
)
2227 # Perforce does not strictly require branches to be defined, so we also
2228 # check git config for a branch list.
2230 # Example of branch definition in git config file:
2232 # branchList=main:branchA
2233 # branchList=main:branchB
2234 # branchList=branchA:branchC
2235 configBranches
= gitConfigList("git-p4.branchList")
2236 for branch
in configBranches
:
2238 (source
, destination
) = branch
.split(":")
2239 self
.knownBranches
[destination
] = source
2241 lostAndFoundBranches
.discard(destination
)
2243 if source
not in self
.knownBranches
:
2244 lostAndFoundBranches
.add(source
)
2247 for branch
in lostAndFoundBranches
:
2248 self
.knownBranches
[branch
] = branch
2250 def getBranchMappingFromGitBranches(self
):
2251 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2252 for branch
in branches
.keys():
2253 if branch
== "master":
2256 branch
= branch
[len(self
.projectName
):]
2257 self
.knownBranches
[branch
] = branch
2259 def listExistingP4GitBranches(self
):
2260 # branches holds mapping from name to commit
2261 branches
= p4BranchesInGit(self
.importIntoRemotes
)
2262 self
.p4BranchesInGit
= branches
.keys()
2263 for branch
in branches
.keys():
2264 self
.initialParents
[self
.refPrefix
+ branch
] = branches
[branch
]
2266 def updateOptionDict(self
, d
):
2268 if self
.keepRepoPath
:
2269 option_keys
['keepRepoPath'] = 1
2271 d
["options"] = ' '.join(sorted(option_keys
.keys()))
2273 def readOptions(self
, d
):
2274 self
.keepRepoPath
= (d
.has_key('options')
2275 and ('keepRepoPath' in d
['options']))
2277 def gitRefForBranch(self
, branch
):
2278 if branch
== "main":
2279 return self
.refPrefix
+ "master"
2281 if len(branch
) <= 0:
2284 return self
.refPrefix
+ self
.projectName
+ branch
2286 def gitCommitByP4Change(self
, ref
, change
):
2288 print "looking in ref " + ref
+ " for change %s using bisect..." % change
2291 latestCommit
= parseRevision(ref
)
2295 print "trying: earliest %s latest %s" % (earliestCommit
, latestCommit
)
2296 next
= read_pipe("git rev-list --bisect %s %s" % (latestCommit
, earliestCommit
)).strip()
2301 log
= extractLogMessageFromGitCommit(next
)
2302 settings
= extractSettingsGitLog(log
)
2303 currentChange
= int(settings
['change'])
2305 print "current change %s" % currentChange
2307 if currentChange
== change
:
2309 print "found %s" % next
2312 if currentChange
< change
:
2313 earliestCommit
= "^%s" % next
2315 latestCommit
= "%s" % next
2319 def importNewBranch(self
, branch
, maxChange
):
2320 # make fast-import flush all changes to disk and update the refs using the checkpoint
2321 # command so that we can try to find the branch parent in the git history
2322 self
.gitStream
.write("checkpoint\n\n");
2323 self
.gitStream
.flush();
2324 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2325 range = "@1,%s" % maxChange
2326 #print "prefix" + branchPrefix
2327 changes
= p4ChangesForPaths([branchPrefix
], range)
2328 if len(changes
) <= 0:
2330 firstChange
= changes
[0]
2331 #print "first change in branch: %s" % firstChange
2332 sourceBranch
= self
.knownBranches
[branch
]
2333 sourceDepotPath
= self
.depotPaths
[0] + sourceBranch
2334 sourceRef
= self
.gitRefForBranch(sourceBranch
)
2335 #print "source " + sourceBranch
2337 branchParentChange
= int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath
, firstChange
)])["change"])
2338 #print "branch parent: %s" % branchParentChange
2339 gitParent
= self
.gitCommitByP4Change(sourceRef
, branchParentChange
)
2340 if len(gitParent
) > 0:
2341 self
.initialParents
[self
.gitRefForBranch(branch
)] = gitParent
2342 #print "parent git commit: %s" % gitParent
2344 self
.importChanges(changes
)
2347 def searchParent(self
, parent
, branch
, target
):
2349 for blob
in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent
]):
2351 if len(read_pipe(["git", "diff-tree", blob
, target
])) == 0:
2354 print "Found parent of %s in commit %s" % (branch
, blob
)
2361 def importChanges(self
, changes
):
2363 for change
in changes
:
2364 description
= p4Cmd(["describe", str(change
)])
2365 self
.updateOptionDict(description
)
2368 sys
.stdout
.write("\rImporting revision %s (%s%%)" % (change
, cnt
* 100 / len(changes
)))
2373 if self
.detectBranches
:
2374 branches
= self
.splitFilesIntoBranches(description
)
2375 for branch
in branches
.keys():
2377 branchPrefix
= self
.depotPaths
[0] + branch
+ "/"
2381 filesForCommit
= branches
[branch
]
2384 print "branch is %s" % branch
2386 self
.updatedBranches
.add(branch
)
2388 if branch
not in self
.createdBranches
:
2389 self
.createdBranches
.add(branch
)
2390 parent
= self
.knownBranches
[branch
]
2391 if parent
== branch
:
2394 fullBranch
= self
.projectName
+ branch
2395 if fullBranch
not in self
.p4BranchesInGit
:
2397 print("\n Importing new branch %s" % fullBranch
);
2398 if self
.importNewBranch(branch
, change
- 1):
2400 self
.p4BranchesInGit
.append(fullBranch
)
2402 print("\n Resuming with change %s" % change
);
2405 print "parent determined through known branches: %s" % parent
2407 branch
= self
.gitRefForBranch(branch
)
2408 parent
= self
.gitRefForBranch(parent
)
2411 print "looking for initial parent for %s; current parent is %s" % (branch
, parent
)
2413 if len(parent
) == 0 and branch
in self
.initialParents
:
2414 parent
= self
.initialParents
[branch
]
2415 del self
.initialParents
[branch
]
2419 tempBranch
= os
.path
.join(self
.tempBranchLocation
, "%d" % (change
))
2421 print "Creating temporary branch: " + tempBranch
2422 self
.commit(description
, filesForCommit
, tempBranch
, [branchPrefix
])
2423 self
.tempBranches
.append(tempBranch
)
2425 blob
= self
.searchParent(parent
, branch
, tempBranch
)
2427 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], blob
)
2430 print "Parent of %s not found. Committing into head of %s" % (branch
, parent
)
2431 self
.commit(description
, filesForCommit
, branch
, [branchPrefix
], parent
)
2433 files
= self
.extractFilesFromCommit(description
)
2434 self
.commit(description
, files
, self
.branch
, self
.depotPaths
,
2436 self
.initialParent
= ""
2438 print self
.gitError
.read()
2441 def importHeadRevision(self
, revision
):
2442 print "Doing initial import of %s from revision %s into %s" % (' '.join(self
.depotPaths
), revision
, self
.branch
)
2445 details
["user"] = "git perforce import user"
2446 details
["desc"] = ("Initial import of %s from the state at revision %s\n"
2447 % (' '.join(self
.depotPaths
), revision
))
2448 details
["change"] = revision
2452 fileArgs
= ["%s...%s" % (p
,revision
) for p
in self
.depotPaths
]
2454 for info
in p4CmdList(["files"] + fileArgs
):
2456 if 'code' in info
and info
['code'] == 'error':
2457 sys
.stderr
.write("p4 returned an error: %s\n"
2459 if info
['data'].find("must refer to client") >= 0:
2460 sys
.stderr
.write("This particular p4 error is misleading.\n")
2461 sys
.stderr
.write("Perhaps the depot path was misspelled.\n");
2462 sys
.stderr
.write("Depot path: %s\n" % " ".join(self
.depotPaths
))
2464 if 'p4ExitCode' in info
:
2465 sys
.stderr
.write("p4 exitcode: %s\n" % info
['p4ExitCode'])
2469 change
= int(info
["change"])
2470 if change
> newestRevision
:
2471 newestRevision
= change
2473 if info
["action"] in self
.delete_actions
:
2474 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2475 #fileCnt = fileCnt + 1
2478 for prop
in ["depotFile", "rev", "action", "type" ]:
2479 details
["%s%s" % (prop
, fileCnt
)] = info
[prop
]
2481 fileCnt
= fileCnt
+ 1
2483 details
["change"] = newestRevision
2485 # Use time from top-most change so that all git p4 clones of
2486 # the same p4 repo have the same commit SHA1s.
2487 res
= p4CmdList("describe -s %d" % newestRevision
)
2490 if r
.has_key('time'):
2491 newestTime
= int(r
['time'])
2492 if newestTime
is None:
2493 die("\"describe -s\" on newest change %d did not give a time")
2494 details
["time"] = newestTime
2496 self
.updateOptionDict(details
)
2498 self
.commit(details
, self
.extractFilesFromCommit(details
), self
.branch
, self
.depotPaths
)
2500 print "IO error with git fast-import. Is your git version recent enough?"
2501 print self
.gitError
.read()
2504 def run(self
, args
):
2505 self
.depotPaths
= []
2506 self
.changeRange
= ""
2507 self
.initialParent
= ""
2508 self
.previousDepotPaths
= []
2510 # map from branch depot path to parent branch
2511 self
.knownBranches
= {}
2512 self
.initialParents
= {}
2513 self
.hasOrigin
= originP4BranchesExist()
2514 if not self
.syncWithOrigin
:
2515 self
.hasOrigin
= False
2517 if self
.importIntoRemotes
:
2518 self
.refPrefix
= "refs/remotes/p4/"
2520 self
.refPrefix
= "refs/heads/p4/"
2522 if self
.syncWithOrigin
and self
.hasOrigin
:
2524 print "Syncing with origin first by calling git fetch origin"
2525 system("git fetch origin")
2527 if len(self
.branch
) == 0:
2528 self
.branch
= self
.refPrefix
+ "master"
2529 if gitBranchExists("refs/heads/p4") and self
.importIntoRemotes
:
2530 system("git update-ref %s refs/heads/p4" % self
.branch
)
2531 system("git branch -D p4");
2532 # create it /after/ importing, when master exists
2533 if not gitBranchExists(self
.refPrefix
+ "HEAD") and self
.importIntoRemotes
and gitBranchExists(self
.branch
):
2534 system("git symbolic-ref %sHEAD %s" % (self
.refPrefix
, self
.branch
))
2536 # accept either the command-line option, or the configuration variable
2537 if self
.useClientSpec
:
2538 # will use this after clone to set the variable
2539 self
.useClientSpec_from_options
= True
2541 if gitConfig("git-p4.useclientspec", "--bool") == "true":
2542 self
.useClientSpec
= True
2543 if self
.useClientSpec
:
2544 self
.clientSpecDirs
= getClientSpec()
2546 # TODO: should always look at previous commits,
2547 # merge with previous imports, if possible.
2550 createOrUpdateBranchesFromOrigin(self
.refPrefix
, self
.silent
)
2551 self
.listExistingP4GitBranches()
2553 if len(self
.p4BranchesInGit
) > 1:
2555 print "Importing from/into multiple branches"
2556 self
.detectBranches
= True
2559 print "branches: %s" % self
.p4BranchesInGit
2562 for branch
in self
.p4BranchesInGit
:
2563 logMsg
= extractLogMessageFromGitCommit(self
.refPrefix
+ branch
)
2565 settings
= extractSettingsGitLog(logMsg
)
2567 self
.readOptions(settings
)
2568 if (settings
.has_key('depot-paths')
2569 and settings
.has_key ('change')):
2570 change
= int(settings
['change']) + 1
2571 p4Change
= max(p4Change
, change
)
2573 depotPaths
= sorted(settings
['depot-paths'])
2574 if self
.previousDepotPaths
== []:
2575 self
.previousDepotPaths
= depotPaths
2578 for (prev
, cur
) in zip(self
.previousDepotPaths
, depotPaths
):
2579 prev_list
= prev
.split("/")
2580 cur_list
= cur
.split("/")
2581 for i
in range(0, min(len(cur_list
), len(prev_list
))):
2582 if cur_list
[i
] <> prev_list
[i
]:
2586 paths
.append ("/".join(cur_list
[:i
+ 1]))
2588 self
.previousDepotPaths
= paths
2591 self
.depotPaths
= sorted(self
.previousDepotPaths
)
2592 self
.changeRange
= "@%s,#head" % p4Change
2593 if not self
.detectBranches
:
2594 self
.initialParent
= parseRevision(self
.branch
)
2595 if not self
.silent
and not self
.detectBranches
:
2596 print "Performing incremental import into %s git branch" % self
.branch
2598 if not self
.branch
.startswith("refs/"):
2599 self
.branch
= "refs/heads/" + self
.branch
2601 if len(args
) == 0 and self
.depotPaths
:
2603 print "Depot paths: %s" % ' '.join(self
.depotPaths
)
2605 if self
.depotPaths
and self
.depotPaths
!= args
:
2606 print ("previous import used depot path %s and now %s was specified. "
2607 "This doesn't work!" % (' '.join (self
.depotPaths
),
2611 self
.depotPaths
= sorted(args
)
2616 # Make sure no revision specifiers are used when --changesfile
2618 bad_changesfile
= False
2619 if len(self
.changesFile
) > 0:
2620 for p
in self
.depotPaths
:
2621 if p
.find("@") >= 0 or p
.find("#") >= 0:
2622 bad_changesfile
= True
2625 die("Option --changesfile is incompatible with revision specifiers")
2628 for p
in self
.depotPaths
:
2629 if p
.find("@") != -1:
2630 atIdx
= p
.index("@")
2631 self
.changeRange
= p
[atIdx
:]
2632 if self
.changeRange
== "@all":
2633 self
.changeRange
= ""
2634 elif ',' not in self
.changeRange
:
2635 revision
= self
.changeRange
2636 self
.changeRange
= ""
2638 elif p
.find("#") != -1:
2639 hashIdx
= p
.index("#")
2640 revision
= p
[hashIdx
:]
2642 elif self
.previousDepotPaths
== []:
2643 # pay attention to changesfile, if given, else import
2644 # the entire p4 tree at the head revision
2645 if len(self
.changesFile
) == 0:
2648 p
= re
.sub ("\.\.\.$", "", p
)
2649 if not p
.endswith("/"):
2654 self
.depotPaths
= newPaths
2656 self
.loadUserMapFromCache()
2658 if self
.detectLabels
:
2661 if self
.detectBranches
:
2662 ## FIXME - what's a P4 projectName ?
2663 self
.projectName
= self
.guessProjectName()
2666 self
.getBranchMappingFromGitBranches()
2668 self
.getBranchMapping()
2670 print "p4-git branches: %s" % self
.p4BranchesInGit
2671 print "initial parents: %s" % self
.initialParents
2672 for b
in self
.p4BranchesInGit
:
2676 b
= b
[len(self
.projectName
):]
2677 self
.createdBranches
.add(b
)
2679 self
.tz
= "%+03d%02d" % (- time
.timezone
/ 3600, ((- time
.timezone
% 3600) / 60))
2681 importProcess
= subprocess
.Popen(["git", "fast-import"],
2682 stdin
=subprocess
.PIPE
, stdout
=subprocess
.PIPE
,
2683 stderr
=subprocess
.PIPE
);
2684 self
.gitOutput
= importProcess
.stdout
2685 self
.gitStream
= importProcess
.stdin
2686 self
.gitError
= importProcess
.stderr
2689 self
.importHeadRevision(revision
)
2693 if len(self
.changesFile
) > 0:
2694 output
= open(self
.changesFile
).readlines()
2697 changeSet
.add(int(line
))
2699 for change
in changeSet
:
2700 changes
.append(change
)
2704 # catch "git p4 sync" with no new branches, in a repo that
2705 # does not have any existing p4 branches
2706 if len(args
) == 0 and not self
.p4BranchesInGit
:
2707 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.");
2709 print "Getting p4 changes for %s...%s" % (', '.join(self
.depotPaths
),
2711 changes
= p4ChangesForPaths(self
.depotPaths
, self
.changeRange
)
2713 if len(self
.maxChanges
) > 0:
2714 changes
= changes
[:min(int(self
.maxChanges
), len(changes
))]
2716 if len(changes
) == 0:
2718 print "No changes to import!"
2720 if not self
.silent
and not self
.detectBranches
:
2721 print "Import destination: %s" % self
.branch
2723 self
.updatedBranches
= set()
2725 self
.importChanges(changes
)
2729 if len(self
.updatedBranches
) > 0:
2730 sys
.stdout
.write("Updated branches: ")
2731 for b
in self
.updatedBranches
:
2732 sys
.stdout
.write("%s " % b
)
2733 sys
.stdout
.write("\n")
2735 if gitConfig("git-p4.importLabels", "--bool") == "true":
2736 self
.importLabels
= True
2738 if self
.importLabels
:
2739 p4Labels
= getP4Labels(self
.depotPaths
)
2740 gitTags
= getGitTags()
2742 missingP4Labels
= p4Labels
- gitTags
2743 self
.importP4Labels(self
.gitStream
, missingP4Labels
)
2745 self
.gitStream
.close()
2746 if importProcess
.wait() != 0:
2747 die("fast-import failed: %s" % self
.gitError
.read())
2748 self
.gitOutput
.close()
2749 self
.gitError
.close()
2751 # Cleanup temporary branches created during import
2752 if self
.tempBranches
!= []:
2753 for branch
in self
.tempBranches
:
2754 read_pipe("git update-ref -d %s" % branch
)
2755 os
.rmdir(os
.path
.join(os
.environ
.get("GIT_DIR", ".git"), self
.tempBranchLocation
))
2759 class P4Rebase(Command
):
2761 Command
.__init
__(self
)
2763 optparse
.make_option("--import-labels", dest
="importLabels", action
="store_true"),
2765 self
.importLabels
= False
2766 self
.description
= ("Fetches the latest revision from perforce and "
2767 + "rebases the current work (branch) against it")
2769 def run(self
, args
):
2771 sync
.importLabels
= self
.importLabels
2774 return self
.rebase()
2777 if os
.system("git update-index --refresh") != 0:
2778 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2779 if len(read_pipe("git diff-index HEAD --")) > 0:
2780 die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2782 [upstream
, settings
] = findUpstreamBranchPoint()
2783 if len(upstream
) == 0:
2784 die("Cannot find upstream branchpoint for rebase")
2786 # the branchpoint may be p4/foo~3, so strip off the parent
2787 upstream
= re
.sub("~[0-9]+$", "", upstream
)
2789 print "Rebasing the current branch onto %s" % upstream
2790 oldHead
= read_pipe("git rev-parse HEAD").strip()
2791 system("git rebase %s" % upstream
)
2792 system("git diff-tree --stat --summary -M %s HEAD" % oldHead
)
2795 class P4Clone(P4Sync
):
2797 P4Sync
.__init
__(self
)
2798 self
.description
= "Creates a new git repository and imports from Perforce into it"
2799 self
.usage
= "usage: %prog [options] //depot/path[@revRange]"
2801 optparse
.make_option("--destination", dest
="cloneDestination",
2802 action
='store', default
=None,
2803 help="where to leave result of the clone"),
2804 optparse
.make_option("-/", dest
="cloneExclude",
2805 action
="append", type="string",
2806 help="exclude depot path"),
2807 optparse
.make_option("--bare", dest
="cloneBare",
2808 action
="store_true", default
=False),
2810 self
.cloneDestination
= None
2811 self
.needsGit
= False
2812 self
.cloneBare
= False
2814 # This is required for the "append" cloneExclude action
2815 def ensure_value(self
, attr
, value
):
2816 if not hasattr(self
, attr
) or getattr(self
, attr
) is None:
2817 setattr(self
, attr
, value
)
2818 return getattr(self
, attr
)
2820 def defaultDestination(self
, args
):
2821 ## TODO: use common prefix of args?
2823 depotDir
= re
.sub("(@[^@]*)$", "", depotPath
)
2824 depotDir
= re
.sub("(#[^#]*)$", "", depotDir
)
2825 depotDir
= re
.sub(r
"\.\.\.$", "", depotDir
)
2826 depotDir
= re
.sub(r
"/$", "", depotDir
)
2827 return os
.path
.split(depotDir
)[1]
2829 def run(self
, args
):
2833 if self
.keepRepoPath
and not self
.cloneDestination
:
2834 sys
.stderr
.write("Must specify destination for --keep-path\n")
2839 if not self
.cloneDestination
and len(depotPaths
) > 1:
2840 self
.cloneDestination
= depotPaths
[-1]
2841 depotPaths
= depotPaths
[:-1]
2843 self
.cloneExclude
= ["/"+p
for p
in self
.cloneExclude
]
2844 for p
in depotPaths
:
2845 if not p
.startswith("//"):
2848 if not self
.cloneDestination
:
2849 self
.cloneDestination
= self
.defaultDestination(args
)
2851 print "Importing from %s into %s" % (', '.join(depotPaths
), self
.cloneDestination
)
2853 if not os
.path
.exists(self
.cloneDestination
):
2854 os
.makedirs(self
.cloneDestination
)
2855 chdir(self
.cloneDestination
)
2857 init_cmd
= [ "git", "init" ]
2859 init_cmd
.append("--bare")
2860 subprocess
.check_call(init_cmd
)
2862 if not P4Sync
.run(self
, depotPaths
):
2864 if self
.branch
!= "master":
2865 if self
.importIntoRemotes
:
2866 masterbranch
= "refs/remotes/p4/master"
2868 masterbranch
= "refs/heads/p4/master"
2869 if gitBranchExists(masterbranch
):
2870 system("git branch master %s" % masterbranch
)
2871 if not self
.cloneBare
:
2872 system("git checkout -f")
2874 print "Could not detect main branch. No checkout/master branch created."
2876 # auto-set this variable if invoked with --use-client-spec
2877 if self
.useClientSpec_from_options
:
2878 system("git config --bool git-p4.useclientspec true")
2882 class P4Branches(Command
):
2884 Command
.__init
__(self
)
2886 self
.description
= ("Shows the git branches that hold imports and their "
2887 + "corresponding perforce depot paths")
2888 self
.verbose
= False
2890 def run(self
, args
):
2891 if originP4BranchesExist():
2892 createOrUpdateBranchesFromOrigin()
2894 cmdline
= "git rev-parse --symbolic "
2895 cmdline
+= " --remotes"
2897 for line
in read_pipe_lines(cmdline
):
2900 if not line
.startswith('p4/') or line
== "p4/HEAD":
2904 log
= extractLogMessageFromGitCommit("refs/remotes/%s" % branch
)
2905 settings
= extractSettingsGitLog(log
)
2907 print "%s <= %s (%s)" % (branch
, ",".join(settings
["depot-paths"]), settings
["change"])
2910 class HelpFormatter(optparse
.IndentedHelpFormatter
):
2912 optparse
.IndentedHelpFormatter
.__init
__(self
)
2914 def format_description(self
, description
):
2916 return description
+ "\n"
2920 def printUsage(commands
):
2921 print "usage: %s <command> [options]" % sys
.argv
[0]
2923 print "valid commands: %s" % ", ".join(commands
)
2925 print "Try %s <command> --help for command specific help." % sys
.argv
[0]
2930 "submit" : P4Submit
,
2931 "commit" : P4Submit
,
2933 "rebase" : P4Rebase
,
2935 "rollback" : P4RollBack
,
2936 "branches" : P4Branches
2941 if len(sys
.argv
[1:]) == 0:
2942 printUsage(commands
.keys())
2946 cmdName
= sys
.argv
[1]
2948 klass
= commands
[cmdName
]
2951 print "unknown command %s" % cmdName
2953 printUsage(commands
.keys())
2956 options
= cmd
.options
2957 cmd
.gitdir
= os
.environ
.get("GIT_DIR", None)
2961 options
.append(optparse
.make_option("--verbose", dest
="verbose", action
="store_true"))
2963 options
.append(optparse
.make_option("--git-dir", dest
="gitdir"))
2965 parser
= optparse
.OptionParser(cmd
.usage
.replace("%prog", "%prog " + cmdName
),
2967 description
= cmd
.description
,
2968 formatter
= HelpFormatter())
2970 (cmd
, args
) = parser
.parse_args(sys
.argv
[2:], cmd
);
2972 verbose
= cmd
.verbose
2974 if cmd
.gitdir
== None:
2975 cmd
.gitdir
= os
.path
.abspath(".git")
2976 if not isValidGitDir(cmd
.gitdir
):
2977 cmd
.gitdir
= read_pipe("git rev-parse --git-dir").strip()
2978 if os
.path
.exists(cmd
.gitdir
):
2979 cdup
= read_pipe("git rev-parse --show-cdup").strip()
2983 if not isValidGitDir(cmd
.gitdir
):
2984 if isValidGitDir(cmd
.gitdir
+ "/.git"):
2985 cmd
.gitdir
+= "/.git"
2987 die("fatal: cannot locate git repository at %s" % cmd
.gitdir
)
2989 os
.environ
["GIT_DIR"] = cmd
.gitdir
2991 if not cmd
.run(args
):
2996 if __name__
== '__main__':