git-multimail: update to release 1.2.0
[git.git] / git-p4.py
blob215f556c13a72cf7b0cb49ae2a651d43a0b2623b
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
26 try:
27 from subprocess import CalledProcessError
28 except ImportError:
29 # from python2.7:subprocess.py
30 # Exception classes used by this module.
31 class CalledProcessError(Exception):
32 """This exception is raised when a process run by check_call() returns
33 a non-zero exit status. The exit status will be stored in the
34 returncode attribute."""
35 def __init__(self, returncode, cmd):
36 self.returncode = returncode
37 self.cmd = cmd
38 def __str__(self):
39 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
41 verbose = False
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
46 # Grab changes in blocks of this many revisions, unless otherwise requested
47 defaultBlockSize = 512
49 def p4_build_cmd(cmd):
50 """Build a suitable p4 command line.
52 This consolidates building and returning a p4 command line into one
53 location. It means that hooking into the environment, or other configuration
54 can be done more easily.
55 """
56 real_cmd = ["p4"]
58 user = gitConfig("git-p4.user")
59 if len(user) > 0:
60 real_cmd += ["-u",user]
62 password = gitConfig("git-p4.password")
63 if len(password) > 0:
64 real_cmd += ["-P", password]
66 port = gitConfig("git-p4.port")
67 if len(port) > 0:
68 real_cmd += ["-p", port]
70 host = gitConfig("git-p4.host")
71 if len(host) > 0:
72 real_cmd += ["-H", host]
74 client = gitConfig("git-p4.client")
75 if len(client) > 0:
76 real_cmd += ["-c", client]
79 if isinstance(cmd,basestring):
80 real_cmd = ' '.join(real_cmd) + ' ' + cmd
81 else:
82 real_cmd += cmd
83 return real_cmd
85 def chdir(path, is_client_path=False):
86 """Do chdir to the given path, and set the PWD environment
87 variable for use by P4. It does not look at getcwd() output.
88 Since we're not using the shell, it is necessary to set the
89 PWD environment variable explicitly.
91 Normally, expand the path to force it to be absolute. This
92 addresses the use of relative path names inside P4 settings,
93 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
94 as given; it looks for .p4config using PWD.
96 If is_client_path, the path was handed to us directly by p4,
97 and may be a symbolic link. Do not call os.getcwd() in this
98 case, because it will cause p4 to think that PWD is not inside
99 the client path.
102 os.chdir(path)
103 if not is_client_path:
104 path = os.getcwd()
105 os.environ['PWD'] = path
107 def die(msg):
108 if verbose:
109 raise Exception(msg)
110 else:
111 sys.stderr.write(msg + "\n")
112 sys.exit(1)
114 def write_pipe(c, stdin):
115 if verbose:
116 sys.stderr.write('Writing pipe: %s\n' % str(c))
118 expand = isinstance(c,basestring)
119 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
120 pipe = p.stdin
121 val = pipe.write(stdin)
122 pipe.close()
123 if p.wait():
124 die('Command failed: %s' % str(c))
126 return val
128 def p4_write_pipe(c, stdin):
129 real_cmd = p4_build_cmd(c)
130 return write_pipe(real_cmd, stdin)
132 def read_pipe(c, ignore_error=False):
133 if verbose:
134 sys.stderr.write('Reading pipe: %s\n' % str(c))
136 expand = isinstance(c,basestring)
137 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
138 pipe = p.stdout
139 val = pipe.read()
140 if p.wait() and not ignore_error:
141 die('Command failed: %s' % str(c))
143 return val
145 def p4_read_pipe(c, ignore_error=False):
146 real_cmd = p4_build_cmd(c)
147 return read_pipe(real_cmd, ignore_error)
149 def read_pipe_lines(c):
150 if verbose:
151 sys.stderr.write('Reading pipe: %s\n' % str(c))
153 expand = isinstance(c, basestring)
154 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
155 pipe = p.stdout
156 val = pipe.readlines()
157 if pipe.close() or p.wait():
158 die('Command failed: %s' % str(c))
160 return val
162 def p4_read_pipe_lines(c):
163 """Specifically invoke p4 on the command supplied. """
164 real_cmd = p4_build_cmd(c)
165 return read_pipe_lines(real_cmd)
167 def p4_has_command(cmd):
168 """Ask p4 for help on this command. If it returns an error, the
169 command does not exist in this version of p4."""
170 real_cmd = p4_build_cmd(["help", cmd])
171 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
172 stderr=subprocess.PIPE)
173 p.communicate()
174 return p.returncode == 0
176 def p4_has_move_command():
177 """See if the move command exists, that it supports -k, and that
178 it has not been administratively disabled. The arguments
179 must be correct, but the filenames do not have to exist. Use
180 ones with wildcards so even if they exist, it will fail."""
182 if not p4_has_command("move"):
183 return False
184 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
185 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
186 (out, err) = p.communicate()
187 # return code will be 1 in either case
188 if err.find("Invalid option") >= 0:
189 return False
190 if err.find("disabled") >= 0:
191 return False
192 # assume it failed because @... was invalid changelist
193 return True
195 def system(cmd):
196 expand = isinstance(cmd,basestring)
197 if verbose:
198 sys.stderr.write("executing %s\n" % str(cmd))
199 retcode = subprocess.call(cmd, shell=expand)
200 if retcode:
201 raise CalledProcessError(retcode, cmd)
203 def p4_system(cmd):
204 """Specifically invoke p4 as the system command. """
205 real_cmd = p4_build_cmd(cmd)
206 expand = isinstance(real_cmd, basestring)
207 retcode = subprocess.call(real_cmd, shell=expand)
208 if retcode:
209 raise CalledProcessError(retcode, real_cmd)
211 _p4_version_string = None
212 def p4_version_string():
213 """Read the version string, showing just the last line, which
214 hopefully is the interesting version bit.
216 $ p4 -V
217 Perforce - The Fast Software Configuration Management System.
218 Copyright 1995-2011 Perforce Software. All rights reserved.
219 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
221 global _p4_version_string
222 if not _p4_version_string:
223 a = p4_read_pipe_lines(["-V"])
224 _p4_version_string = a[-1].rstrip()
225 return _p4_version_string
227 def p4_integrate(src, dest):
228 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
230 def p4_sync(f, *options):
231 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
233 def p4_add(f):
234 # forcibly add file names with wildcards
235 if wildcard_present(f):
236 p4_system(["add", "-f", f])
237 else:
238 p4_system(["add", f])
240 def p4_delete(f):
241 p4_system(["delete", wildcard_encode(f)])
243 def p4_edit(f):
244 p4_system(["edit", wildcard_encode(f)])
246 def p4_revert(f):
247 p4_system(["revert", wildcard_encode(f)])
249 def p4_reopen(type, f):
250 p4_system(["reopen", "-t", type, wildcard_encode(f)])
252 def p4_move(src, dest):
253 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
255 def p4_last_change():
256 results = p4CmdList(["changes", "-m", "1"])
257 return int(results[0]['change'])
259 def p4_describe(change):
260 """Make sure it returns a valid result by checking for
261 the presence of field "time". Return a dict of the
262 results."""
264 ds = p4CmdList(["describe", "-s", str(change)])
265 if len(ds) != 1:
266 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
268 d = ds[0]
270 if "p4ExitCode" in d:
271 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
272 str(d)))
273 if "code" in d:
274 if d["code"] == "error":
275 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
277 if "time" not in d:
278 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
280 return d
283 # Canonicalize the p4 type and return a tuple of the
284 # base type, plus any modifiers. See "p4 help filetypes"
285 # for a list and explanation.
287 def split_p4_type(p4type):
289 p4_filetypes_historical = {
290 "ctempobj": "binary+Sw",
291 "ctext": "text+C",
292 "cxtext": "text+Cx",
293 "ktext": "text+k",
294 "kxtext": "text+kx",
295 "ltext": "text+F",
296 "tempobj": "binary+FSw",
297 "ubinary": "binary+F",
298 "uresource": "resource+F",
299 "uxbinary": "binary+Fx",
300 "xbinary": "binary+x",
301 "xltext": "text+Fx",
302 "xtempobj": "binary+Swx",
303 "xtext": "text+x",
304 "xunicode": "unicode+x",
305 "xutf16": "utf16+x",
307 if p4type in p4_filetypes_historical:
308 p4type = p4_filetypes_historical[p4type]
309 mods = ""
310 s = p4type.split("+")
311 base = s[0]
312 mods = ""
313 if len(s) > 1:
314 mods = s[1]
315 return (base, mods)
318 # return the raw p4 type of a file (text, text+ko, etc)
320 def p4_type(f):
321 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
322 return results[0]['headType']
325 # Given a type base and modifier, return a regexp matching
326 # the keywords that can be expanded in the file
328 def p4_keywords_regexp_for_type(base, type_mods):
329 if base in ("text", "unicode", "binary"):
330 kwords = None
331 if "ko" in type_mods:
332 kwords = 'Id|Header'
333 elif "k" in type_mods:
334 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
335 else:
336 return None
337 pattern = r"""
338 \$ # Starts with a dollar, followed by...
339 (%s) # one of the keywords, followed by...
340 (:[^$\n]+)? # possibly an old expansion, followed by...
341 \$ # another dollar
342 """ % kwords
343 return pattern
344 else:
345 return None
348 # Given a file, return a regexp matching the possible
349 # RCS keywords that will be expanded, or None for files
350 # with kw expansion turned off.
352 def p4_keywords_regexp_for_file(file):
353 if not os.path.exists(file):
354 return None
355 else:
356 (type_base, type_mods) = split_p4_type(p4_type(file))
357 return p4_keywords_regexp_for_type(type_base, type_mods)
359 def setP4ExecBit(file, mode):
360 # Reopens an already open file and changes the execute bit to match
361 # the execute bit setting in the passed in mode.
363 p4Type = "+x"
365 if not isModeExec(mode):
366 p4Type = getP4OpenedType(file)
367 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
368 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
369 if p4Type[-1] == "+":
370 p4Type = p4Type[0:-1]
372 p4_reopen(p4Type, file)
374 def getP4OpenedType(file):
375 # Returns the perforce file type for the given file.
377 result = p4_read_pipe(["opened", wildcard_encode(file)])
378 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
379 if match:
380 return match.group(1)
381 else:
382 die("Could not determine file type for %s (result: '%s')" % (file, result))
384 # Return the set of all p4 labels
385 def getP4Labels(depotPaths):
386 labels = set()
387 if isinstance(depotPaths,basestring):
388 depotPaths = [depotPaths]
390 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
391 label = l['label']
392 labels.add(label)
394 return labels
396 # Return the set of all git tags
397 def getGitTags():
398 gitTags = set()
399 for line in read_pipe_lines(["git", "tag"]):
400 tag = line.strip()
401 gitTags.add(tag)
402 return gitTags
404 def diffTreePattern():
405 # This is a simple generator for the diff tree regex pattern. This could be
406 # a class variable if this and parseDiffTreeEntry were a part of a class.
407 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
408 while True:
409 yield pattern
411 def parseDiffTreeEntry(entry):
412 """Parses a single diff tree entry into its component elements.
414 See git-diff-tree(1) manpage for details about the format of the diff
415 output. This method returns a dictionary with the following elements:
417 src_mode - The mode of the source file
418 dst_mode - The mode of the destination file
419 src_sha1 - The sha1 for the source file
420 dst_sha1 - The sha1 fr the destination file
421 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
422 status_score - The score for the status (applicable for 'C' and 'R'
423 statuses). This is None if there is no score.
424 src - The path for the source file.
425 dst - The path for the destination file. This is only present for
426 copy or renames. If it is not present, this is None.
428 If the pattern is not matched, None is returned."""
430 match = diffTreePattern().next().match(entry)
431 if match:
432 return {
433 'src_mode': match.group(1),
434 'dst_mode': match.group(2),
435 'src_sha1': match.group(3),
436 'dst_sha1': match.group(4),
437 'status': match.group(5),
438 'status_score': match.group(6),
439 'src': match.group(7),
440 'dst': match.group(10)
442 return None
444 def isModeExec(mode):
445 # Returns True if the given git mode represents an executable file,
446 # otherwise False.
447 return mode[-3:] == "755"
449 def isModeExecChanged(src_mode, dst_mode):
450 return isModeExec(src_mode) != isModeExec(dst_mode)
452 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
454 if isinstance(cmd,basestring):
455 cmd = "-G " + cmd
456 expand = True
457 else:
458 cmd = ["-G"] + cmd
459 expand = False
461 cmd = p4_build_cmd(cmd)
462 if verbose:
463 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
465 # Use a temporary file to avoid deadlocks without
466 # subprocess.communicate(), which would put another copy
467 # of stdout into memory.
468 stdin_file = None
469 if stdin is not None:
470 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
471 if isinstance(stdin,basestring):
472 stdin_file.write(stdin)
473 else:
474 for i in stdin:
475 stdin_file.write(i + '\n')
476 stdin_file.flush()
477 stdin_file.seek(0)
479 p4 = subprocess.Popen(cmd,
480 shell=expand,
481 stdin=stdin_file,
482 stdout=subprocess.PIPE)
484 result = []
485 try:
486 while True:
487 entry = marshal.load(p4.stdout)
488 if cb is not None:
489 cb(entry)
490 else:
491 result.append(entry)
492 except EOFError:
493 pass
494 exitCode = p4.wait()
495 if exitCode != 0:
496 entry = {}
497 entry["p4ExitCode"] = exitCode
498 result.append(entry)
500 return result
502 def p4Cmd(cmd):
503 list = p4CmdList(cmd)
504 result = {}
505 for entry in list:
506 result.update(entry)
507 return result;
509 def p4Where(depotPath):
510 if not depotPath.endswith("/"):
511 depotPath += "/"
512 depotPathLong = depotPath + "..."
513 outputList = p4CmdList(["where", depotPathLong])
514 output = None
515 for entry in outputList:
516 if "depotFile" in entry:
517 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
518 # The base path always ends with "/...".
519 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
520 output = entry
521 break
522 elif "data" in entry:
523 data = entry.get("data")
524 space = data.find(" ")
525 if data[:space] == depotPath:
526 output = entry
527 break
528 if output == None:
529 return ""
530 if output["code"] == "error":
531 return ""
532 clientPath = ""
533 if "path" in output:
534 clientPath = output.get("path")
535 elif "data" in output:
536 data = output.get("data")
537 lastSpace = data.rfind(" ")
538 clientPath = data[lastSpace + 1:]
540 if clientPath.endswith("..."):
541 clientPath = clientPath[:-3]
542 return clientPath
544 def currentGitBranch():
545 return read_pipe("git name-rev HEAD").split(" ")[1].strip()
547 def isValidGitDir(path):
548 if (os.path.exists(path + "/HEAD")
549 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
550 return True;
551 return False
553 def parseRevision(ref):
554 return read_pipe("git rev-parse %s" % ref).strip()
556 def branchExists(ref):
557 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
558 ignore_error=True)
559 return len(rev) > 0
561 def extractLogMessageFromGitCommit(commit):
562 logMessage = ""
564 ## fixme: title is first line of commit, not 1st paragraph.
565 foundTitle = False
566 for log in read_pipe_lines("git cat-file commit %s" % commit):
567 if not foundTitle:
568 if len(log) == 1:
569 foundTitle = True
570 continue
572 logMessage += log
573 return logMessage
575 def extractSettingsGitLog(log):
576 values = {}
577 for line in log.split("\n"):
578 line = line.strip()
579 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
580 if not m:
581 continue
583 assignments = m.group(1).split (':')
584 for a in assignments:
585 vals = a.split ('=')
586 key = vals[0].strip()
587 val = ('='.join (vals[1:])).strip()
588 if val.endswith ('\"') and val.startswith('"'):
589 val = val[1:-1]
591 values[key] = val
593 paths = values.get("depot-paths")
594 if not paths:
595 paths = values.get("depot-path")
596 if paths:
597 values['depot-paths'] = paths.split(',')
598 return values
600 def gitBranchExists(branch):
601 proc = subprocess.Popen(["git", "rev-parse", branch],
602 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
603 return proc.wait() == 0;
605 _gitConfig = {}
607 def gitConfig(key):
608 if not _gitConfig.has_key(key):
609 cmd = [ "git", "config", key ]
610 s = read_pipe(cmd, ignore_error=True)
611 _gitConfig[key] = s.strip()
612 return _gitConfig[key]
614 def gitConfigBool(key):
615 """Return a bool, using git config --bool. It is True only if the
616 variable is set to true, and False if set to false or not present
617 in the config."""
619 if not _gitConfig.has_key(key):
620 cmd = [ "git", "config", "--bool", key ]
621 s = read_pipe(cmd, ignore_error=True)
622 v = s.strip()
623 _gitConfig[key] = v == "true"
624 return _gitConfig[key]
626 def gitConfigList(key):
627 if not _gitConfig.has_key(key):
628 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
629 _gitConfig[key] = s.strip().split(os.linesep)
630 return _gitConfig[key]
632 def p4BranchesInGit(branchesAreInRemotes=True):
633 """Find all the branches whose names start with "p4/", looking
634 in remotes or heads as specified by the argument. Return
635 a dictionary of { branch: revision } for each one found.
636 The branch names are the short names, without any
637 "p4/" prefix."""
639 branches = {}
641 cmdline = "git rev-parse --symbolic "
642 if branchesAreInRemotes:
643 cmdline += "--remotes"
644 else:
645 cmdline += "--branches"
647 for line in read_pipe_lines(cmdline):
648 line = line.strip()
650 # only import to p4/
651 if not line.startswith('p4/'):
652 continue
653 # special symbolic ref to p4/master
654 if line == "p4/HEAD":
655 continue
657 # strip off p4/ prefix
658 branch = line[len("p4/"):]
660 branches[branch] = parseRevision(line)
662 return branches
664 def branch_exists(branch):
665 """Make sure that the given ref name really exists."""
667 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
668 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
669 out, _ = p.communicate()
670 if p.returncode:
671 return False
672 # expect exactly one line of output: the branch name
673 return out.rstrip() == branch
675 def findUpstreamBranchPoint(head = "HEAD"):
676 branches = p4BranchesInGit()
677 # map from depot-path to branch name
678 branchByDepotPath = {}
679 for branch in branches.keys():
680 tip = branches[branch]
681 log = extractLogMessageFromGitCommit(tip)
682 settings = extractSettingsGitLog(log)
683 if settings.has_key("depot-paths"):
684 paths = ",".join(settings["depot-paths"])
685 branchByDepotPath[paths] = "remotes/p4/" + branch
687 settings = None
688 parent = 0
689 while parent < 65535:
690 commit = head + "~%s" % parent
691 log = extractLogMessageFromGitCommit(commit)
692 settings = extractSettingsGitLog(log)
693 if settings.has_key("depot-paths"):
694 paths = ",".join(settings["depot-paths"])
695 if branchByDepotPath.has_key(paths):
696 return [branchByDepotPath[paths], settings]
698 parent = parent + 1
700 return ["", settings]
702 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
703 if not silent:
704 print ("Creating/updating branch(es) in %s based on origin branch(es)"
705 % localRefPrefix)
707 originPrefix = "origin/p4/"
709 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
710 line = line.strip()
711 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
712 continue
714 headName = line[len(originPrefix):]
715 remoteHead = localRefPrefix + headName
716 originHead = line
718 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
719 if (not original.has_key('depot-paths')
720 or not original.has_key('change')):
721 continue
723 update = False
724 if not gitBranchExists(remoteHead):
725 if verbose:
726 print "creating %s" % remoteHead
727 update = True
728 else:
729 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
730 if settings.has_key('change') > 0:
731 if settings['depot-paths'] == original['depot-paths']:
732 originP4Change = int(original['change'])
733 p4Change = int(settings['change'])
734 if originP4Change > p4Change:
735 print ("%s (%s) is newer than %s (%s). "
736 "Updating p4 branch from origin."
737 % (originHead, originP4Change,
738 remoteHead, p4Change))
739 update = True
740 else:
741 print ("Ignoring: %s was imported from %s while "
742 "%s was imported from %s"
743 % (originHead, ','.join(original['depot-paths']),
744 remoteHead, ','.join(settings['depot-paths'])))
746 if update:
747 system("git update-ref %s %s" % (remoteHead, originHead))
749 def originP4BranchesExist():
750 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
753 def p4ParseNumericChangeRange(parts):
754 changeStart = int(parts[0][1:])
755 if parts[1] == '#head':
756 changeEnd = p4_last_change()
757 else:
758 changeEnd = int(parts[1])
760 return (changeStart, changeEnd)
762 def chooseBlockSize(blockSize):
763 if blockSize:
764 return blockSize
765 else:
766 return defaultBlockSize
768 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
769 assert depotPaths
771 # Parse the change range into start and end. Try to find integer
772 # revision ranges as these can be broken up into blocks to avoid
773 # hitting server-side limits (maxrows, maxscanresults). But if
774 # that doesn't work, fall back to using the raw revision specifier
775 # strings, without using block mode.
777 if changeRange is None or changeRange == '':
778 changeStart = 1
779 changeEnd = p4_last_change()
780 block_size = chooseBlockSize(requestedBlockSize)
781 else:
782 parts = changeRange.split(',')
783 assert len(parts) == 2
784 try:
785 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
786 block_size = chooseBlockSize(requestedBlockSize)
787 except:
788 changeStart = parts[0][1:]
789 changeEnd = parts[1]
790 if requestedBlockSize:
791 die("cannot use --changes-block-size with non-numeric revisions")
792 block_size = None
794 # Accumulate change numbers in a dictionary to avoid duplicates
795 changes = {}
797 for p in depotPaths:
798 # Retrieve changes a block at a time, to prevent running
799 # into a MaxResults/MaxScanRows error from the server.
801 while True:
802 cmd = ['changes']
804 if block_size:
805 end = min(changeEnd, changeStart + block_size)
806 revisionRange = "%d,%d" % (changeStart, end)
807 else:
808 revisionRange = "%s,%s" % (changeStart, changeEnd)
810 cmd += ["%s...@%s" % (p, revisionRange)]
812 for line in p4_read_pipe_lines(cmd):
813 changeNum = int(line.split(" ")[1])
814 changes[changeNum] = True
816 if not block_size:
817 break
819 if end >= changeEnd:
820 break
822 changeStart = end + 1
824 changelist = changes.keys()
825 changelist.sort()
826 return changelist
828 def p4PathStartsWith(path, prefix):
829 # This method tries to remedy a potential mixed-case issue:
831 # If UserA adds //depot/DirA/file1
832 # and UserB adds //depot/dira/file2
834 # we may or may not have a problem. If you have core.ignorecase=true,
835 # we treat DirA and dira as the same directory
836 if gitConfigBool("core.ignorecase"):
837 return path.lower().startswith(prefix.lower())
838 return path.startswith(prefix)
840 def getClientSpec():
841 """Look at the p4 client spec, create a View() object that contains
842 all the mappings, and return it."""
844 specList = p4CmdList("client -o")
845 if len(specList) != 1:
846 die('Output from "client -o" is %d lines, expecting 1' %
847 len(specList))
849 # dictionary of all client parameters
850 entry = specList[0]
852 # the //client/ name
853 client_name = entry["Client"]
855 # just the keys that start with "View"
856 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
858 # hold this new View
859 view = View(client_name)
861 # append the lines, in order, to the view
862 for view_num in range(len(view_keys)):
863 k = "View%d" % view_num
864 if k not in view_keys:
865 die("Expected view key %s missing" % k)
866 view.append(entry[k])
868 return view
870 def getClientRoot():
871 """Grab the client directory."""
873 output = p4CmdList("client -o")
874 if len(output) != 1:
875 die('Output from "client -o" is %d lines, expecting 1' % len(output))
877 entry = output[0]
878 if "Root" not in entry:
879 die('Client has no "Root"')
881 return entry["Root"]
884 # P4 wildcards are not allowed in filenames. P4 complains
885 # if you simply add them, but you can force it with "-f", in
886 # which case it translates them into %xx encoding internally.
888 def wildcard_decode(path):
889 # Search for and fix just these four characters. Do % last so
890 # that fixing it does not inadvertently create new %-escapes.
891 # Cannot have * in a filename in windows; untested as to
892 # what p4 would do in such a case.
893 if not platform.system() == "Windows":
894 path = path.replace("%2A", "*")
895 path = path.replace("%23", "#") \
896 .replace("%40", "@") \
897 .replace("%25", "%")
898 return path
900 def wildcard_encode(path):
901 # do % first to avoid double-encoding the %s introduced here
902 path = path.replace("%", "%25") \
903 .replace("*", "%2A") \
904 .replace("#", "%23") \
905 .replace("@", "%40")
906 return path
908 def wildcard_present(path):
909 m = re.search("[*#@%]", path)
910 return m is not None
912 class Command:
913 def __init__(self):
914 self.usage = "usage: %prog [options]"
915 self.needsGit = True
916 self.verbose = False
918 class P4UserMap:
919 def __init__(self):
920 self.userMapFromPerforceServer = False
921 self.myP4UserId = None
923 def p4UserId(self):
924 if self.myP4UserId:
925 return self.myP4UserId
927 results = p4CmdList("user -o")
928 for r in results:
929 if r.has_key('User'):
930 self.myP4UserId = r['User']
931 return r['User']
932 die("Could not find your p4 user id")
934 def p4UserIsMe(self, p4User):
935 # return True if the given p4 user is actually me
936 me = self.p4UserId()
937 if not p4User or p4User != me:
938 return False
939 else:
940 return True
942 def getUserCacheFilename(self):
943 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
944 return home + "/.gitp4-usercache.txt"
946 def getUserMapFromPerforceServer(self):
947 if self.userMapFromPerforceServer:
948 return
949 self.users = {}
950 self.emails = {}
952 for output in p4CmdList("users"):
953 if not output.has_key("User"):
954 continue
955 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
956 self.emails[output["Email"]] = output["User"]
959 s = ''
960 for (key, val) in self.users.items():
961 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
963 open(self.getUserCacheFilename(), "wb").write(s)
964 self.userMapFromPerforceServer = True
966 def loadUserMapFromCache(self):
967 self.users = {}
968 self.userMapFromPerforceServer = False
969 try:
970 cache = open(self.getUserCacheFilename(), "rb")
971 lines = cache.readlines()
972 cache.close()
973 for line in lines:
974 entry = line.strip().split("\t")
975 self.users[entry[0]] = entry[1]
976 except IOError:
977 self.getUserMapFromPerforceServer()
979 class P4Debug(Command):
980 def __init__(self):
981 Command.__init__(self)
982 self.options = []
983 self.description = "A tool to debug the output of p4 -G."
984 self.needsGit = False
986 def run(self, args):
987 j = 0
988 for output in p4CmdList(args):
989 print 'Element: %d' % j
990 j += 1
991 print output
992 return True
994 class P4RollBack(Command):
995 def __init__(self):
996 Command.__init__(self)
997 self.options = [
998 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1000 self.description = "A tool to debug the multi-branch import. Don't use :)"
1001 self.rollbackLocalBranches = False
1003 def run(self, args):
1004 if len(args) != 1:
1005 return False
1006 maxChange = int(args[0])
1008 if "p4ExitCode" in p4Cmd("changes -m 1"):
1009 die("Problems executing p4");
1011 if self.rollbackLocalBranches:
1012 refPrefix = "refs/heads/"
1013 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1014 else:
1015 refPrefix = "refs/remotes/"
1016 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1018 for line in lines:
1019 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1020 line = line.strip()
1021 ref = refPrefix + line
1022 log = extractLogMessageFromGitCommit(ref)
1023 settings = extractSettingsGitLog(log)
1025 depotPaths = settings['depot-paths']
1026 change = settings['change']
1028 changed = False
1030 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1031 for p in depotPaths]))) == 0:
1032 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1033 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034 continue
1036 while change and int(change) > maxChange:
1037 changed = True
1038 if self.verbose:
1039 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1040 system("git update-ref %s \"%s^\"" % (ref, ref))
1041 log = extractLogMessageFromGitCommit(ref)
1042 settings = extractSettingsGitLog(log)
1045 depotPaths = settings['depot-paths']
1046 change = settings['change']
1048 if changed:
1049 print "%s rewound to %s" % (ref, change)
1051 return True
1053 class P4Submit(Command, P4UserMap):
1055 conflict_behavior_choices = ("ask", "skip", "quit")
1057 def __init__(self):
1058 Command.__init__(self)
1059 P4UserMap.__init__(self)
1060 self.options = [
1061 optparse.make_option("--origin", dest="origin"),
1062 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1063 # preserve the user, requires relevant p4 permissions
1064 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1065 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1066 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1067 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1068 optparse.make_option("--conflict", dest="conflict_behavior",
1069 choices=self.conflict_behavior_choices),
1070 optparse.make_option("--branch", dest="branch"),
1072 self.description = "Submit changes from git to the perforce depot."
1073 self.usage += " [name of git branch to submit into perforce depot]"
1074 self.origin = ""
1075 self.detectRenames = False
1076 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1077 self.dry_run = False
1078 self.prepare_p4_only = False
1079 self.conflict_behavior = None
1080 self.isWindows = (platform.system() == "Windows")
1081 self.exportLabels = False
1082 self.p4HasMoveCommand = p4_has_move_command()
1083 self.branch = None
1085 def check(self):
1086 if len(p4CmdList("opened ...")) > 0:
1087 die("You have files opened with perforce! Close them before starting the sync.")
1089 def separate_jobs_from_description(self, message):
1090 """Extract and return a possible Jobs field in the commit
1091 message. It goes into a separate section in the p4 change
1092 specification.
1094 A jobs line starts with "Jobs:" and looks like a new field
1095 in a form. Values are white-space separated on the same
1096 line or on following lines that start with a tab.
1098 This does not parse and extract the full git commit message
1099 like a p4 form. It just sees the Jobs: line as a marker
1100 to pass everything from then on directly into the p4 form,
1101 but outside the description section.
1103 Return a tuple (stripped log message, jobs string)."""
1105 m = re.search(r'^Jobs:', message, re.MULTILINE)
1106 if m is None:
1107 return (message, None)
1109 jobtext = message[m.start():]
1110 stripped_message = message[:m.start()].rstrip()
1111 return (stripped_message, jobtext)
1113 def prepareLogMessage(self, template, message, jobs):
1114 """Edits the template returned from "p4 change -o" to insert
1115 the message in the Description field, and the jobs text in
1116 the Jobs field."""
1117 result = ""
1119 inDescriptionSection = False
1121 for line in template.split("\n"):
1122 if line.startswith("#"):
1123 result += line + "\n"
1124 continue
1126 if inDescriptionSection:
1127 if line.startswith("Files:") or line.startswith("Jobs:"):
1128 inDescriptionSection = False
1129 # insert Jobs section
1130 if jobs:
1131 result += jobs + "\n"
1132 else:
1133 continue
1134 else:
1135 if line.startswith("Description:"):
1136 inDescriptionSection = True
1137 line += "\n"
1138 for messageLine in message.split("\n"):
1139 line += "\t" + messageLine + "\n"
1141 result += line + "\n"
1143 return result
1145 def patchRCSKeywords(self, file, pattern):
1146 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1147 (handle, outFileName) = tempfile.mkstemp(dir='.')
1148 try:
1149 outFile = os.fdopen(handle, "w+")
1150 inFile = open(file, "r")
1151 regexp = re.compile(pattern, re.VERBOSE)
1152 for line in inFile.readlines():
1153 line = regexp.sub(r'$\1$', line)
1154 outFile.write(line)
1155 inFile.close()
1156 outFile.close()
1157 # Forcibly overwrite the original file
1158 os.unlink(file)
1159 shutil.move(outFileName, file)
1160 except:
1161 # cleanup our temporary file
1162 os.unlink(outFileName)
1163 print "Failed to strip RCS keywords in %s" % file
1164 raise
1166 print "Patched up RCS keywords in %s" % file
1168 def p4UserForCommit(self,id):
1169 # Return the tuple (perforce user,git email) for a given git commit id
1170 self.getUserMapFromPerforceServer()
1171 gitEmail = read_pipe(["git", "log", "--max-count=1",
1172 "--format=%ae", id])
1173 gitEmail = gitEmail.strip()
1174 if not self.emails.has_key(gitEmail):
1175 return (None,gitEmail)
1176 else:
1177 return (self.emails[gitEmail],gitEmail)
1179 def checkValidP4Users(self,commits):
1180 # check if any git authors cannot be mapped to p4 users
1181 for id in commits:
1182 (user,email) = self.p4UserForCommit(id)
1183 if not user:
1184 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1185 if gitConfigBool("git-p4.allowMissingP4Users"):
1186 print "%s" % msg
1187 else:
1188 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1190 def lastP4Changelist(self):
1191 # Get back the last changelist number submitted in this client spec. This
1192 # then gets used to patch up the username in the change. If the same
1193 # client spec is being used by multiple processes then this might go
1194 # wrong.
1195 results = p4CmdList("client -o") # find the current client
1196 client = None
1197 for r in results:
1198 if r.has_key('Client'):
1199 client = r['Client']
1200 break
1201 if not client:
1202 die("could not get client spec")
1203 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1204 for r in results:
1205 if r.has_key('change'):
1206 return r['change']
1207 die("Could not get changelist number for last submit - cannot patch up user details")
1209 def modifyChangelistUser(self, changelist, newUser):
1210 # fixup the user field of a changelist after it has been submitted.
1211 changes = p4CmdList("change -o %s" % changelist)
1212 if len(changes) != 1:
1213 die("Bad output from p4 change modifying %s to user %s" %
1214 (changelist, newUser))
1216 c = changes[0]
1217 if c['User'] == newUser: return # nothing to do
1218 c['User'] = newUser
1219 input = marshal.dumps(c)
1221 result = p4CmdList("change -f -i", stdin=input)
1222 for r in result:
1223 if r.has_key('code'):
1224 if r['code'] == 'error':
1225 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1226 if r.has_key('data'):
1227 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1228 return
1229 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1231 def canChangeChangelists(self):
1232 # check to see if we have p4 admin or super-user permissions, either of
1233 # which are required to modify changelists.
1234 results = p4CmdList(["protects", self.depotPath])
1235 for r in results:
1236 if r.has_key('perm'):
1237 if r['perm'] == 'admin':
1238 return 1
1239 if r['perm'] == 'super':
1240 return 1
1241 return 0
1243 def prepareSubmitTemplate(self):
1244 """Run "p4 change -o" to grab a change specification template.
1245 This does not use "p4 -G", as it is nice to keep the submission
1246 template in original order, since a human might edit it.
1248 Remove lines in the Files section that show changes to files
1249 outside the depot path we're committing into."""
1251 template = ""
1252 inFilesSection = False
1253 for line in p4_read_pipe_lines(['change', '-o']):
1254 if line.endswith("\r\n"):
1255 line = line[:-2] + "\n"
1256 if inFilesSection:
1257 if line.startswith("\t"):
1258 # path starts and ends with a tab
1259 path = line[1:]
1260 lastTab = path.rfind("\t")
1261 if lastTab != -1:
1262 path = path[:lastTab]
1263 if not p4PathStartsWith(path, self.depotPath):
1264 continue
1265 else:
1266 inFilesSection = False
1267 else:
1268 if line.startswith("Files:"):
1269 inFilesSection = True
1271 template += line
1273 return template
1275 def edit_template(self, template_file):
1276 """Invoke the editor to let the user change the submission
1277 message. Return true if okay to continue with the submit."""
1279 # if configured to skip the editing part, just submit
1280 if gitConfigBool("git-p4.skipSubmitEdit"):
1281 return True
1283 # look at the modification time, to check later if the user saved
1284 # the file
1285 mtime = os.stat(template_file).st_mtime
1287 # invoke the editor
1288 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1289 editor = os.environ.get("P4EDITOR")
1290 else:
1291 editor = read_pipe("git var GIT_EDITOR").strip()
1292 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1294 # If the file was not saved, prompt to see if this patch should
1295 # be skipped. But skip this verification step if configured so.
1296 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1297 return True
1299 # modification time updated means user saved the file
1300 if os.stat(template_file).st_mtime > mtime:
1301 return True
1303 while True:
1304 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1305 if response == 'y':
1306 return True
1307 if response == 'n':
1308 return False
1310 def get_diff_description(self, editedFiles, filesToAdd):
1311 # diff
1312 if os.environ.has_key("P4DIFF"):
1313 del(os.environ["P4DIFF"])
1314 diff = ""
1315 for editedFile in editedFiles:
1316 diff += p4_read_pipe(['diff', '-du',
1317 wildcard_encode(editedFile)])
1319 # new file diff
1320 newdiff = ""
1321 for newFile in filesToAdd:
1322 newdiff += "==== new file ====\n"
1323 newdiff += "--- /dev/null\n"
1324 newdiff += "+++ %s\n" % newFile
1325 f = open(newFile, "r")
1326 for line in f.readlines():
1327 newdiff += "+" + line
1328 f.close()
1330 return (diff + newdiff).replace('\r\n', '\n')
1332 def applyCommit(self, id):
1333 """Apply one commit, return True if it succeeded."""
1335 print "Applying", read_pipe(["git", "show", "-s",
1336 "--format=format:%h %s", id])
1338 (p4User, gitEmail) = self.p4UserForCommit(id)
1340 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1341 filesToAdd = set()
1342 filesToDelete = set()
1343 editedFiles = set()
1344 pureRenameCopy = set()
1345 filesToChangeExecBit = {}
1347 for line in diff:
1348 diff = parseDiffTreeEntry(line)
1349 modifier = diff['status']
1350 path = diff['src']
1351 if modifier == "M":
1352 p4_edit(path)
1353 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1354 filesToChangeExecBit[path] = diff['dst_mode']
1355 editedFiles.add(path)
1356 elif modifier == "A":
1357 filesToAdd.add(path)
1358 filesToChangeExecBit[path] = diff['dst_mode']
1359 if path in filesToDelete:
1360 filesToDelete.remove(path)
1361 elif modifier == "D":
1362 filesToDelete.add(path)
1363 if path in filesToAdd:
1364 filesToAdd.remove(path)
1365 elif modifier == "C":
1366 src, dest = diff['src'], diff['dst']
1367 p4_integrate(src, dest)
1368 pureRenameCopy.add(dest)
1369 if diff['src_sha1'] != diff['dst_sha1']:
1370 p4_edit(dest)
1371 pureRenameCopy.discard(dest)
1372 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1373 p4_edit(dest)
1374 pureRenameCopy.discard(dest)
1375 filesToChangeExecBit[dest] = diff['dst_mode']
1376 if self.isWindows:
1377 # turn off read-only attribute
1378 os.chmod(dest, stat.S_IWRITE)
1379 os.unlink(dest)
1380 editedFiles.add(dest)
1381 elif modifier == "R":
1382 src, dest = diff['src'], diff['dst']
1383 if self.p4HasMoveCommand:
1384 p4_edit(src) # src must be open before move
1385 p4_move(src, dest) # opens for (move/delete, move/add)
1386 else:
1387 p4_integrate(src, dest)
1388 if diff['src_sha1'] != diff['dst_sha1']:
1389 p4_edit(dest)
1390 else:
1391 pureRenameCopy.add(dest)
1392 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1393 if not self.p4HasMoveCommand:
1394 p4_edit(dest) # with move: already open, writable
1395 filesToChangeExecBit[dest] = diff['dst_mode']
1396 if not self.p4HasMoveCommand:
1397 if self.isWindows:
1398 os.chmod(dest, stat.S_IWRITE)
1399 os.unlink(dest)
1400 filesToDelete.add(src)
1401 editedFiles.add(dest)
1402 else:
1403 die("unknown modifier %s for %s" % (modifier, path))
1405 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1406 patchcmd = diffcmd + " | git apply "
1407 tryPatchCmd = patchcmd + "--check -"
1408 applyPatchCmd = patchcmd + "--check --apply -"
1409 patch_succeeded = True
1411 if os.system(tryPatchCmd) != 0:
1412 fixed_rcs_keywords = False
1413 patch_succeeded = False
1414 print "Unfortunately applying the change failed!"
1416 # Patch failed, maybe it's just RCS keyword woes. Look through
1417 # the patch to see if that's possible.
1418 if gitConfigBool("git-p4.attemptRCSCleanup"):
1419 file = None
1420 pattern = None
1421 kwfiles = {}
1422 for file in editedFiles | filesToDelete:
1423 # did this file's delta contain RCS keywords?
1424 pattern = p4_keywords_regexp_for_file(file)
1426 if pattern:
1427 # this file is a possibility...look for RCS keywords.
1428 regexp = re.compile(pattern, re.VERBOSE)
1429 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1430 if regexp.search(line):
1431 if verbose:
1432 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1433 kwfiles[file] = pattern
1434 break
1436 for file in kwfiles:
1437 if verbose:
1438 print "zapping %s with %s" % (line,pattern)
1439 # File is being deleted, so not open in p4. Must
1440 # disable the read-only bit on windows.
1441 if self.isWindows and file not in editedFiles:
1442 os.chmod(file, stat.S_IWRITE)
1443 self.patchRCSKeywords(file, kwfiles[file])
1444 fixed_rcs_keywords = True
1446 if fixed_rcs_keywords:
1447 print "Retrying the patch with RCS keywords cleaned up"
1448 if os.system(tryPatchCmd) == 0:
1449 patch_succeeded = True
1451 if not patch_succeeded:
1452 for f in editedFiles:
1453 p4_revert(f)
1454 return False
1457 # Apply the patch for real, and do add/delete/+x handling.
1459 system(applyPatchCmd)
1461 for f in filesToAdd:
1462 p4_add(f)
1463 for f in filesToDelete:
1464 p4_revert(f)
1465 p4_delete(f)
1467 # Set/clear executable bits
1468 for f in filesToChangeExecBit.keys():
1469 mode = filesToChangeExecBit[f]
1470 setP4ExecBit(f, mode)
1473 # Build p4 change description, starting with the contents
1474 # of the git commit message.
1476 logMessage = extractLogMessageFromGitCommit(id)
1477 logMessage = logMessage.strip()
1478 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1480 template = self.prepareSubmitTemplate()
1481 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1483 if self.preserveUser:
1484 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1486 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1487 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1488 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1489 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1491 separatorLine = "######## everything below this line is just the diff #######\n"
1492 if not self.prepare_p4_only:
1493 submitTemplate += separatorLine
1494 submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1496 (handle, fileName) = tempfile.mkstemp()
1497 tmpFile = os.fdopen(handle, "w+b")
1498 if self.isWindows:
1499 submitTemplate = submitTemplate.replace("\n", "\r\n")
1500 tmpFile.write(submitTemplate)
1501 tmpFile.close()
1503 if self.prepare_p4_only:
1505 # Leave the p4 tree prepared, and the submit template around
1506 # and let the user decide what to do next
1508 print
1509 print "P4 workspace prepared for submission."
1510 print "To submit or revert, go to client workspace"
1511 print " " + self.clientPath
1512 print
1513 print "To submit, use \"p4 submit\" to write a new description,"
1514 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1515 " \"git p4\"." % fileName
1516 print "You can delete the file \"%s\" when finished." % fileName
1518 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1519 print "To preserve change ownership by user %s, you must\n" \
1520 "do \"p4 change -f <change>\" after submitting and\n" \
1521 "edit the User field."
1522 if pureRenameCopy:
1523 print "After submitting, renamed files must be re-synced."
1524 print "Invoke \"p4 sync -f\" on each of these files:"
1525 for f in pureRenameCopy:
1526 print " " + f
1528 print
1529 print "To revert the changes, use \"p4 revert ...\", and delete"
1530 print "the submit template file \"%s\"" % fileName
1531 if filesToAdd:
1532 print "Since the commit adds new files, they must be deleted:"
1533 for f in filesToAdd:
1534 print " " + f
1535 print
1536 return True
1539 # Let the user edit the change description, then submit it.
1541 if self.edit_template(fileName):
1542 # read the edited message and submit
1543 ret = True
1544 tmpFile = open(fileName, "rb")
1545 message = tmpFile.read()
1546 tmpFile.close()
1547 if self.isWindows:
1548 message = message.replace("\r\n", "\n")
1549 submitTemplate = message[:message.index(separatorLine)]
1550 p4_write_pipe(['submit', '-i'], submitTemplate)
1552 if self.preserveUser:
1553 if p4User:
1554 # Get last changelist number. Cannot easily get it from
1555 # the submit command output as the output is
1556 # unmarshalled.
1557 changelist = self.lastP4Changelist()
1558 self.modifyChangelistUser(changelist, p4User)
1560 # The rename/copy happened by applying a patch that created a
1561 # new file. This leaves it writable, which confuses p4.
1562 for f in pureRenameCopy:
1563 p4_sync(f, "-f")
1565 else:
1566 # skip this patch
1567 ret = False
1568 print "Submission cancelled, undoing p4 changes."
1569 for f in editedFiles:
1570 p4_revert(f)
1571 for f in filesToAdd:
1572 p4_revert(f)
1573 os.remove(f)
1574 for f in filesToDelete:
1575 p4_revert(f)
1577 os.remove(fileName)
1578 return ret
1580 # Export git tags as p4 labels. Create a p4 label and then tag
1581 # with that.
1582 def exportGitTags(self, gitTags):
1583 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1584 if len(validLabelRegexp) == 0:
1585 validLabelRegexp = defaultLabelRegexp
1586 m = re.compile(validLabelRegexp)
1588 for name in gitTags:
1590 if not m.match(name):
1591 if verbose:
1592 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1593 continue
1595 # Get the p4 commit this corresponds to
1596 logMessage = extractLogMessageFromGitCommit(name)
1597 values = extractSettingsGitLog(logMessage)
1599 if not values.has_key('change'):
1600 # a tag pointing to something not sent to p4; ignore
1601 if verbose:
1602 print "git tag %s does not give a p4 commit" % name
1603 continue
1604 else:
1605 changelist = values['change']
1607 # Get the tag details.
1608 inHeader = True
1609 isAnnotated = False
1610 body = []
1611 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612 l = l.strip()
1613 if inHeader:
1614 if re.match(r'tag\s+', l):
1615 isAnnotated = True
1616 elif re.match(r'\s*$', l):
1617 inHeader = False
1618 continue
1619 else:
1620 body.append(l)
1622 if not isAnnotated:
1623 body = ["lightweight tag imported by git p4\n"]
1625 # Create the label - use the same view as the client spec we are using
1626 clientSpec = getClientSpec()
1628 labelTemplate = "Label: %s\n" % name
1629 labelTemplate += "Description:\n"
1630 for b in body:
1631 labelTemplate += "\t" + b + "\n"
1632 labelTemplate += "View:\n"
1633 for depot_side in clientSpec.mappings:
1634 labelTemplate += "\t%s\n" % depot_side
1636 if self.dry_run:
1637 print "Would create p4 label %s for tag" % name
1638 elif self.prepare_p4_only:
1639 print "Not creating p4 label %s for tag due to option" \
1640 " --prepare-p4-only" % name
1641 else:
1642 p4_write_pipe(["label", "-i"], labelTemplate)
1644 # Use the label
1645 p4_system(["tag", "-l", name] +
1646 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1648 if verbose:
1649 print "created p4 label for tag %s" % name
1651 def run(self, args):
1652 if len(args) == 0:
1653 self.master = currentGitBranch()
1654 if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1655 die("Detecting current git branch failed!")
1656 elif len(args) == 1:
1657 self.master = args[0]
1658 if not branchExists(self.master):
1659 die("Branch %s does not exist" % self.master)
1660 else:
1661 return False
1663 allowSubmit = gitConfig("git-p4.allowSubmit")
1664 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1665 die("%s is not in git-p4.allowSubmit" % self.master)
1667 [upstream, settings] = findUpstreamBranchPoint()
1668 self.depotPath = settings['depot-paths'][0]
1669 if len(self.origin) == 0:
1670 self.origin = upstream
1672 if self.preserveUser:
1673 if not self.canChangeChangelists():
1674 die("Cannot preserve user names without p4 super-user or admin permissions")
1676 # if not set from the command line, try the config file
1677 if self.conflict_behavior is None:
1678 val = gitConfig("git-p4.conflict")
1679 if val:
1680 if val not in self.conflict_behavior_choices:
1681 die("Invalid value '%s' for config git-p4.conflict" % val)
1682 else:
1683 val = "ask"
1684 self.conflict_behavior = val
1686 if self.verbose:
1687 print "Origin branch is " + self.origin
1689 if len(self.depotPath) == 0:
1690 print "Internal error: cannot locate perforce depot path from existing branches"
1691 sys.exit(128)
1693 self.useClientSpec = False
1694 if gitConfigBool("git-p4.useclientspec"):
1695 self.useClientSpec = True
1696 if self.useClientSpec:
1697 self.clientSpecDirs = getClientSpec()
1699 # Check for the existance of P4 branches
1700 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1702 if self.useClientSpec and not branchesDetected:
1703 # all files are relative to the client spec
1704 self.clientPath = getClientRoot()
1705 else:
1706 self.clientPath = p4Where(self.depotPath)
1708 if self.clientPath == "":
1709 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1711 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1712 self.oldWorkingDirectory = os.getcwd()
1714 # ensure the clientPath exists
1715 new_client_dir = False
1716 if not os.path.exists(self.clientPath):
1717 new_client_dir = True
1718 os.makedirs(self.clientPath)
1720 chdir(self.clientPath, is_client_path=True)
1721 if self.dry_run:
1722 print "Would synchronize p4 checkout in %s" % self.clientPath
1723 else:
1724 print "Synchronizing p4 checkout..."
1725 if new_client_dir:
1726 # old one was destroyed, and maybe nobody told p4
1727 p4_sync("...", "-f")
1728 else:
1729 p4_sync("...")
1730 self.check()
1732 commits = []
1733 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1734 commits.append(line.strip())
1735 commits.reverse()
1737 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1738 self.checkAuthorship = False
1739 else:
1740 self.checkAuthorship = True
1742 if self.preserveUser:
1743 self.checkValidP4Users(commits)
1746 # Build up a set of options to be passed to diff when
1747 # submitting each commit to p4.
1749 if self.detectRenames:
1750 # command-line -M arg
1751 self.diffOpts = "-M"
1752 else:
1753 # If not explicitly set check the config variable
1754 detectRenames = gitConfig("git-p4.detectRenames")
1756 if detectRenames.lower() == "false" or detectRenames == "":
1757 self.diffOpts = ""
1758 elif detectRenames.lower() == "true":
1759 self.diffOpts = "-M"
1760 else:
1761 self.diffOpts = "-M%s" % detectRenames
1763 # no command-line arg for -C or --find-copies-harder, just
1764 # config variables
1765 detectCopies = gitConfig("git-p4.detectCopies")
1766 if detectCopies.lower() == "false" or detectCopies == "":
1767 pass
1768 elif detectCopies.lower() == "true":
1769 self.diffOpts += " -C"
1770 else:
1771 self.diffOpts += " -C%s" % detectCopies
1773 if gitConfigBool("git-p4.detectCopiesHarder"):
1774 self.diffOpts += " --find-copies-harder"
1777 # Apply the commits, one at a time. On failure, ask if should
1778 # continue to try the rest of the patches, or quit.
1780 if self.dry_run:
1781 print "Would apply"
1782 applied = []
1783 last = len(commits) - 1
1784 for i, commit in enumerate(commits):
1785 if self.dry_run:
1786 print " ", read_pipe(["git", "show", "-s",
1787 "--format=format:%h %s", commit])
1788 ok = True
1789 else:
1790 ok = self.applyCommit(commit)
1791 if ok:
1792 applied.append(commit)
1793 else:
1794 if self.prepare_p4_only and i < last:
1795 print "Processing only the first commit due to option" \
1796 " --prepare-p4-only"
1797 break
1798 if i < last:
1799 quit = False
1800 while True:
1801 # prompt for what to do, or use the option/variable
1802 if self.conflict_behavior == "ask":
1803 print "What do you want to do?"
1804 response = raw_input("[s]kip this commit but apply"
1805 " the rest, or [q]uit? ")
1806 if not response:
1807 continue
1808 elif self.conflict_behavior == "skip":
1809 response = "s"
1810 elif self.conflict_behavior == "quit":
1811 response = "q"
1812 else:
1813 die("Unknown conflict_behavior '%s'" %
1814 self.conflict_behavior)
1816 if response[0] == "s":
1817 print "Skipping this commit, but applying the rest"
1818 break
1819 if response[0] == "q":
1820 print "Quitting"
1821 quit = True
1822 break
1823 if quit:
1824 break
1826 chdir(self.oldWorkingDirectory)
1828 if self.dry_run:
1829 pass
1830 elif self.prepare_p4_only:
1831 pass
1832 elif len(commits) == len(applied):
1833 print "All commits applied!"
1835 sync = P4Sync()
1836 if self.branch:
1837 sync.branch = self.branch
1838 sync.run([])
1840 rebase = P4Rebase()
1841 rebase.rebase()
1843 else:
1844 if len(applied) == 0:
1845 print "No commits applied."
1846 else:
1847 print "Applied only the commits marked with '*':"
1848 for c in commits:
1849 if c in applied:
1850 star = "*"
1851 else:
1852 star = " "
1853 print star, read_pipe(["git", "show", "-s",
1854 "--format=format:%h %s", c])
1855 print "You will have to do 'git p4 sync' and rebase."
1857 if gitConfigBool("git-p4.exportLabels"):
1858 self.exportLabels = True
1860 if self.exportLabels:
1861 p4Labels = getP4Labels(self.depotPath)
1862 gitTags = getGitTags()
1864 missingGitTags = gitTags - p4Labels
1865 self.exportGitTags(missingGitTags)
1867 # exit with error unless everything applied perfectly
1868 if len(commits) != len(applied):
1869 sys.exit(1)
1871 return True
1873 class View(object):
1874 """Represent a p4 view ("p4 help views"), and map files in a
1875 repo according to the view."""
1877 def __init__(self, client_name):
1878 self.mappings = []
1879 self.client_prefix = "//%s/" % client_name
1880 # cache results of "p4 where" to lookup client file locations
1881 self.client_spec_path_cache = {}
1883 def append(self, view_line):
1884 """Parse a view line, splitting it into depot and client
1885 sides. Append to self.mappings, preserving order. This
1886 is only needed for tag creation."""
1888 # Split the view line into exactly two words. P4 enforces
1889 # structure on these lines that simplifies this quite a bit.
1891 # Either or both words may be double-quoted.
1892 # Single quotes do not matter.
1893 # Double-quote marks cannot occur inside the words.
1894 # A + or - prefix is also inside the quotes.
1895 # There are no quotes unless they contain a space.
1896 # The line is already white-space stripped.
1897 # The two words are separated by a single space.
1899 if view_line[0] == '"':
1900 # First word is double quoted. Find its end.
1901 close_quote_index = view_line.find('"', 1)
1902 if close_quote_index <= 0:
1903 die("No first-word closing quote found: %s" % view_line)
1904 depot_side = view_line[1:close_quote_index]
1905 # skip closing quote and space
1906 rhs_index = close_quote_index + 1 + 1
1907 else:
1908 space_index = view_line.find(" ")
1909 if space_index <= 0:
1910 die("No word-splitting space found: %s" % view_line)
1911 depot_side = view_line[0:space_index]
1912 rhs_index = space_index + 1
1914 # prefix + means overlay on previous mapping
1915 if depot_side.startswith("+"):
1916 depot_side = depot_side[1:]
1918 # prefix - means exclude this path, leave out of mappings
1919 exclude = False
1920 if depot_side.startswith("-"):
1921 exclude = True
1922 depot_side = depot_side[1:]
1924 if not exclude:
1925 self.mappings.append(depot_side)
1927 def convert_client_path(self, clientFile):
1928 # chop off //client/ part to make it relative
1929 if not clientFile.startswith(self.client_prefix):
1930 die("No prefix '%s' on clientFile '%s'" %
1931 (self.client_prefix, clientFile))
1932 return clientFile[len(self.client_prefix):]
1934 def update_client_spec_path_cache(self, files):
1935 """ Caching file paths by "p4 where" batch query """
1937 # List depot file paths exclude that already cached
1938 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1940 if len(fileArgs) == 0:
1941 return # All files in cache
1943 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1944 for res in where_result:
1945 if "code" in res and res["code"] == "error":
1946 # assume error is "... file(s) not in client view"
1947 continue
1948 if "clientFile" not in res:
1949 die("No clientFile in 'p4 where' output")
1950 if "unmap" in res:
1951 # it will list all of them, but only one not unmap-ped
1952 continue
1953 if gitConfigBool("core.ignorecase"):
1954 res['depotFile'] = res['depotFile'].lower()
1955 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1957 # not found files or unmap files set to ""
1958 for depotFile in fileArgs:
1959 if gitConfigBool("core.ignorecase"):
1960 depotFile = depotFile.lower()
1961 if depotFile not in self.client_spec_path_cache:
1962 self.client_spec_path_cache[depotFile] = ""
1964 def map_in_client(self, depot_path):
1965 """Return the relative location in the client where this
1966 depot file should live. Returns "" if the file should
1967 not be mapped in the client."""
1969 if gitConfigBool("core.ignorecase"):
1970 depot_path = depot_path.lower()
1972 if depot_path in self.client_spec_path_cache:
1973 return self.client_spec_path_cache[depot_path]
1975 die( "Error: %s is not found in client spec path" % depot_path )
1976 return ""
1978 class P4Sync(Command, P4UserMap):
1979 delete_actions = ( "delete", "move/delete", "purge" )
1981 def __init__(self):
1982 Command.__init__(self)
1983 P4UserMap.__init__(self)
1984 self.options = [
1985 optparse.make_option("--branch", dest="branch"),
1986 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1987 optparse.make_option("--changesfile", dest="changesFile"),
1988 optparse.make_option("--silent", dest="silent", action="store_true"),
1989 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1990 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1991 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1992 help="Import into refs/heads/ , not refs/remotes"),
1993 optparse.make_option("--max-changes", dest="maxChanges",
1994 help="Maximum number of changes to import"),
1995 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1996 help="Internal block size to use when iteratively calling p4 changes"),
1997 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1998 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1999 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2000 help="Only sync files that are included in the Perforce Client Spec"),
2001 optparse.make_option("-/", dest="cloneExclude",
2002 action="append", type="string",
2003 help="exclude depot path"),
2005 self.description = """Imports from Perforce into a git repository.\n
2006 example:
2007 //depot/my/project/ -- to import the current head
2008 //depot/my/project/@all -- to import everything
2009 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2011 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2013 self.usage += " //depot/path[@revRange]"
2014 self.silent = False
2015 self.createdBranches = set()
2016 self.committedChanges = set()
2017 self.branch = ""
2018 self.detectBranches = False
2019 self.detectLabels = False
2020 self.importLabels = False
2021 self.changesFile = ""
2022 self.syncWithOrigin = True
2023 self.importIntoRemotes = True
2024 self.maxChanges = ""
2025 self.changes_block_size = None
2026 self.keepRepoPath = False
2027 self.depotPaths = None
2028 self.p4BranchesInGit = []
2029 self.cloneExclude = []
2030 self.useClientSpec = False
2031 self.useClientSpec_from_options = False
2032 self.clientSpecDirs = None
2033 self.tempBranches = []
2034 self.tempBranchLocation = "git-p4-tmp"
2036 if gitConfig("git-p4.syncFromOrigin") == "false":
2037 self.syncWithOrigin = False
2039 # This is required for the "append" cloneExclude action
2040 def ensure_value(self, attr, value):
2041 if not hasattr(self, attr) or getattr(self, attr) is None:
2042 setattr(self, attr, value)
2043 return getattr(self, attr)
2045 # Force a checkpoint in fast-import and wait for it to finish
2046 def checkpoint(self):
2047 self.gitStream.write("checkpoint\n\n")
2048 self.gitStream.write("progress checkpoint\n\n")
2049 out = self.gitOutput.readline()
2050 if self.verbose:
2051 print "checkpoint finished: " + out
2053 def extractFilesFromCommit(self, commit):
2054 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2055 for path in self.cloneExclude]
2056 files = []
2057 fnum = 0
2058 while commit.has_key("depotFile%s" % fnum):
2059 path = commit["depotFile%s" % fnum]
2061 if [p for p in self.cloneExclude
2062 if p4PathStartsWith(path, p)]:
2063 found = False
2064 else:
2065 found = [p for p in self.depotPaths
2066 if p4PathStartsWith(path, p)]
2067 if not found:
2068 fnum = fnum + 1
2069 continue
2071 file = {}
2072 file["path"] = path
2073 file["rev"] = commit["rev%s" % fnum]
2074 file["action"] = commit["action%s" % fnum]
2075 file["type"] = commit["type%s" % fnum]
2076 files.append(file)
2077 fnum = fnum + 1
2078 return files
2080 def stripRepoPath(self, path, prefixes):
2081 """When streaming files, this is called to map a p4 depot path
2082 to where it should go in git. The prefixes are either
2083 self.depotPaths, or self.branchPrefixes in the case of
2084 branch detection."""
2086 if self.useClientSpec:
2087 # branch detection moves files up a level (the branch name)
2088 # from what client spec interpretation gives
2089 path = self.clientSpecDirs.map_in_client(path)
2090 if self.detectBranches:
2091 for b in self.knownBranches:
2092 if path.startswith(b + "/"):
2093 path = path[len(b)+1:]
2095 elif self.keepRepoPath:
2096 # Preserve everything in relative path name except leading
2097 # //depot/; just look at first prefix as they all should
2098 # be in the same depot.
2099 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2100 if p4PathStartsWith(path, depot):
2101 path = path[len(depot):]
2103 else:
2104 for p in prefixes:
2105 if p4PathStartsWith(path, p):
2106 path = path[len(p):]
2107 break
2109 path = wildcard_decode(path)
2110 return path
2112 def splitFilesIntoBranches(self, commit):
2113 """Look at each depotFile in the commit to figure out to what
2114 branch it belongs."""
2116 if self.clientSpecDirs:
2117 files = self.extractFilesFromCommit(commit)
2118 self.clientSpecDirs.update_client_spec_path_cache(files)
2120 branches = {}
2121 fnum = 0
2122 while commit.has_key("depotFile%s" % fnum):
2123 path = commit["depotFile%s" % fnum]
2124 found = [p for p in self.depotPaths
2125 if p4PathStartsWith(path, p)]
2126 if not found:
2127 fnum = fnum + 1
2128 continue
2130 file = {}
2131 file["path"] = path
2132 file["rev"] = commit["rev%s" % fnum]
2133 file["action"] = commit["action%s" % fnum]
2134 file["type"] = commit["type%s" % fnum]
2135 fnum = fnum + 1
2137 # start with the full relative path where this file would
2138 # go in a p4 client
2139 if self.useClientSpec:
2140 relPath = self.clientSpecDirs.map_in_client(path)
2141 else:
2142 relPath = self.stripRepoPath(path, self.depotPaths)
2144 for branch in self.knownBranches.keys():
2145 # add a trailing slash so that a commit into qt/4.2foo
2146 # doesn't end up in qt/4.2, e.g.
2147 if relPath.startswith(branch + "/"):
2148 if branch not in branches:
2149 branches[branch] = []
2150 branches[branch].append(file)
2151 break
2153 return branches
2155 # output one file from the P4 stream
2156 # - helper for streamP4Files
2158 def streamOneP4File(self, file, contents):
2159 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2160 if verbose:
2161 sys.stderr.write("%s\n" % relPath)
2163 (type_base, type_mods) = split_p4_type(file["type"])
2165 git_mode = "100644"
2166 if "x" in type_mods:
2167 git_mode = "100755"
2168 if type_base == "symlink":
2169 git_mode = "120000"
2170 # p4 print on a symlink sometimes contains "target\n";
2171 # if it does, remove the newline
2172 data = ''.join(contents)
2173 if not data:
2174 # Some version of p4 allowed creating a symlink that pointed
2175 # to nothing. This causes p4 errors when checking out such
2176 # a change, and errors here too. Work around it by ignoring
2177 # the bad symlink; hopefully a future change fixes it.
2178 print "\nIgnoring empty symlink in %s" % file['depotFile']
2179 return
2180 elif data[-1] == '\n':
2181 contents = [data[:-1]]
2182 else:
2183 contents = [data]
2185 if type_base == "utf16":
2186 # p4 delivers different text in the python output to -G
2187 # than it does when using "print -o", or normal p4 client
2188 # operations. utf16 is converted to ascii or utf8, perhaps.
2189 # But ascii text saved as -t utf16 is completely mangled.
2190 # Invoke print -o to get the real contents.
2192 # On windows, the newlines will always be mangled by print, so put
2193 # them back too. This is not needed to the cygwin windows version,
2194 # just the native "NT" type.
2196 text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
2197 if p4_version_string().find("/NT") >= 0:
2198 text = text.replace("\r\n", "\n")
2199 contents = [ text ]
2201 if type_base == "apple":
2202 # Apple filetype files will be streamed as a concatenation of
2203 # its appledouble header and the contents. This is useless
2204 # on both macs and non-macs. If using "print -q -o xx", it
2205 # will create "xx" with the data, and "%xx" with the header.
2206 # This is also not very useful.
2208 # Ideally, someday, this script can learn how to generate
2209 # appledouble files directly and import those to git, but
2210 # non-mac machines can never find a use for apple filetype.
2211 print "\nIgnoring apple filetype file %s" % file['depotFile']
2212 return
2214 # Note that we do not try to de-mangle keywords on utf16 files,
2215 # even though in theory somebody may want that.
2216 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2217 if pattern:
2218 regexp = re.compile(pattern, re.VERBOSE)
2219 text = ''.join(contents)
2220 text = regexp.sub(r'$\1$', text)
2221 contents = [ text ]
2223 try:
2224 relPath.decode('ascii')
2225 except:
2226 encoding = 'utf8'
2227 if gitConfig('git-p4.pathEncoding'):
2228 encoding = gitConfig('git-p4.pathEncoding')
2229 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2230 if self.verbose:
2231 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2233 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2235 # total length...
2236 length = 0
2237 for d in contents:
2238 length = length + len(d)
2240 self.gitStream.write("data %d\n" % length)
2241 for d in contents:
2242 self.gitStream.write(d)
2243 self.gitStream.write("\n")
2245 def streamOneP4Deletion(self, file):
2246 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2247 if verbose:
2248 sys.stderr.write("delete %s\n" % relPath)
2249 self.gitStream.write("D %s\n" % relPath)
2251 # handle another chunk of streaming data
2252 def streamP4FilesCb(self, marshalled):
2254 # catch p4 errors and complain
2255 err = None
2256 if "code" in marshalled:
2257 if marshalled["code"] == "error":
2258 if "data" in marshalled:
2259 err = marshalled["data"].rstrip()
2260 if err:
2261 f = None
2262 if self.stream_have_file_info:
2263 if "depotFile" in self.stream_file:
2264 f = self.stream_file["depotFile"]
2265 # force a failure in fast-import, else an empty
2266 # commit will be made
2267 self.gitStream.write("\n")
2268 self.gitStream.write("die-now\n")
2269 self.gitStream.close()
2270 # ignore errors, but make sure it exits first
2271 self.importProcess.wait()
2272 if f:
2273 die("Error from p4 print for %s: %s" % (f, err))
2274 else:
2275 die("Error from p4 print: %s" % err)
2277 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2278 # start of a new file - output the old one first
2279 self.streamOneP4File(self.stream_file, self.stream_contents)
2280 self.stream_file = {}
2281 self.stream_contents = []
2282 self.stream_have_file_info = False
2284 # pick up the new file information... for the
2285 # 'data' field we need to append to our array
2286 for k in marshalled.keys():
2287 if k == 'data':
2288 self.stream_contents.append(marshalled['data'])
2289 else:
2290 self.stream_file[k] = marshalled[k]
2292 self.stream_have_file_info = True
2294 # Stream directly from "p4 files" into "git fast-import"
2295 def streamP4Files(self, files):
2296 filesForCommit = []
2297 filesToRead = []
2298 filesToDelete = []
2300 for f in files:
2301 # if using a client spec, only add the files that have
2302 # a path in the client
2303 if self.clientSpecDirs:
2304 if self.clientSpecDirs.map_in_client(f['path']) == "":
2305 continue
2307 filesForCommit.append(f)
2308 if f['action'] in self.delete_actions:
2309 filesToDelete.append(f)
2310 else:
2311 filesToRead.append(f)
2313 # deleted files...
2314 for f in filesToDelete:
2315 self.streamOneP4Deletion(f)
2317 if len(filesToRead) > 0:
2318 self.stream_file = {}
2319 self.stream_contents = []
2320 self.stream_have_file_info = False
2322 # curry self argument
2323 def streamP4FilesCbSelf(entry):
2324 self.streamP4FilesCb(entry)
2326 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2328 p4CmdList(["-x", "-", "print"],
2329 stdin=fileArgs,
2330 cb=streamP4FilesCbSelf)
2332 # do the last chunk
2333 if self.stream_file.has_key('depotFile'):
2334 self.streamOneP4File(self.stream_file, self.stream_contents)
2336 def make_email(self, userid):
2337 if userid in self.users:
2338 return self.users[userid]
2339 else:
2340 return "%s <a@b>" % userid
2342 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2343 """ Stream a p4 tag.
2344 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2347 if verbose:
2348 print "writing tag %s for commit %s" % (labelName, commit)
2349 gitStream.write("tag %s\n" % labelName)
2350 gitStream.write("from %s\n" % commit)
2352 if labelDetails.has_key('Owner'):
2353 owner = labelDetails["Owner"]
2354 else:
2355 owner = None
2357 # Try to use the owner of the p4 label, or failing that,
2358 # the current p4 user id.
2359 if owner:
2360 email = self.make_email(owner)
2361 else:
2362 email = self.make_email(self.p4UserId())
2363 tagger = "%s %s %s" % (email, epoch, self.tz)
2365 gitStream.write("tagger %s\n" % tagger)
2367 print "labelDetails=",labelDetails
2368 if labelDetails.has_key('Description'):
2369 description = labelDetails['Description']
2370 else:
2371 description = 'Label from git p4'
2373 gitStream.write("data %d\n" % len(description))
2374 gitStream.write(description)
2375 gitStream.write("\n")
2377 def commit(self, details, files, branch, parent = ""):
2378 epoch = details["time"]
2379 author = details["user"]
2381 if self.verbose:
2382 print "commit into %s" % branch
2384 # start with reading files; if that fails, we should not
2385 # create a commit.
2386 new_files = []
2387 for f in files:
2388 if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2389 new_files.append (f)
2390 else:
2391 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2393 if self.clientSpecDirs:
2394 self.clientSpecDirs.update_client_spec_path_cache(files)
2396 self.gitStream.write("commit %s\n" % branch)
2397 self.gitStream.write("mark :%s\n" % details["change"])
2398 self.committedChanges.add(int(details["change"]))
2399 committer = ""
2400 if author not in self.users:
2401 self.getUserMapFromPerforceServer()
2402 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2404 self.gitStream.write("committer %s\n" % committer)
2406 self.gitStream.write("data <<EOT\n")
2407 self.gitStream.write(details["desc"])
2408 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2409 (','.join(self.branchPrefixes), details["change"]))
2410 if len(details['options']) > 0:
2411 self.gitStream.write(": options = %s" % details['options'])
2412 self.gitStream.write("]\nEOT\n\n")
2414 if len(parent) > 0:
2415 if self.verbose:
2416 print "parent %s" % parent
2417 self.gitStream.write("from %s\n" % parent)
2419 self.streamP4Files(new_files)
2420 self.gitStream.write("\n")
2422 change = int(details["change"])
2424 if self.labels.has_key(change):
2425 label = self.labels[change]
2426 labelDetails = label[0]
2427 labelRevisions = label[1]
2428 if self.verbose:
2429 print "Change %s is labelled %s" % (change, labelDetails)
2431 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2432 for p in self.branchPrefixes])
2434 if len(files) == len(labelRevisions):
2436 cleanedFiles = {}
2437 for info in files:
2438 if info["action"] in self.delete_actions:
2439 continue
2440 cleanedFiles[info["depotFile"]] = info["rev"]
2442 if cleanedFiles == labelRevisions:
2443 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2445 else:
2446 if not self.silent:
2447 print ("Tag %s does not match with change %s: files do not match."
2448 % (labelDetails["label"], change))
2450 else:
2451 if not self.silent:
2452 print ("Tag %s does not match with change %s: file count is different."
2453 % (labelDetails["label"], change))
2455 # Build a dictionary of changelists and labels, for "detect-labels" option.
2456 def getLabels(self):
2457 self.labels = {}
2459 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2460 if len(l) > 0 and not self.silent:
2461 print "Finding files belonging to labels in %s" % `self.depotPaths`
2463 for output in l:
2464 label = output["label"]
2465 revisions = {}
2466 newestChange = 0
2467 if self.verbose:
2468 print "Querying files for label %s" % label
2469 for file in p4CmdList(["files"] +
2470 ["%s...@%s" % (p, label)
2471 for p in self.depotPaths]):
2472 revisions[file["depotFile"]] = file["rev"]
2473 change = int(file["change"])
2474 if change > newestChange:
2475 newestChange = change
2477 self.labels[newestChange] = [output, revisions]
2479 if self.verbose:
2480 print "Label changes: %s" % self.labels.keys()
2482 # Import p4 labels as git tags. A direct mapping does not
2483 # exist, so assume that if all the files are at the same revision
2484 # then we can use that, or it's something more complicated we should
2485 # just ignore.
2486 def importP4Labels(self, stream, p4Labels):
2487 if verbose:
2488 print "import p4 labels: " + ' '.join(p4Labels)
2490 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2491 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2492 if len(validLabelRegexp) == 0:
2493 validLabelRegexp = defaultLabelRegexp
2494 m = re.compile(validLabelRegexp)
2496 for name in p4Labels:
2497 commitFound = False
2499 if not m.match(name):
2500 if verbose:
2501 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2502 continue
2504 if name in ignoredP4Labels:
2505 continue
2507 labelDetails = p4CmdList(['label', "-o", name])[0]
2509 # get the most recent changelist for each file in this label
2510 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2511 for p in self.depotPaths])
2513 if change.has_key('change'):
2514 # find the corresponding git commit; take the oldest commit
2515 changelist = int(change['change'])
2516 if changelist in self.committedChanges:
2517 gitCommit = ":%d" % changelist # use a fast-import mark
2518 commitFound = True
2519 else:
2520 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2521 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2522 if len(gitCommit) == 0:
2523 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2524 else:
2525 commitFound = True
2526 gitCommit = gitCommit.strip()
2528 if commitFound:
2529 # Convert from p4 time format
2530 try:
2531 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2532 except ValueError:
2533 print "Could not convert label time %s" % labelDetails['Update']
2534 tmwhen = 1
2536 when = int(time.mktime(tmwhen))
2537 self.streamTag(stream, name, labelDetails, gitCommit, when)
2538 if verbose:
2539 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2540 else:
2541 if verbose:
2542 print "Label %s has no changelists - possibly deleted?" % name
2544 if not commitFound:
2545 # We can't import this label; don't try again as it will get very
2546 # expensive repeatedly fetching all the files for labels that will
2547 # never be imported. If the label is moved in the future, the
2548 # ignore will need to be removed manually.
2549 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2551 def guessProjectName(self):
2552 for p in self.depotPaths:
2553 if p.endswith("/"):
2554 p = p[:-1]
2555 p = p[p.strip().rfind("/") + 1:]
2556 if not p.endswith("/"):
2557 p += "/"
2558 return p
2560 def getBranchMapping(self):
2561 lostAndFoundBranches = set()
2563 user = gitConfig("git-p4.branchUser")
2564 if len(user) > 0:
2565 command = "branches -u %s" % user
2566 else:
2567 command = "branches"
2569 for info in p4CmdList(command):
2570 details = p4Cmd(["branch", "-o", info["branch"]])
2571 viewIdx = 0
2572 while details.has_key("View%s" % viewIdx):
2573 paths = details["View%s" % viewIdx].split(" ")
2574 viewIdx = viewIdx + 1
2575 # require standard //depot/foo/... //depot/bar/... mapping
2576 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2577 continue
2578 source = paths[0]
2579 destination = paths[1]
2580 ## HACK
2581 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2582 source = source[len(self.depotPaths[0]):-4]
2583 destination = destination[len(self.depotPaths[0]):-4]
2585 if destination in self.knownBranches:
2586 if not self.silent:
2587 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2588 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2589 continue
2591 self.knownBranches[destination] = source
2593 lostAndFoundBranches.discard(destination)
2595 if source not in self.knownBranches:
2596 lostAndFoundBranches.add(source)
2598 # Perforce does not strictly require branches to be defined, so we also
2599 # check git config for a branch list.
2601 # Example of branch definition in git config file:
2602 # [git-p4]
2603 # branchList=main:branchA
2604 # branchList=main:branchB
2605 # branchList=branchA:branchC
2606 configBranches = gitConfigList("git-p4.branchList")
2607 for branch in configBranches:
2608 if branch:
2609 (source, destination) = branch.split(":")
2610 self.knownBranches[destination] = source
2612 lostAndFoundBranches.discard(destination)
2614 if source not in self.knownBranches:
2615 lostAndFoundBranches.add(source)
2618 for branch in lostAndFoundBranches:
2619 self.knownBranches[branch] = branch
2621 def getBranchMappingFromGitBranches(self):
2622 branches = p4BranchesInGit(self.importIntoRemotes)
2623 for branch in branches.keys():
2624 if branch == "master":
2625 branch = "main"
2626 else:
2627 branch = branch[len(self.projectName):]
2628 self.knownBranches[branch] = branch
2630 def updateOptionDict(self, d):
2631 option_keys = {}
2632 if self.keepRepoPath:
2633 option_keys['keepRepoPath'] = 1
2635 d["options"] = ' '.join(sorted(option_keys.keys()))
2637 def readOptions(self, d):
2638 self.keepRepoPath = (d.has_key('options')
2639 and ('keepRepoPath' in d['options']))
2641 def gitRefForBranch(self, branch):
2642 if branch == "main":
2643 return self.refPrefix + "master"
2645 if len(branch) <= 0:
2646 return branch
2648 return self.refPrefix + self.projectName + branch
2650 def gitCommitByP4Change(self, ref, change):
2651 if self.verbose:
2652 print "looking in ref " + ref + " for change %s using bisect..." % change
2654 earliestCommit = ""
2655 latestCommit = parseRevision(ref)
2657 while True:
2658 if self.verbose:
2659 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2660 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2661 if len(next) == 0:
2662 if self.verbose:
2663 print "argh"
2664 return ""
2665 log = extractLogMessageFromGitCommit(next)
2666 settings = extractSettingsGitLog(log)
2667 currentChange = int(settings['change'])
2668 if self.verbose:
2669 print "current change %s" % currentChange
2671 if currentChange == change:
2672 if self.verbose:
2673 print "found %s" % next
2674 return next
2676 if currentChange < change:
2677 earliestCommit = "^%s" % next
2678 else:
2679 latestCommit = "%s" % next
2681 return ""
2683 def importNewBranch(self, branch, maxChange):
2684 # make fast-import flush all changes to disk and update the refs using the checkpoint
2685 # command so that we can try to find the branch parent in the git history
2686 self.gitStream.write("checkpoint\n\n");
2687 self.gitStream.flush();
2688 branchPrefix = self.depotPaths[0] + branch + "/"
2689 range = "@1,%s" % maxChange
2690 #print "prefix" + branchPrefix
2691 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2692 if len(changes) <= 0:
2693 return False
2694 firstChange = changes[0]
2695 #print "first change in branch: %s" % firstChange
2696 sourceBranch = self.knownBranches[branch]
2697 sourceDepotPath = self.depotPaths[0] + sourceBranch
2698 sourceRef = self.gitRefForBranch(sourceBranch)
2699 #print "source " + sourceBranch
2701 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2702 #print "branch parent: %s" % branchParentChange
2703 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2704 if len(gitParent) > 0:
2705 self.initialParents[self.gitRefForBranch(branch)] = gitParent
2706 #print "parent git commit: %s" % gitParent
2708 self.importChanges(changes)
2709 return True
2711 def searchParent(self, parent, branch, target):
2712 parentFound = False
2713 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2714 "--no-merges", parent]):
2715 blob = blob.strip()
2716 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2717 parentFound = True
2718 if self.verbose:
2719 print "Found parent of %s in commit %s" % (branch, blob)
2720 break
2721 if parentFound:
2722 return blob
2723 else:
2724 return None
2726 def importChanges(self, changes):
2727 cnt = 1
2728 for change in changes:
2729 description = p4_describe(change)
2730 self.updateOptionDict(description)
2732 if not self.silent:
2733 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2734 sys.stdout.flush()
2735 cnt = cnt + 1
2737 try:
2738 if self.detectBranches:
2739 branches = self.splitFilesIntoBranches(description)
2740 for branch in branches.keys():
2741 ## HACK --hwn
2742 branchPrefix = self.depotPaths[0] + branch + "/"
2743 self.branchPrefixes = [ branchPrefix ]
2745 parent = ""
2747 filesForCommit = branches[branch]
2749 if self.verbose:
2750 print "branch is %s" % branch
2752 self.updatedBranches.add(branch)
2754 if branch not in self.createdBranches:
2755 self.createdBranches.add(branch)
2756 parent = self.knownBranches[branch]
2757 if parent == branch:
2758 parent = ""
2759 else:
2760 fullBranch = self.projectName + branch
2761 if fullBranch not in self.p4BranchesInGit:
2762 if not self.silent:
2763 print("\n Importing new branch %s" % fullBranch);
2764 if self.importNewBranch(branch, change - 1):
2765 parent = ""
2766 self.p4BranchesInGit.append(fullBranch)
2767 if not self.silent:
2768 print("\n Resuming with change %s" % change);
2770 if self.verbose:
2771 print "parent determined through known branches: %s" % parent
2773 branch = self.gitRefForBranch(branch)
2774 parent = self.gitRefForBranch(parent)
2776 if self.verbose:
2777 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2779 if len(parent) == 0 and branch in self.initialParents:
2780 parent = self.initialParents[branch]
2781 del self.initialParents[branch]
2783 blob = None
2784 if len(parent) > 0:
2785 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2786 if self.verbose:
2787 print "Creating temporary branch: " + tempBranch
2788 self.commit(description, filesForCommit, tempBranch)
2789 self.tempBranches.append(tempBranch)
2790 self.checkpoint()
2791 blob = self.searchParent(parent, branch, tempBranch)
2792 if blob:
2793 self.commit(description, filesForCommit, branch, blob)
2794 else:
2795 if self.verbose:
2796 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2797 self.commit(description, filesForCommit, branch, parent)
2798 else:
2799 files = self.extractFilesFromCommit(description)
2800 self.commit(description, files, self.branch,
2801 self.initialParent)
2802 # only needed once, to connect to the previous commit
2803 self.initialParent = ""
2804 except IOError:
2805 print self.gitError.read()
2806 sys.exit(1)
2808 def importHeadRevision(self, revision):
2809 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2811 details = {}
2812 details["user"] = "git perforce import user"
2813 details["desc"] = ("Initial import of %s from the state at revision %s\n"
2814 % (' '.join(self.depotPaths), revision))
2815 details["change"] = revision
2816 newestRevision = 0
2818 fileCnt = 0
2819 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2821 for info in p4CmdList(["files"] + fileArgs):
2823 if 'code' in info and info['code'] == 'error':
2824 sys.stderr.write("p4 returned an error: %s\n"
2825 % info['data'])
2826 if info['data'].find("must refer to client") >= 0:
2827 sys.stderr.write("This particular p4 error is misleading.\n")
2828 sys.stderr.write("Perhaps the depot path was misspelled.\n");
2829 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
2830 sys.exit(1)
2831 if 'p4ExitCode' in info:
2832 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2833 sys.exit(1)
2836 change = int(info["change"])
2837 if change > newestRevision:
2838 newestRevision = change
2840 if info["action"] in self.delete_actions:
2841 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2842 #fileCnt = fileCnt + 1
2843 continue
2845 for prop in ["depotFile", "rev", "action", "type" ]:
2846 details["%s%s" % (prop, fileCnt)] = info[prop]
2848 fileCnt = fileCnt + 1
2850 details["change"] = newestRevision
2852 # Use time from top-most change so that all git p4 clones of
2853 # the same p4 repo have the same commit SHA1s.
2854 res = p4_describe(newestRevision)
2855 details["time"] = res["time"]
2857 self.updateOptionDict(details)
2858 try:
2859 self.commit(details, self.extractFilesFromCommit(details), self.branch)
2860 except IOError:
2861 print "IO error with git fast-import. Is your git version recent enough?"
2862 print self.gitError.read()
2865 def run(self, args):
2866 self.depotPaths = []
2867 self.changeRange = ""
2868 self.previousDepotPaths = []
2869 self.hasOrigin = False
2871 # map from branch depot path to parent branch
2872 self.knownBranches = {}
2873 self.initialParents = {}
2875 if self.importIntoRemotes:
2876 self.refPrefix = "refs/remotes/p4/"
2877 else:
2878 self.refPrefix = "refs/heads/p4/"
2880 if self.syncWithOrigin:
2881 self.hasOrigin = originP4BranchesExist()
2882 if self.hasOrigin:
2883 if not self.silent:
2884 print 'Syncing with origin first, using "git fetch origin"'
2885 system("git fetch origin")
2887 branch_arg_given = bool(self.branch)
2888 if len(self.branch) == 0:
2889 self.branch = self.refPrefix + "master"
2890 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2891 system("git update-ref %s refs/heads/p4" % self.branch)
2892 system("git branch -D p4")
2894 # accept either the command-line option, or the configuration variable
2895 if self.useClientSpec:
2896 # will use this after clone to set the variable
2897 self.useClientSpec_from_options = True
2898 else:
2899 if gitConfigBool("git-p4.useclientspec"):
2900 self.useClientSpec = True
2901 if self.useClientSpec:
2902 self.clientSpecDirs = getClientSpec()
2904 # TODO: should always look at previous commits,
2905 # merge with previous imports, if possible.
2906 if args == []:
2907 if self.hasOrigin:
2908 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2910 # branches holds mapping from branch name to sha1
2911 branches = p4BranchesInGit(self.importIntoRemotes)
2913 # restrict to just this one, disabling detect-branches
2914 if branch_arg_given:
2915 short = self.branch.split("/")[-1]
2916 if short in branches:
2917 self.p4BranchesInGit = [ short ]
2918 else:
2919 self.p4BranchesInGit = branches.keys()
2921 if len(self.p4BranchesInGit) > 1:
2922 if not self.silent:
2923 print "Importing from/into multiple branches"
2924 self.detectBranches = True
2925 for branch in branches.keys():
2926 self.initialParents[self.refPrefix + branch] = \
2927 branches[branch]
2929 if self.verbose:
2930 print "branches: %s" % self.p4BranchesInGit
2932 p4Change = 0
2933 for branch in self.p4BranchesInGit:
2934 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
2936 settings = extractSettingsGitLog(logMsg)
2938 self.readOptions(settings)
2939 if (settings.has_key('depot-paths')
2940 and settings.has_key ('change')):
2941 change = int(settings['change']) + 1
2942 p4Change = max(p4Change, change)
2944 depotPaths = sorted(settings['depot-paths'])
2945 if self.previousDepotPaths == []:
2946 self.previousDepotPaths = depotPaths
2947 else:
2948 paths = []
2949 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2950 prev_list = prev.split("/")
2951 cur_list = cur.split("/")
2952 for i in range(0, min(len(cur_list), len(prev_list))):
2953 if cur_list[i] <> prev_list[i]:
2954 i = i - 1
2955 break
2957 paths.append ("/".join(cur_list[:i + 1]))
2959 self.previousDepotPaths = paths
2961 if p4Change > 0:
2962 self.depotPaths = sorted(self.previousDepotPaths)
2963 self.changeRange = "@%s,#head" % p4Change
2964 if not self.silent and not self.detectBranches:
2965 print "Performing incremental import into %s git branch" % self.branch
2967 # accept multiple ref name abbreviations:
2968 # refs/foo/bar/branch -> use it exactly
2969 # p4/branch -> prepend refs/remotes/ or refs/heads/
2970 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2971 if not self.branch.startswith("refs/"):
2972 if self.importIntoRemotes:
2973 prepend = "refs/remotes/"
2974 else:
2975 prepend = "refs/heads/"
2976 if not self.branch.startswith("p4/"):
2977 prepend += "p4/"
2978 self.branch = prepend + self.branch
2980 if len(args) == 0 and self.depotPaths:
2981 if not self.silent:
2982 print "Depot paths: %s" % ' '.join(self.depotPaths)
2983 else:
2984 if self.depotPaths and self.depotPaths != args:
2985 print ("previous import used depot path %s and now %s was specified. "
2986 "This doesn't work!" % (' '.join (self.depotPaths),
2987 ' '.join (args)))
2988 sys.exit(1)
2990 self.depotPaths = sorted(args)
2992 revision = ""
2993 self.users = {}
2995 # Make sure no revision specifiers are used when --changesfile
2996 # is specified.
2997 bad_changesfile = False
2998 if len(self.changesFile) > 0:
2999 for p in self.depotPaths:
3000 if p.find("@") >= 0 or p.find("#") >= 0:
3001 bad_changesfile = True
3002 break
3003 if bad_changesfile:
3004 die("Option --changesfile is incompatible with revision specifiers")
3006 newPaths = []
3007 for p in self.depotPaths:
3008 if p.find("@") != -1:
3009 atIdx = p.index("@")
3010 self.changeRange = p[atIdx:]
3011 if self.changeRange == "@all":
3012 self.changeRange = ""
3013 elif ',' not in self.changeRange:
3014 revision = self.changeRange
3015 self.changeRange = ""
3016 p = p[:atIdx]
3017 elif p.find("#") != -1:
3018 hashIdx = p.index("#")
3019 revision = p[hashIdx:]
3020 p = p[:hashIdx]
3021 elif self.previousDepotPaths == []:
3022 # pay attention to changesfile, if given, else import
3023 # the entire p4 tree at the head revision
3024 if len(self.changesFile) == 0:
3025 revision = "#head"
3027 p = re.sub ("\.\.\.$", "", p)
3028 if not p.endswith("/"):
3029 p += "/"
3031 newPaths.append(p)
3033 self.depotPaths = newPaths
3035 # --detect-branches may change this for each branch
3036 self.branchPrefixes = self.depotPaths
3038 self.loadUserMapFromCache()
3039 self.labels = {}
3040 if self.detectLabels:
3041 self.getLabels();
3043 if self.detectBranches:
3044 ## FIXME - what's a P4 projectName ?
3045 self.projectName = self.guessProjectName()
3047 if self.hasOrigin:
3048 self.getBranchMappingFromGitBranches()
3049 else:
3050 self.getBranchMapping()
3051 if self.verbose:
3052 print "p4-git branches: %s" % self.p4BranchesInGit
3053 print "initial parents: %s" % self.initialParents
3054 for b in self.p4BranchesInGit:
3055 if b != "master":
3057 ## FIXME
3058 b = b[len(self.projectName):]
3059 self.createdBranches.add(b)
3061 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3063 self.importProcess = subprocess.Popen(["git", "fast-import"],
3064 stdin=subprocess.PIPE,
3065 stdout=subprocess.PIPE,
3066 stderr=subprocess.PIPE);
3067 self.gitOutput = self.importProcess.stdout
3068 self.gitStream = self.importProcess.stdin
3069 self.gitError = self.importProcess.stderr
3071 if revision:
3072 self.importHeadRevision(revision)
3073 else:
3074 changes = []
3076 if len(self.changesFile) > 0:
3077 output = open(self.changesFile).readlines()
3078 changeSet = set()
3079 for line in output:
3080 changeSet.add(int(line))
3082 for change in changeSet:
3083 changes.append(change)
3085 changes.sort()
3086 else:
3087 # catch "git p4 sync" with no new branches, in a repo that
3088 # does not have any existing p4 branches
3089 if len(args) == 0:
3090 if not self.p4BranchesInGit:
3091 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3093 # The default branch is master, unless --branch is used to
3094 # specify something else. Make sure it exists, or complain
3095 # nicely about how to use --branch.
3096 if not self.detectBranches:
3097 if not branch_exists(self.branch):
3098 if branch_arg_given:
3099 die("Error: branch %s does not exist." % self.branch)
3100 else:
3101 die("Error: no branch %s; perhaps specify one with --branch." %
3102 self.branch)
3104 if self.verbose:
3105 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3106 self.changeRange)
3107 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3109 if len(self.maxChanges) > 0:
3110 changes = changes[:min(int(self.maxChanges), len(changes))]
3112 if len(changes) == 0:
3113 if not self.silent:
3114 print "No changes to import!"
3115 else:
3116 if not self.silent and not self.detectBranches:
3117 print "Import destination: %s" % self.branch
3119 self.updatedBranches = set()
3121 if not self.detectBranches:
3122 if args:
3123 # start a new branch
3124 self.initialParent = ""
3125 else:
3126 # build on a previous revision
3127 self.initialParent = parseRevision(self.branch)
3129 self.importChanges(changes)
3131 if not self.silent:
3132 print ""
3133 if len(self.updatedBranches) > 0:
3134 sys.stdout.write("Updated branches: ")
3135 for b in self.updatedBranches:
3136 sys.stdout.write("%s " % b)
3137 sys.stdout.write("\n")
3139 if gitConfigBool("git-p4.importLabels"):
3140 self.importLabels = True
3142 if self.importLabels:
3143 p4Labels = getP4Labels(self.depotPaths)
3144 gitTags = getGitTags()
3146 missingP4Labels = p4Labels - gitTags
3147 self.importP4Labels(self.gitStream, missingP4Labels)
3149 self.gitStream.close()
3150 if self.importProcess.wait() != 0:
3151 die("fast-import failed: %s" % self.gitError.read())
3152 self.gitOutput.close()
3153 self.gitError.close()
3155 # Cleanup temporary branches created during import
3156 if self.tempBranches != []:
3157 for branch in self.tempBranches:
3158 read_pipe("git update-ref -d %s" % branch)
3159 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3161 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3162 # a convenient shortcut refname "p4".
3163 if self.importIntoRemotes:
3164 head_ref = self.refPrefix + "HEAD"
3165 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3166 system(["git", "symbolic-ref", head_ref, self.branch])
3168 return True
3170 class P4Rebase(Command):
3171 def __init__(self):
3172 Command.__init__(self)
3173 self.options = [
3174 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3176 self.importLabels = False
3177 self.description = ("Fetches the latest revision from perforce and "
3178 + "rebases the current work (branch) against it")
3180 def run(self, args):
3181 sync = P4Sync()
3182 sync.importLabels = self.importLabels
3183 sync.run([])
3185 return self.rebase()
3187 def rebase(self):
3188 if os.system("git update-index --refresh") != 0:
3189 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3190 if len(read_pipe("git diff-index HEAD --")) > 0:
3191 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3193 [upstream, settings] = findUpstreamBranchPoint()
3194 if len(upstream) == 0:
3195 die("Cannot find upstream branchpoint for rebase")
3197 # the branchpoint may be p4/foo~3, so strip off the parent
3198 upstream = re.sub("~[0-9]+$", "", upstream)
3200 print "Rebasing the current branch onto %s" % upstream
3201 oldHead = read_pipe("git rev-parse HEAD").strip()
3202 system("git rebase %s" % upstream)
3203 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3204 return True
3206 class P4Clone(P4Sync):
3207 def __init__(self):
3208 P4Sync.__init__(self)
3209 self.description = "Creates a new git repository and imports from Perforce into it"
3210 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3211 self.options += [
3212 optparse.make_option("--destination", dest="cloneDestination",
3213 action='store', default=None,
3214 help="where to leave result of the clone"),
3215 optparse.make_option("--bare", dest="cloneBare",
3216 action="store_true", default=False),
3218 self.cloneDestination = None
3219 self.needsGit = False
3220 self.cloneBare = False
3222 def defaultDestination(self, args):
3223 ## TODO: use common prefix of args?
3224 depotPath = args[0]
3225 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3226 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3227 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3228 depotDir = re.sub(r"/$", "", depotDir)
3229 return os.path.split(depotDir)[1]
3231 def run(self, args):
3232 if len(args) < 1:
3233 return False
3235 if self.keepRepoPath and not self.cloneDestination:
3236 sys.stderr.write("Must specify destination for --keep-path\n")
3237 sys.exit(1)
3239 depotPaths = args
3241 if not self.cloneDestination and len(depotPaths) > 1:
3242 self.cloneDestination = depotPaths[-1]
3243 depotPaths = depotPaths[:-1]
3245 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3246 for p in depotPaths:
3247 if not p.startswith("//"):
3248 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3249 return False
3251 if not self.cloneDestination:
3252 self.cloneDestination = self.defaultDestination(args)
3254 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3256 if not os.path.exists(self.cloneDestination):
3257 os.makedirs(self.cloneDestination)
3258 chdir(self.cloneDestination)
3260 init_cmd = [ "git", "init" ]
3261 if self.cloneBare:
3262 init_cmd.append("--bare")
3263 retcode = subprocess.call(init_cmd)
3264 if retcode:
3265 raise CalledProcessError(retcode, init_cmd)
3267 if not P4Sync.run(self, depotPaths):
3268 return False
3270 # create a master branch and check out a work tree
3271 if gitBranchExists(self.branch):
3272 system([ "git", "branch", "master", self.branch ])
3273 if not self.cloneBare:
3274 system([ "git", "checkout", "-f" ])
3275 else:
3276 print 'Not checking out any branch, use ' \
3277 '"git checkout -q -b master <branch>"'
3279 # auto-set this variable if invoked with --use-client-spec
3280 if self.useClientSpec_from_options:
3281 system("git config --bool git-p4.useclientspec true")
3283 return True
3285 class P4Branches(Command):
3286 def __init__(self):
3287 Command.__init__(self)
3288 self.options = [ ]
3289 self.description = ("Shows the git branches that hold imports and their "
3290 + "corresponding perforce depot paths")
3291 self.verbose = False
3293 def run(self, args):
3294 if originP4BranchesExist():
3295 createOrUpdateBranchesFromOrigin()
3297 cmdline = "git rev-parse --symbolic "
3298 cmdline += " --remotes"
3300 for line in read_pipe_lines(cmdline):
3301 line = line.strip()
3303 if not line.startswith('p4/') or line == "p4/HEAD":
3304 continue
3305 branch = line
3307 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3308 settings = extractSettingsGitLog(log)
3310 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3311 return True
3313 class HelpFormatter(optparse.IndentedHelpFormatter):
3314 def __init__(self):
3315 optparse.IndentedHelpFormatter.__init__(self)
3317 def format_description(self, description):
3318 if description:
3319 return description + "\n"
3320 else:
3321 return ""
3323 def printUsage(commands):
3324 print "usage: %s <command> [options]" % sys.argv[0]
3325 print ""
3326 print "valid commands: %s" % ", ".join(commands)
3327 print ""
3328 print "Try %s <command> --help for command specific help." % sys.argv[0]
3329 print ""
3331 commands = {
3332 "debug" : P4Debug,
3333 "submit" : P4Submit,
3334 "commit" : P4Submit,
3335 "sync" : P4Sync,
3336 "rebase" : P4Rebase,
3337 "clone" : P4Clone,
3338 "rollback" : P4RollBack,
3339 "branches" : P4Branches
3343 def main():
3344 if len(sys.argv[1:]) == 0:
3345 printUsage(commands.keys())
3346 sys.exit(2)
3348 cmdName = sys.argv[1]
3349 try:
3350 klass = commands[cmdName]
3351 cmd = klass()
3352 except KeyError:
3353 print "unknown command %s" % cmdName
3354 print ""
3355 printUsage(commands.keys())
3356 sys.exit(2)
3358 options = cmd.options
3359 cmd.gitdir = os.environ.get("GIT_DIR", None)
3361 args = sys.argv[2:]
3363 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3364 if cmd.needsGit:
3365 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3367 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3368 options,
3369 description = cmd.description,
3370 formatter = HelpFormatter())
3372 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3373 global verbose
3374 verbose = cmd.verbose
3375 if cmd.needsGit:
3376 if cmd.gitdir == None:
3377 cmd.gitdir = os.path.abspath(".git")
3378 if not isValidGitDir(cmd.gitdir):
3379 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3380 if os.path.exists(cmd.gitdir):
3381 cdup = read_pipe("git rev-parse --show-cdup").strip()
3382 if len(cdup) > 0:
3383 chdir(cdup);
3385 if not isValidGitDir(cmd.gitdir):
3386 if isValidGitDir(cmd.gitdir + "/.git"):
3387 cmd.gitdir += "/.git"
3388 else:
3389 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3391 os.environ["GIT_DIR"] = cmd.gitdir
3393 if not cmd.run(args):
3394 parser.print_help()
3395 sys.exit(2)
3398 if __name__ == '__main__':
3399 main()