tgupdate: merge girocco base into girocco
[girocco-gitweb.git] / git-p4.py
blobf427bf6299ed3239ce4e73b674dbb2273dc5e086
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
45 verbose = False
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
83 if isinstance(cmd,basestring):
84 real_cmd = ' '.join(real_cmd) + ' ' + cmd
85 else:
86 real_cmd += cmd
87 return real_cmd
89 def chdir(path, is_client_path=False):
90 """Do chdir to the given path, and set the PWD environment
91 variable for use by P4. It does not look at getcwd() output.
92 Since we're not using the shell, it is necessary to set the
93 PWD environment variable explicitly.
95 Normally, expand the path to force it to be absolute. This
96 addresses the use of relative path names inside P4 settings,
97 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
98 as given; it looks for .p4config using PWD.
100 If is_client_path, the path was handed to us directly by p4,
101 and may be a symbolic link. Do not call os.getcwd() in this
102 case, because it will cause p4 to think that PWD is not inside
103 the client path.
106 os.chdir(path)
107 if not is_client_path:
108 path = os.getcwd()
109 os.environ['PWD'] = path
111 def calcDiskFree():
112 """Return free space in bytes on the disk of the given dirname."""
113 if platform.system() == 'Windows':
114 free_bytes = ctypes.c_ulonglong(0)
115 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
116 return free_bytes.value
117 else:
118 st = os.statvfs(os.getcwd())
119 return st.f_bavail * st.f_frsize
121 def die(msg):
122 if verbose:
123 raise Exception(msg)
124 else:
125 sys.stderr.write(msg + "\n")
126 sys.exit(1)
128 def write_pipe(c, stdin):
129 if verbose:
130 sys.stderr.write('Writing pipe: %s\n' % str(c))
132 expand = isinstance(c,basestring)
133 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
134 pipe = p.stdin
135 val = pipe.write(stdin)
136 pipe.close()
137 if p.wait():
138 die('Command failed: %s' % str(c))
140 return val
142 def p4_write_pipe(c, stdin):
143 real_cmd = p4_build_cmd(c)
144 return write_pipe(real_cmd, stdin)
146 def read_pipe(c, ignore_error=False):
147 if verbose:
148 sys.stderr.write('Reading pipe: %s\n' % str(c))
150 expand = isinstance(c,basestring)
151 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
152 (out, err) = p.communicate()
153 if p.returncode != 0 and not ignore_error:
154 die('Command failed: %s\nError: %s' % (str(c), err))
155 return out
157 def p4_read_pipe(c, ignore_error=False):
158 real_cmd = p4_build_cmd(c)
159 return read_pipe(real_cmd, ignore_error)
161 def read_pipe_lines(c):
162 if verbose:
163 sys.stderr.write('Reading pipe: %s\n' % str(c))
165 expand = isinstance(c, basestring)
166 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
167 pipe = p.stdout
168 val = pipe.readlines()
169 if pipe.close() or p.wait():
170 die('Command failed: %s' % str(c))
172 return val
174 def p4_read_pipe_lines(c):
175 """Specifically invoke p4 on the command supplied. """
176 real_cmd = p4_build_cmd(c)
177 return read_pipe_lines(real_cmd)
179 def p4_has_command(cmd):
180 """Ask p4 for help on this command. If it returns an error, the
181 command does not exist in this version of p4."""
182 real_cmd = p4_build_cmd(["help", cmd])
183 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
184 stderr=subprocess.PIPE)
185 p.communicate()
186 return p.returncode == 0
188 def p4_has_move_command():
189 """See if the move command exists, that it supports -k, and that
190 it has not been administratively disabled. The arguments
191 must be correct, but the filenames do not have to exist. Use
192 ones with wildcards so even if they exist, it will fail."""
194 if not p4_has_command("move"):
195 return False
196 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
197 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
198 (out, err) = p.communicate()
199 # return code will be 1 in either case
200 if err.find("Invalid option") >= 0:
201 return False
202 if err.find("disabled") >= 0:
203 return False
204 # assume it failed because @... was invalid changelist
205 return True
207 def system(cmd, ignore_error=False):
208 expand = isinstance(cmd,basestring)
209 if verbose:
210 sys.stderr.write("executing %s\n" % str(cmd))
211 retcode = subprocess.call(cmd, shell=expand)
212 if retcode and not ignore_error:
213 raise CalledProcessError(retcode, cmd)
215 return retcode
217 def p4_system(cmd):
218 """Specifically invoke p4 as the system command. """
219 real_cmd = p4_build_cmd(cmd)
220 expand = isinstance(real_cmd, basestring)
221 retcode = subprocess.call(real_cmd, shell=expand)
222 if retcode:
223 raise CalledProcessError(retcode, real_cmd)
225 _p4_version_string = None
226 def p4_version_string():
227 """Read the version string, showing just the last line, which
228 hopefully is the interesting version bit.
230 $ p4 -V
231 Perforce - The Fast Software Configuration Management System.
232 Copyright 1995-2011 Perforce Software. All rights reserved.
233 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
235 global _p4_version_string
236 if not _p4_version_string:
237 a = p4_read_pipe_lines(["-V"])
238 _p4_version_string = a[-1].rstrip()
239 return _p4_version_string
241 def p4_integrate(src, dest):
242 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
244 def p4_sync(f, *options):
245 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
247 def p4_add(f):
248 # forcibly add file names with wildcards
249 if wildcard_present(f):
250 p4_system(["add", "-f", f])
251 else:
252 p4_system(["add", f])
254 def p4_delete(f):
255 p4_system(["delete", wildcard_encode(f)])
257 def p4_edit(f, *options):
258 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
260 def p4_revert(f):
261 p4_system(["revert", wildcard_encode(f)])
263 def p4_reopen(type, f):
264 p4_system(["reopen", "-t", type, wildcard_encode(f)])
266 def p4_move(src, dest):
267 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
269 def p4_last_change():
270 results = p4CmdList(["changes", "-m", "1"])
271 return int(results[0]['change'])
273 def p4_describe(change):
274 """Make sure it returns a valid result by checking for
275 the presence of field "time". Return a dict of the
276 results."""
278 ds = p4CmdList(["describe", "-s", str(change)])
279 if len(ds) != 1:
280 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
282 d = ds[0]
284 if "p4ExitCode" in d:
285 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
286 str(d)))
287 if "code" in d:
288 if d["code"] == "error":
289 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
291 if "time" not in d:
292 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
294 return d
297 # Canonicalize the p4 type and return a tuple of the
298 # base type, plus any modifiers. See "p4 help filetypes"
299 # for a list and explanation.
301 def split_p4_type(p4type):
303 p4_filetypes_historical = {
304 "ctempobj": "binary+Sw",
305 "ctext": "text+C",
306 "cxtext": "text+Cx",
307 "ktext": "text+k",
308 "kxtext": "text+kx",
309 "ltext": "text+F",
310 "tempobj": "binary+FSw",
311 "ubinary": "binary+F",
312 "uresource": "resource+F",
313 "uxbinary": "binary+Fx",
314 "xbinary": "binary+x",
315 "xltext": "text+Fx",
316 "xtempobj": "binary+Swx",
317 "xtext": "text+x",
318 "xunicode": "unicode+x",
319 "xutf16": "utf16+x",
321 if p4type in p4_filetypes_historical:
322 p4type = p4_filetypes_historical[p4type]
323 mods = ""
324 s = p4type.split("+")
325 base = s[0]
326 mods = ""
327 if len(s) > 1:
328 mods = s[1]
329 return (base, mods)
332 # return the raw p4 type of a file (text, text+ko, etc)
334 def p4_type(f):
335 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
336 return results[0]['headType']
339 # Given a type base and modifier, return a regexp matching
340 # the keywords that can be expanded in the file
342 def p4_keywords_regexp_for_type(base, type_mods):
343 if base in ("text", "unicode", "binary"):
344 kwords = None
345 if "ko" in type_mods:
346 kwords = 'Id|Header'
347 elif "k" in type_mods:
348 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
349 else:
350 return None
351 pattern = r"""
352 \$ # Starts with a dollar, followed by...
353 (%s) # one of the keywords, followed by...
354 (:[^$\n]+)? # possibly an old expansion, followed by...
355 \$ # another dollar
356 """ % kwords
357 return pattern
358 else:
359 return None
362 # Given a file, return a regexp matching the possible
363 # RCS keywords that will be expanded, or None for files
364 # with kw expansion turned off.
366 def p4_keywords_regexp_for_file(file):
367 if not os.path.exists(file):
368 return None
369 else:
370 (type_base, type_mods) = split_p4_type(p4_type(file))
371 return p4_keywords_regexp_for_type(type_base, type_mods)
373 def setP4ExecBit(file, mode):
374 # Reopens an already open file and changes the execute bit to match
375 # the execute bit setting in the passed in mode.
377 p4Type = "+x"
379 if not isModeExec(mode):
380 p4Type = getP4OpenedType(file)
381 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
382 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
383 if p4Type[-1] == "+":
384 p4Type = p4Type[0:-1]
386 p4_reopen(p4Type, file)
388 def getP4OpenedType(file):
389 # Returns the perforce file type for the given file.
391 result = p4_read_pipe(["opened", wildcard_encode(file)])
392 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
393 if match:
394 return match.group(1)
395 else:
396 die("Could not determine file type for %s (result: '%s')" % (file, result))
398 # Return the set of all p4 labels
399 def getP4Labels(depotPaths):
400 labels = set()
401 if isinstance(depotPaths,basestring):
402 depotPaths = [depotPaths]
404 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
405 label = l['label']
406 labels.add(label)
408 return labels
410 # Return the set of all git tags
411 def getGitTags():
412 gitTags = set()
413 for line in read_pipe_lines(["git", "tag"]):
414 tag = line.strip()
415 gitTags.add(tag)
416 return gitTags
418 def diffTreePattern():
419 # This is a simple generator for the diff tree regex pattern. This could be
420 # a class variable if this and parseDiffTreeEntry were a part of a class.
421 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
422 while True:
423 yield pattern
425 def parseDiffTreeEntry(entry):
426 """Parses a single diff tree entry into its component elements.
428 See git-diff-tree(1) manpage for details about the format of the diff
429 output. This method returns a dictionary with the following elements:
431 src_mode - The mode of the source file
432 dst_mode - The mode of the destination file
433 src_sha1 - The sha1 for the source file
434 dst_sha1 - The sha1 fr the destination file
435 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
436 status_score - The score for the status (applicable for 'C' and 'R'
437 statuses). This is None if there is no score.
438 src - The path for the source file.
439 dst - The path for the destination file. This is only present for
440 copy or renames. If it is not present, this is None.
442 If the pattern is not matched, None is returned."""
444 match = diffTreePattern().next().match(entry)
445 if match:
446 return {
447 'src_mode': match.group(1),
448 'dst_mode': match.group(2),
449 'src_sha1': match.group(3),
450 'dst_sha1': match.group(4),
451 'status': match.group(5),
452 'status_score': match.group(6),
453 'src': match.group(7),
454 'dst': match.group(10)
456 return None
458 def isModeExec(mode):
459 # Returns True if the given git mode represents an executable file,
460 # otherwise False.
461 return mode[-3:] == "755"
463 def isModeExecChanged(src_mode, dst_mode):
464 return isModeExec(src_mode) != isModeExec(dst_mode)
466 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
468 if isinstance(cmd,basestring):
469 cmd = "-G " + cmd
470 expand = True
471 else:
472 cmd = ["-G"] + cmd
473 expand = False
475 cmd = p4_build_cmd(cmd)
476 if verbose:
477 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
479 # Use a temporary file to avoid deadlocks without
480 # subprocess.communicate(), which would put another copy
481 # of stdout into memory.
482 stdin_file = None
483 if stdin is not None:
484 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
485 if isinstance(stdin,basestring):
486 stdin_file.write(stdin)
487 else:
488 for i in stdin:
489 stdin_file.write(i + '\n')
490 stdin_file.flush()
491 stdin_file.seek(0)
493 p4 = subprocess.Popen(cmd,
494 shell=expand,
495 stdin=stdin_file,
496 stdout=subprocess.PIPE)
498 result = []
499 try:
500 while True:
501 entry = marshal.load(p4.stdout)
502 if cb is not None:
503 cb(entry)
504 else:
505 result.append(entry)
506 except EOFError:
507 pass
508 exitCode = p4.wait()
509 if exitCode != 0:
510 entry = {}
511 entry["p4ExitCode"] = exitCode
512 result.append(entry)
514 return result
516 def p4Cmd(cmd):
517 list = p4CmdList(cmd)
518 result = {}
519 for entry in list:
520 result.update(entry)
521 return result;
523 def p4Where(depotPath):
524 if not depotPath.endswith("/"):
525 depotPath += "/"
526 depotPathLong = depotPath + "..."
527 outputList = p4CmdList(["where", depotPathLong])
528 output = None
529 for entry in outputList:
530 if "depotFile" in entry:
531 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
532 # The base path always ends with "/...".
533 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
534 output = entry
535 break
536 elif "data" in entry:
537 data = entry.get("data")
538 space = data.find(" ")
539 if data[:space] == depotPath:
540 output = entry
541 break
542 if output == None:
543 return ""
544 if output["code"] == "error":
545 return ""
546 clientPath = ""
547 if "path" in output:
548 clientPath = output.get("path")
549 elif "data" in output:
550 data = output.get("data")
551 lastSpace = data.rfind(" ")
552 clientPath = data[lastSpace + 1:]
554 if clientPath.endswith("..."):
555 clientPath = clientPath[:-3]
556 return clientPath
558 def currentGitBranch():
559 retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
560 if retcode != 0:
561 # on a detached head
562 return None
563 else:
564 return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
566 def isValidGitDir(path):
567 if (os.path.exists(path + "/HEAD")
568 and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
569 return True;
570 return False
572 def parseRevision(ref):
573 return read_pipe("git rev-parse %s" % ref).strip()
575 def branchExists(ref):
576 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
577 ignore_error=True)
578 return len(rev) > 0
580 def extractLogMessageFromGitCommit(commit):
581 logMessage = ""
583 ## fixme: title is first line of commit, not 1st paragraph.
584 foundTitle = False
585 for log in read_pipe_lines("git cat-file commit %s" % commit):
586 if not foundTitle:
587 if len(log) == 1:
588 foundTitle = True
589 continue
591 logMessage += log
592 return logMessage
594 def extractSettingsGitLog(log):
595 values = {}
596 for line in log.split("\n"):
597 line = line.strip()
598 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
599 if not m:
600 continue
602 assignments = m.group(1).split (':')
603 for a in assignments:
604 vals = a.split ('=')
605 key = vals[0].strip()
606 val = ('='.join (vals[1:])).strip()
607 if val.endswith ('\"') and val.startswith('"'):
608 val = val[1:-1]
610 values[key] = val
612 paths = values.get("depot-paths")
613 if not paths:
614 paths = values.get("depot-path")
615 if paths:
616 values['depot-paths'] = paths.split(',')
617 return values
619 def gitBranchExists(branch):
620 proc = subprocess.Popen(["git", "rev-parse", branch],
621 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
622 return proc.wait() == 0;
624 _gitConfig = {}
626 def gitConfig(key, typeSpecifier=None):
627 if not _gitConfig.has_key(key):
628 cmd = [ "git", "config" ]
629 if typeSpecifier:
630 cmd += [ typeSpecifier ]
631 cmd += [ key ]
632 s = read_pipe(cmd, ignore_error=True)
633 _gitConfig[key] = s.strip()
634 return _gitConfig[key]
636 def gitConfigBool(key):
637 """Return a bool, using git config --bool. It is True only if the
638 variable is set to true, and False if set to false or not present
639 in the config."""
641 if not _gitConfig.has_key(key):
642 _gitConfig[key] = gitConfig(key, '--bool') == "true"
643 return _gitConfig[key]
645 def gitConfigInt(key):
646 if not _gitConfig.has_key(key):
647 cmd = [ "git", "config", "--int", key ]
648 s = read_pipe(cmd, ignore_error=True)
649 v = s.strip()
650 try:
651 _gitConfig[key] = int(gitConfig(key, '--int'))
652 except ValueError:
653 _gitConfig[key] = None
654 return _gitConfig[key]
656 def gitConfigList(key):
657 if not _gitConfig.has_key(key):
658 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
659 _gitConfig[key] = s.strip().split(os.linesep)
660 if _gitConfig[key] == ['']:
661 _gitConfig[key] = []
662 return _gitConfig[key]
664 def p4BranchesInGit(branchesAreInRemotes=True):
665 """Find all the branches whose names start with "p4/", looking
666 in remotes or heads as specified by the argument. Return
667 a dictionary of { branch: revision } for each one found.
668 The branch names are the short names, without any
669 "p4/" prefix."""
671 branches = {}
673 cmdline = "git rev-parse --symbolic "
674 if branchesAreInRemotes:
675 cmdline += "--remotes"
676 else:
677 cmdline += "--branches"
679 for line in read_pipe_lines(cmdline):
680 line = line.strip()
682 # only import to p4/
683 if not line.startswith('p4/'):
684 continue
685 # special symbolic ref to p4/master
686 if line == "p4/HEAD":
687 continue
689 # strip off p4/ prefix
690 branch = line[len("p4/"):]
692 branches[branch] = parseRevision(line)
694 return branches
696 def branch_exists(branch):
697 """Make sure that the given ref name really exists."""
699 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
700 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
701 out, _ = p.communicate()
702 if p.returncode:
703 return False
704 # expect exactly one line of output: the branch name
705 return out.rstrip() == branch
707 def findUpstreamBranchPoint(head = "HEAD"):
708 branches = p4BranchesInGit()
709 # map from depot-path to branch name
710 branchByDepotPath = {}
711 for branch in branches.keys():
712 tip = branches[branch]
713 log = extractLogMessageFromGitCommit(tip)
714 settings = extractSettingsGitLog(log)
715 if settings.has_key("depot-paths"):
716 paths = ",".join(settings["depot-paths"])
717 branchByDepotPath[paths] = "remotes/p4/" + branch
719 settings = None
720 parent = 0
721 while parent < 65535:
722 commit = head + "~%s" % parent
723 log = extractLogMessageFromGitCommit(commit)
724 settings = extractSettingsGitLog(log)
725 if settings.has_key("depot-paths"):
726 paths = ",".join(settings["depot-paths"])
727 if branchByDepotPath.has_key(paths):
728 return [branchByDepotPath[paths], settings]
730 parent = parent + 1
732 return ["", settings]
734 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
735 if not silent:
736 print ("Creating/updating branch(es) in %s based on origin branch(es)"
737 % localRefPrefix)
739 originPrefix = "origin/p4/"
741 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
742 line = line.strip()
743 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
744 continue
746 headName = line[len(originPrefix):]
747 remoteHead = localRefPrefix + headName
748 originHead = line
750 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
751 if (not original.has_key('depot-paths')
752 or not original.has_key('change')):
753 continue
755 update = False
756 if not gitBranchExists(remoteHead):
757 if verbose:
758 print "creating %s" % remoteHead
759 update = True
760 else:
761 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
762 if settings.has_key('change') > 0:
763 if settings['depot-paths'] == original['depot-paths']:
764 originP4Change = int(original['change'])
765 p4Change = int(settings['change'])
766 if originP4Change > p4Change:
767 print ("%s (%s) is newer than %s (%s). "
768 "Updating p4 branch from origin."
769 % (originHead, originP4Change,
770 remoteHead, p4Change))
771 update = True
772 else:
773 print ("Ignoring: %s was imported from %s while "
774 "%s was imported from %s"
775 % (originHead, ','.join(original['depot-paths']),
776 remoteHead, ','.join(settings['depot-paths'])))
778 if update:
779 system("git update-ref %s %s" % (remoteHead, originHead))
781 def originP4BranchesExist():
782 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
785 def p4ParseNumericChangeRange(parts):
786 changeStart = int(parts[0][1:])
787 if parts[1] == '#head':
788 changeEnd = p4_last_change()
789 else:
790 changeEnd = int(parts[1])
792 return (changeStart, changeEnd)
794 def chooseBlockSize(blockSize):
795 if blockSize:
796 return blockSize
797 else:
798 return defaultBlockSize
800 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
801 assert depotPaths
803 # Parse the change range into start and end. Try to find integer
804 # revision ranges as these can be broken up into blocks to avoid
805 # hitting server-side limits (maxrows, maxscanresults). But if
806 # that doesn't work, fall back to using the raw revision specifier
807 # strings, without using block mode.
809 if changeRange is None or changeRange == '':
810 changeStart = 1
811 changeEnd = p4_last_change()
812 block_size = chooseBlockSize(requestedBlockSize)
813 else:
814 parts = changeRange.split(',')
815 assert len(parts) == 2
816 try:
817 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
818 block_size = chooseBlockSize(requestedBlockSize)
819 except:
820 changeStart = parts[0][1:]
821 changeEnd = parts[1]
822 if requestedBlockSize:
823 die("cannot use --changes-block-size with non-numeric revisions")
824 block_size = None
826 changes = set()
828 # Retrieve changes a block at a time, to prevent running
829 # into a MaxResults/MaxScanRows error from the server.
831 while True:
832 cmd = ['changes']
834 if block_size:
835 end = min(changeEnd, changeStart + block_size)
836 revisionRange = "%d,%d" % (changeStart, end)
837 else:
838 revisionRange = "%s,%s" % (changeStart, changeEnd)
840 for p in depotPaths:
841 cmd += ["%s...@%s" % (p, revisionRange)]
843 # Insert changes in chronological order
844 for line in reversed(p4_read_pipe_lines(cmd)):
845 changes.add(int(line.split(" ")[1]))
847 if not block_size:
848 break
850 if end >= changeEnd:
851 break
853 changeStart = end + 1
855 changes = sorted(changes)
856 return changes
858 def p4PathStartsWith(path, prefix):
859 # This method tries to remedy a potential mixed-case issue:
861 # If UserA adds //depot/DirA/file1
862 # and UserB adds //depot/dira/file2
864 # we may or may not have a problem. If you have core.ignorecase=true,
865 # we treat DirA and dira as the same directory
866 if gitConfigBool("core.ignorecase"):
867 return path.lower().startswith(prefix.lower())
868 return path.startswith(prefix)
870 def getClientSpec():
871 """Look at the p4 client spec, create a View() object that contains
872 all the mappings, and return it."""
874 specList = p4CmdList("client -o")
875 if len(specList) != 1:
876 die('Output from "client -o" is %d lines, expecting 1' %
877 len(specList))
879 # dictionary of all client parameters
880 entry = specList[0]
882 # the //client/ name
883 client_name = entry["Client"]
885 # just the keys that start with "View"
886 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
888 # hold this new View
889 view = View(client_name)
891 # append the lines, in order, to the view
892 for view_num in range(len(view_keys)):
893 k = "View%d" % view_num
894 if k not in view_keys:
895 die("Expected view key %s missing" % k)
896 view.append(entry[k])
898 return view
900 def getClientRoot():
901 """Grab the client directory."""
903 output = p4CmdList("client -o")
904 if len(output) != 1:
905 die('Output from "client -o" is %d lines, expecting 1' % len(output))
907 entry = output[0]
908 if "Root" not in entry:
909 die('Client has no "Root"')
911 return entry["Root"]
914 # P4 wildcards are not allowed in filenames. P4 complains
915 # if you simply add them, but you can force it with "-f", in
916 # which case it translates them into %xx encoding internally.
918 def wildcard_decode(path):
919 # Search for and fix just these four characters. Do % last so
920 # that fixing it does not inadvertently create new %-escapes.
921 # Cannot have * in a filename in windows; untested as to
922 # what p4 would do in such a case.
923 if not platform.system() == "Windows":
924 path = path.replace("%2A", "*")
925 path = path.replace("%23", "#") \
926 .replace("%40", "@") \
927 .replace("%25", "%")
928 return path
930 def wildcard_encode(path):
931 # do % first to avoid double-encoding the %s introduced here
932 path = path.replace("%", "%25") \
933 .replace("*", "%2A") \
934 .replace("#", "%23") \
935 .replace("@", "%40")
936 return path
938 def wildcard_present(path):
939 m = re.search("[*#@%]", path)
940 return m is not None
942 class LargeFileSystem(object):
943 """Base class for large file system support."""
945 def __init__(self, writeToGitStream):
946 self.largeFiles = set()
947 self.writeToGitStream = writeToGitStream
949 def generatePointer(self, cloneDestination, contentFile):
950 """Return the content of a pointer file that is stored in Git instead of
951 the actual content."""
952 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
954 def pushFile(self, localLargeFile):
955 """Push the actual content which is not stored in the Git repository to
956 a server."""
957 assert False, "Method 'pushFile' required in " + self.__class__.__name__
959 def hasLargeFileExtension(self, relPath):
960 return reduce(
961 lambda a, b: a or b,
962 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
963 False
966 def generateTempFile(self, contents):
967 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
968 for d in contents:
969 contentFile.write(d)
970 contentFile.close()
971 return contentFile.name
973 def exceedsLargeFileThreshold(self, relPath, contents):
974 if gitConfigInt('git-p4.largeFileThreshold'):
975 contentsSize = sum(len(d) for d in contents)
976 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
977 return True
978 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
979 contentsSize = sum(len(d) for d in contents)
980 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
981 return False
982 contentTempFile = self.generateTempFile(contents)
983 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
984 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
985 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
986 zf.close()
987 compressedContentsSize = zf.infolist()[0].compress_size
988 os.remove(contentTempFile)
989 os.remove(compressedContentFile.name)
990 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
991 return True
992 return False
994 def addLargeFile(self, relPath):
995 self.largeFiles.add(relPath)
997 def removeLargeFile(self, relPath):
998 self.largeFiles.remove(relPath)
1000 def isLargeFile(self, relPath):
1001 return relPath in self.largeFiles
1003 def processContent(self, git_mode, relPath, contents):
1004 """Processes the content of git fast import. This method decides if a
1005 file is stored in the large file system and handles all necessary
1006 steps."""
1007 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1008 contentTempFile = self.generateTempFile(contents)
1009 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1010 if pointer_git_mode:
1011 git_mode = pointer_git_mode
1012 if localLargeFile:
1013 # Move temp file to final location in large file system
1014 largeFileDir = os.path.dirname(localLargeFile)
1015 if not os.path.isdir(largeFileDir):
1016 os.makedirs(largeFileDir)
1017 shutil.move(contentTempFile, localLargeFile)
1018 self.addLargeFile(relPath)
1019 if gitConfigBool('git-p4.largeFilePush'):
1020 self.pushFile(localLargeFile)
1021 if verbose:
1022 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1023 return (git_mode, contents)
1025 class MockLFS(LargeFileSystem):
1026 """Mock large file system for testing."""
1028 def generatePointer(self, contentFile):
1029 """The pointer content is the original content prefixed with "pointer-".
1030 The local filename of the large file storage is derived from the file content.
1032 with open(contentFile, 'r') as f:
1033 content = next(f)
1034 gitMode = '100644'
1035 pointerContents = 'pointer-' + content
1036 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1037 return (gitMode, pointerContents, localLargeFile)
1039 def pushFile(self, localLargeFile):
1040 """The remote filename of the large file storage is the same as the local
1041 one but in a different directory.
1043 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1044 if not os.path.exists(remotePath):
1045 os.makedirs(remotePath)
1046 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1048 class GitLFS(LargeFileSystem):
1049 """Git LFS as backend for the git-p4 large file system.
1050 See https://git-lfs.github.com/ for details."""
1052 def __init__(self, *args):
1053 LargeFileSystem.__init__(self, *args)
1054 self.baseGitAttributes = []
1056 def generatePointer(self, contentFile):
1057 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1058 mode and content which is stored in the Git repository instead of
1059 the actual content. Return also the new location of the actual
1060 content.
1062 if os.path.getsize(contentFile) == 0:
1063 return (None, '', None)
1065 pointerProcess = subprocess.Popen(
1066 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1067 stdout=subprocess.PIPE
1069 pointerFile = pointerProcess.stdout.read()
1070 if pointerProcess.wait():
1071 os.remove(contentFile)
1072 die('git-lfs pointer command failed. Did you install the extension?')
1074 # Git LFS removed the preamble in the output of the 'pointer' command
1075 # starting from version 1.2.0. Check for the preamble here to support
1076 # earlier versions.
1077 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1078 if pointerFile.startswith('Git LFS pointer for'):
1079 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1081 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1082 localLargeFile = os.path.join(
1083 os.getcwd(),
1084 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1085 oid,
1087 # LFS Spec states that pointer files should not have the executable bit set.
1088 gitMode = '100644'
1089 return (gitMode, pointerFile, localLargeFile)
1091 def pushFile(self, localLargeFile):
1092 uploadProcess = subprocess.Popen(
1093 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1095 if uploadProcess.wait():
1096 die('git-lfs push command failed. Did you define a remote?')
1098 def generateGitAttributes(self):
1099 return (
1100 self.baseGitAttributes +
1102 '\n',
1103 '#\n',
1104 '# Git LFS (see https://git-lfs.github.com/)\n',
1105 '#\n',
1107 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1108 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1110 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1111 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1115 def addLargeFile(self, relPath):
1116 LargeFileSystem.addLargeFile(self, relPath)
1117 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1119 def removeLargeFile(self, relPath):
1120 LargeFileSystem.removeLargeFile(self, relPath)
1121 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1123 def processContent(self, git_mode, relPath, contents):
1124 if relPath == '.gitattributes':
1125 self.baseGitAttributes = contents
1126 return (git_mode, self.generateGitAttributes())
1127 else:
1128 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1130 class Command:
1131 def __init__(self):
1132 self.usage = "usage: %prog [options]"
1133 self.needsGit = True
1134 self.verbose = False
1136 class P4UserMap:
1137 def __init__(self):
1138 self.userMapFromPerforceServer = False
1139 self.myP4UserId = None
1141 def p4UserId(self):
1142 if self.myP4UserId:
1143 return self.myP4UserId
1145 results = p4CmdList("user -o")
1146 for r in results:
1147 if r.has_key('User'):
1148 self.myP4UserId = r['User']
1149 return r['User']
1150 die("Could not find your p4 user id")
1152 def p4UserIsMe(self, p4User):
1153 # return True if the given p4 user is actually me
1154 me = self.p4UserId()
1155 if not p4User or p4User != me:
1156 return False
1157 else:
1158 return True
1160 def getUserCacheFilename(self):
1161 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1162 return home + "/.gitp4-usercache.txt"
1164 def getUserMapFromPerforceServer(self):
1165 if self.userMapFromPerforceServer:
1166 return
1167 self.users = {}
1168 self.emails = {}
1170 for output in p4CmdList("users"):
1171 if not output.has_key("User"):
1172 continue
1173 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1174 self.emails[output["Email"]] = output["User"]
1176 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1177 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1178 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1179 if mapUser and len(mapUser[0]) == 3:
1180 user = mapUser[0][0]
1181 fullname = mapUser[0][1]
1182 email = mapUser[0][2]
1183 self.users[user] = fullname + " <" + email + ">"
1184 self.emails[email] = user
1186 s = ''
1187 for (key, val) in self.users.items():
1188 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1190 open(self.getUserCacheFilename(), "wb").write(s)
1191 self.userMapFromPerforceServer = True
1193 def loadUserMapFromCache(self):
1194 self.users = {}
1195 self.userMapFromPerforceServer = False
1196 try:
1197 cache = open(self.getUserCacheFilename(), "rb")
1198 lines = cache.readlines()
1199 cache.close()
1200 for line in lines:
1201 entry = line.strip().split("\t")
1202 self.users[entry[0]] = entry[1]
1203 except IOError:
1204 self.getUserMapFromPerforceServer()
1206 class P4Debug(Command):
1207 def __init__(self):
1208 Command.__init__(self)
1209 self.options = []
1210 self.description = "A tool to debug the output of p4 -G."
1211 self.needsGit = False
1213 def run(self, args):
1214 j = 0
1215 for output in p4CmdList(args):
1216 print 'Element: %d' % j
1217 j += 1
1218 print output
1219 return True
1221 class P4RollBack(Command):
1222 def __init__(self):
1223 Command.__init__(self)
1224 self.options = [
1225 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1227 self.description = "A tool to debug the multi-branch import. Don't use :)"
1228 self.rollbackLocalBranches = False
1230 def run(self, args):
1231 if len(args) != 1:
1232 return False
1233 maxChange = int(args[0])
1235 if "p4ExitCode" in p4Cmd("changes -m 1"):
1236 die("Problems executing p4");
1238 if self.rollbackLocalBranches:
1239 refPrefix = "refs/heads/"
1240 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1241 else:
1242 refPrefix = "refs/remotes/"
1243 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1245 for line in lines:
1246 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1247 line = line.strip()
1248 ref = refPrefix + line
1249 log = extractLogMessageFromGitCommit(ref)
1250 settings = extractSettingsGitLog(log)
1252 depotPaths = settings['depot-paths']
1253 change = settings['change']
1255 changed = False
1257 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1258 for p in depotPaths]))) == 0:
1259 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1260 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1261 continue
1263 while change and int(change) > maxChange:
1264 changed = True
1265 if self.verbose:
1266 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1267 system("git update-ref %s \"%s^\"" % (ref, ref))
1268 log = extractLogMessageFromGitCommit(ref)
1269 settings = extractSettingsGitLog(log)
1272 depotPaths = settings['depot-paths']
1273 change = settings['change']
1275 if changed:
1276 print "%s rewound to %s" % (ref, change)
1278 return True
1280 class P4Submit(Command, P4UserMap):
1282 conflict_behavior_choices = ("ask", "skip", "quit")
1284 def __init__(self):
1285 Command.__init__(self)
1286 P4UserMap.__init__(self)
1287 self.options = [
1288 optparse.make_option("--origin", dest="origin"),
1289 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1290 # preserve the user, requires relevant p4 permissions
1291 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1292 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1293 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1294 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1295 optparse.make_option("--conflict", dest="conflict_behavior",
1296 choices=self.conflict_behavior_choices),
1297 optparse.make_option("--branch", dest="branch"),
1299 self.description = "Submit changes from git to the perforce depot."
1300 self.usage += " [name of git branch to submit into perforce depot]"
1301 self.origin = ""
1302 self.detectRenames = False
1303 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1304 self.dry_run = False
1305 self.prepare_p4_only = False
1306 self.conflict_behavior = None
1307 self.isWindows = (platform.system() == "Windows")
1308 self.exportLabels = False
1309 self.p4HasMoveCommand = p4_has_move_command()
1310 self.branch = None
1312 if gitConfig('git-p4.largeFileSystem'):
1313 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1315 def check(self):
1316 if len(p4CmdList("opened ...")) > 0:
1317 die("You have files opened with perforce! Close them before starting the sync.")
1319 def separate_jobs_from_description(self, message):
1320 """Extract and return a possible Jobs field in the commit
1321 message. It goes into a separate section in the p4 change
1322 specification.
1324 A jobs line starts with "Jobs:" and looks like a new field
1325 in a form. Values are white-space separated on the same
1326 line or on following lines that start with a tab.
1328 This does not parse and extract the full git commit message
1329 like a p4 form. It just sees the Jobs: line as a marker
1330 to pass everything from then on directly into the p4 form,
1331 but outside the description section.
1333 Return a tuple (stripped log message, jobs string)."""
1335 m = re.search(r'^Jobs:', message, re.MULTILINE)
1336 if m is None:
1337 return (message, None)
1339 jobtext = message[m.start():]
1340 stripped_message = message[:m.start()].rstrip()
1341 return (stripped_message, jobtext)
1343 def prepareLogMessage(self, template, message, jobs):
1344 """Edits the template returned from "p4 change -o" to insert
1345 the message in the Description field, and the jobs text in
1346 the Jobs field."""
1347 result = ""
1349 inDescriptionSection = False
1351 for line in template.split("\n"):
1352 if line.startswith("#"):
1353 result += line + "\n"
1354 continue
1356 if inDescriptionSection:
1357 if line.startswith("Files:") or line.startswith("Jobs:"):
1358 inDescriptionSection = False
1359 # insert Jobs section
1360 if jobs:
1361 result += jobs + "\n"
1362 else:
1363 continue
1364 else:
1365 if line.startswith("Description:"):
1366 inDescriptionSection = True
1367 line += "\n"
1368 for messageLine in message.split("\n"):
1369 line += "\t" + messageLine + "\n"
1371 result += line + "\n"
1373 return result
1375 def patchRCSKeywords(self, file, pattern):
1376 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1377 (handle, outFileName) = tempfile.mkstemp(dir='.')
1378 try:
1379 outFile = os.fdopen(handle, "w+")
1380 inFile = open(file, "r")
1381 regexp = re.compile(pattern, re.VERBOSE)
1382 for line in inFile.readlines():
1383 line = regexp.sub(r'$\1$', line)
1384 outFile.write(line)
1385 inFile.close()
1386 outFile.close()
1387 # Forcibly overwrite the original file
1388 os.unlink(file)
1389 shutil.move(outFileName, file)
1390 except:
1391 # cleanup our temporary file
1392 os.unlink(outFileName)
1393 print "Failed to strip RCS keywords in %s" % file
1394 raise
1396 print "Patched up RCS keywords in %s" % file
1398 def p4UserForCommit(self,id):
1399 # Return the tuple (perforce user,git email) for a given git commit id
1400 self.getUserMapFromPerforceServer()
1401 gitEmail = read_pipe(["git", "log", "--max-count=1",
1402 "--format=%ae", id])
1403 gitEmail = gitEmail.strip()
1404 if not self.emails.has_key(gitEmail):
1405 return (None,gitEmail)
1406 else:
1407 return (self.emails[gitEmail],gitEmail)
1409 def checkValidP4Users(self,commits):
1410 # check if any git authors cannot be mapped to p4 users
1411 for id in commits:
1412 (user,email) = self.p4UserForCommit(id)
1413 if not user:
1414 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1415 if gitConfigBool("git-p4.allowMissingP4Users"):
1416 print "%s" % msg
1417 else:
1418 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1420 def lastP4Changelist(self):
1421 # Get back the last changelist number submitted in this client spec. This
1422 # then gets used to patch up the username in the change. If the same
1423 # client spec is being used by multiple processes then this might go
1424 # wrong.
1425 results = p4CmdList("client -o") # find the current client
1426 client = None
1427 for r in results:
1428 if r.has_key('Client'):
1429 client = r['Client']
1430 break
1431 if not client:
1432 die("could not get client spec")
1433 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1434 for r in results:
1435 if r.has_key('change'):
1436 return r['change']
1437 die("Could not get changelist number for last submit - cannot patch up user details")
1439 def modifyChangelistUser(self, changelist, newUser):
1440 # fixup the user field of a changelist after it has been submitted.
1441 changes = p4CmdList("change -o %s" % changelist)
1442 if len(changes) != 1:
1443 die("Bad output from p4 change modifying %s to user %s" %
1444 (changelist, newUser))
1446 c = changes[0]
1447 if c['User'] == newUser: return # nothing to do
1448 c['User'] = newUser
1449 input = marshal.dumps(c)
1451 result = p4CmdList("change -f -i", stdin=input)
1452 for r in result:
1453 if r.has_key('code'):
1454 if r['code'] == 'error':
1455 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1456 if r.has_key('data'):
1457 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1458 return
1459 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1461 def canChangeChangelists(self):
1462 # check to see if we have p4 admin or super-user permissions, either of
1463 # which are required to modify changelists.
1464 results = p4CmdList(["protects", self.depotPath])
1465 for r in results:
1466 if r.has_key('perm'):
1467 if r['perm'] == 'admin':
1468 return 1
1469 if r['perm'] == 'super':
1470 return 1
1471 return 0
1473 def prepareSubmitTemplate(self):
1474 """Run "p4 change -o" to grab a change specification template.
1475 This does not use "p4 -G", as it is nice to keep the submission
1476 template in original order, since a human might edit it.
1478 Remove lines in the Files section that show changes to files
1479 outside the depot path we're committing into."""
1481 [upstream, settings] = findUpstreamBranchPoint()
1483 template = ""
1484 inFilesSection = False
1485 for line in p4_read_pipe_lines(['change', '-o']):
1486 if line.endswith("\r\n"):
1487 line = line[:-2] + "\n"
1488 if inFilesSection:
1489 if line.startswith("\t"):
1490 # path starts and ends with a tab
1491 path = line[1:]
1492 lastTab = path.rfind("\t")
1493 if lastTab != -1:
1494 path = path[:lastTab]
1495 if settings.has_key('depot-paths'):
1496 if not [p for p in settings['depot-paths']
1497 if p4PathStartsWith(path, p)]:
1498 continue
1499 else:
1500 if not p4PathStartsWith(path, self.depotPath):
1501 continue
1502 else:
1503 inFilesSection = False
1504 else:
1505 if line.startswith("Files:"):
1506 inFilesSection = True
1508 template += line
1510 return template
1512 def edit_template(self, template_file):
1513 """Invoke the editor to let the user change the submission
1514 message. Return true if okay to continue with the submit."""
1516 # if configured to skip the editing part, just submit
1517 if gitConfigBool("git-p4.skipSubmitEdit"):
1518 return True
1520 # look at the modification time, to check later if the user saved
1521 # the file
1522 mtime = os.stat(template_file).st_mtime
1524 # invoke the editor
1525 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1526 editor = os.environ.get("P4EDITOR")
1527 else:
1528 editor = read_pipe("git var GIT_EDITOR").strip()
1529 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1531 # If the file was not saved, prompt to see if this patch should
1532 # be skipped. But skip this verification step if configured so.
1533 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1534 return True
1536 # modification time updated means user saved the file
1537 if os.stat(template_file).st_mtime > mtime:
1538 return True
1540 while True:
1541 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1542 if response == 'y':
1543 return True
1544 if response == 'n':
1545 return False
1547 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1548 # diff
1549 if os.environ.has_key("P4DIFF"):
1550 del(os.environ["P4DIFF"])
1551 diff = ""
1552 for editedFile in editedFiles:
1553 diff += p4_read_pipe(['diff', '-du',
1554 wildcard_encode(editedFile)])
1556 # new file diff
1557 newdiff = ""
1558 for newFile in filesToAdd:
1559 newdiff += "==== new file ====\n"
1560 newdiff += "--- /dev/null\n"
1561 newdiff += "+++ %s\n" % newFile
1563 is_link = os.path.islink(newFile)
1564 expect_link = newFile in symlinks
1566 if is_link and expect_link:
1567 newdiff += "+%s\n" % os.readlink(newFile)
1568 else:
1569 f = open(newFile, "r")
1570 for line in f.readlines():
1571 newdiff += "+" + line
1572 f.close()
1574 return (diff + newdiff).replace('\r\n', '\n')
1576 def applyCommit(self, id):
1577 """Apply one commit, return True if it succeeded."""
1579 print "Applying", read_pipe(["git", "show", "-s",
1580 "--format=format:%h %s", id])
1582 (p4User, gitEmail) = self.p4UserForCommit(id)
1584 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1585 filesToAdd = set()
1586 filesToChangeType = set()
1587 filesToDelete = set()
1588 editedFiles = set()
1589 pureRenameCopy = set()
1590 symlinks = set()
1591 filesToChangeExecBit = {}
1593 for line in diff:
1594 diff = parseDiffTreeEntry(line)
1595 modifier = diff['status']
1596 path = diff['src']
1597 if modifier == "M":
1598 p4_edit(path)
1599 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1600 filesToChangeExecBit[path] = diff['dst_mode']
1601 editedFiles.add(path)
1602 elif modifier == "A":
1603 filesToAdd.add(path)
1604 filesToChangeExecBit[path] = diff['dst_mode']
1605 if path in filesToDelete:
1606 filesToDelete.remove(path)
1608 dst_mode = int(diff['dst_mode'], 8)
1609 if dst_mode == 0120000:
1610 symlinks.add(path)
1612 elif modifier == "D":
1613 filesToDelete.add(path)
1614 if path in filesToAdd:
1615 filesToAdd.remove(path)
1616 elif modifier == "C":
1617 src, dest = diff['src'], diff['dst']
1618 p4_integrate(src, dest)
1619 pureRenameCopy.add(dest)
1620 if diff['src_sha1'] != diff['dst_sha1']:
1621 p4_edit(dest)
1622 pureRenameCopy.discard(dest)
1623 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1624 p4_edit(dest)
1625 pureRenameCopy.discard(dest)
1626 filesToChangeExecBit[dest] = diff['dst_mode']
1627 if self.isWindows:
1628 # turn off read-only attribute
1629 os.chmod(dest, stat.S_IWRITE)
1630 os.unlink(dest)
1631 editedFiles.add(dest)
1632 elif modifier == "R":
1633 src, dest = diff['src'], diff['dst']
1634 if self.p4HasMoveCommand:
1635 p4_edit(src) # src must be open before move
1636 p4_move(src, dest) # opens for (move/delete, move/add)
1637 else:
1638 p4_integrate(src, dest)
1639 if diff['src_sha1'] != diff['dst_sha1']:
1640 p4_edit(dest)
1641 else:
1642 pureRenameCopy.add(dest)
1643 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1644 if not self.p4HasMoveCommand:
1645 p4_edit(dest) # with move: already open, writable
1646 filesToChangeExecBit[dest] = diff['dst_mode']
1647 if not self.p4HasMoveCommand:
1648 if self.isWindows:
1649 os.chmod(dest, stat.S_IWRITE)
1650 os.unlink(dest)
1651 filesToDelete.add(src)
1652 editedFiles.add(dest)
1653 elif modifier == "T":
1654 filesToChangeType.add(path)
1655 else:
1656 die("unknown modifier %s for %s" % (modifier, path))
1658 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1659 patchcmd = diffcmd + " | git apply "
1660 tryPatchCmd = patchcmd + "--check -"
1661 applyPatchCmd = patchcmd + "--check --apply -"
1662 patch_succeeded = True
1664 if os.system(tryPatchCmd) != 0:
1665 fixed_rcs_keywords = False
1666 patch_succeeded = False
1667 print "Unfortunately applying the change failed!"
1669 # Patch failed, maybe it's just RCS keyword woes. Look through
1670 # the patch to see if that's possible.
1671 if gitConfigBool("git-p4.attemptRCSCleanup"):
1672 file = None
1673 pattern = None
1674 kwfiles = {}
1675 for file in editedFiles | filesToDelete:
1676 # did this file's delta contain RCS keywords?
1677 pattern = p4_keywords_regexp_for_file(file)
1679 if pattern:
1680 # this file is a possibility...look for RCS keywords.
1681 regexp = re.compile(pattern, re.VERBOSE)
1682 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1683 if regexp.search(line):
1684 if verbose:
1685 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1686 kwfiles[file] = pattern
1687 break
1689 for file in kwfiles:
1690 if verbose:
1691 print "zapping %s with %s" % (line,pattern)
1692 # File is being deleted, so not open in p4. Must
1693 # disable the read-only bit on windows.
1694 if self.isWindows and file not in editedFiles:
1695 os.chmod(file, stat.S_IWRITE)
1696 self.patchRCSKeywords(file, kwfiles[file])
1697 fixed_rcs_keywords = True
1699 if fixed_rcs_keywords:
1700 print "Retrying the patch with RCS keywords cleaned up"
1701 if os.system(tryPatchCmd) == 0:
1702 patch_succeeded = True
1704 if not patch_succeeded:
1705 for f in editedFiles:
1706 p4_revert(f)
1707 return False
1710 # Apply the patch for real, and do add/delete/+x handling.
1712 system(applyPatchCmd)
1714 for f in filesToChangeType:
1715 p4_edit(f, "-t", "auto")
1716 for f in filesToAdd:
1717 p4_add(f)
1718 for f in filesToDelete:
1719 p4_revert(f)
1720 p4_delete(f)
1722 # Set/clear executable bits
1723 for f in filesToChangeExecBit.keys():
1724 mode = filesToChangeExecBit[f]
1725 setP4ExecBit(f, mode)
1728 # Build p4 change description, starting with the contents
1729 # of the git commit message.
1731 logMessage = extractLogMessageFromGitCommit(id)
1732 logMessage = logMessage.strip()
1733 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1735 template = self.prepareSubmitTemplate()
1736 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1738 if self.preserveUser:
1739 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1741 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1742 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1743 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1744 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1746 separatorLine = "######## everything below this line is just the diff #######\n"
1747 if not self.prepare_p4_only:
1748 submitTemplate += separatorLine
1749 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1751 (handle, fileName) = tempfile.mkstemp()
1752 tmpFile = os.fdopen(handle, "w+b")
1753 if self.isWindows:
1754 submitTemplate = submitTemplate.replace("\n", "\r\n")
1755 tmpFile.write(submitTemplate)
1756 tmpFile.close()
1758 if self.prepare_p4_only:
1760 # Leave the p4 tree prepared, and the submit template around
1761 # and let the user decide what to do next
1763 print
1764 print "P4 workspace prepared for submission."
1765 print "To submit or revert, go to client workspace"
1766 print " " + self.clientPath
1767 print
1768 print "To submit, use \"p4 submit\" to write a new description,"
1769 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1770 " \"git p4\"." % fileName
1771 print "You can delete the file \"%s\" when finished." % fileName
1773 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1774 print "To preserve change ownership by user %s, you must\n" \
1775 "do \"p4 change -f <change>\" after submitting and\n" \
1776 "edit the User field."
1777 if pureRenameCopy:
1778 print "After submitting, renamed files must be re-synced."
1779 print "Invoke \"p4 sync -f\" on each of these files:"
1780 for f in pureRenameCopy:
1781 print " " + f
1783 print
1784 print "To revert the changes, use \"p4 revert ...\", and delete"
1785 print "the submit template file \"%s\"" % fileName
1786 if filesToAdd:
1787 print "Since the commit adds new files, they must be deleted:"
1788 for f in filesToAdd:
1789 print " " + f
1790 print
1791 return True
1794 # Let the user edit the change description, then submit it.
1796 submitted = False
1798 try:
1799 if self.edit_template(fileName):
1800 # read the edited message and submit
1801 tmpFile = open(fileName, "rb")
1802 message = tmpFile.read()
1803 tmpFile.close()
1804 if self.isWindows:
1805 message = message.replace("\r\n", "\n")
1806 submitTemplate = message[:message.index(separatorLine)]
1807 p4_write_pipe(['submit', '-i'], submitTemplate)
1809 if self.preserveUser:
1810 if p4User:
1811 # Get last changelist number. Cannot easily get it from
1812 # the submit command output as the output is
1813 # unmarshalled.
1814 changelist = self.lastP4Changelist()
1815 self.modifyChangelistUser(changelist, p4User)
1817 # The rename/copy happened by applying a patch that created a
1818 # new file. This leaves it writable, which confuses p4.
1819 for f in pureRenameCopy:
1820 p4_sync(f, "-f")
1821 submitted = True
1823 finally:
1824 # skip this patch
1825 if not submitted:
1826 print "Submission cancelled, undoing p4 changes."
1827 for f in editedFiles:
1828 p4_revert(f)
1829 for f in filesToAdd:
1830 p4_revert(f)
1831 os.remove(f)
1832 for f in filesToDelete:
1833 p4_revert(f)
1835 os.remove(fileName)
1836 return submitted
1838 # Export git tags as p4 labels. Create a p4 label and then tag
1839 # with that.
1840 def exportGitTags(self, gitTags):
1841 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1842 if len(validLabelRegexp) == 0:
1843 validLabelRegexp = defaultLabelRegexp
1844 m = re.compile(validLabelRegexp)
1846 for name in gitTags:
1848 if not m.match(name):
1849 if verbose:
1850 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1851 continue
1853 # Get the p4 commit this corresponds to
1854 logMessage = extractLogMessageFromGitCommit(name)
1855 values = extractSettingsGitLog(logMessage)
1857 if not values.has_key('change'):
1858 # a tag pointing to something not sent to p4; ignore
1859 if verbose:
1860 print "git tag %s does not give a p4 commit" % name
1861 continue
1862 else:
1863 changelist = values['change']
1865 # Get the tag details.
1866 inHeader = True
1867 isAnnotated = False
1868 body = []
1869 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1870 l = l.strip()
1871 if inHeader:
1872 if re.match(r'tag\s+', l):
1873 isAnnotated = True
1874 elif re.match(r'\s*$', l):
1875 inHeader = False
1876 continue
1877 else:
1878 body.append(l)
1880 if not isAnnotated:
1881 body = ["lightweight tag imported by git p4\n"]
1883 # Create the label - use the same view as the client spec we are using
1884 clientSpec = getClientSpec()
1886 labelTemplate = "Label: %s\n" % name
1887 labelTemplate += "Description:\n"
1888 for b in body:
1889 labelTemplate += "\t" + b + "\n"
1890 labelTemplate += "View:\n"
1891 for depot_side in clientSpec.mappings:
1892 labelTemplate += "\t%s\n" % depot_side
1894 if self.dry_run:
1895 print "Would create p4 label %s for tag" % name
1896 elif self.prepare_p4_only:
1897 print "Not creating p4 label %s for tag due to option" \
1898 " --prepare-p4-only" % name
1899 else:
1900 p4_write_pipe(["label", "-i"], labelTemplate)
1902 # Use the label
1903 p4_system(["tag", "-l", name] +
1904 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1906 if verbose:
1907 print "created p4 label for tag %s" % name
1909 def run(self, args):
1910 if len(args) == 0:
1911 self.master = currentGitBranch()
1912 elif len(args) == 1:
1913 self.master = args[0]
1914 if not branchExists(self.master):
1915 die("Branch %s does not exist" % self.master)
1916 else:
1917 return False
1919 if self.master:
1920 allowSubmit = gitConfig("git-p4.allowSubmit")
1921 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1922 die("%s is not in git-p4.allowSubmit" % self.master)
1924 [upstream, settings] = findUpstreamBranchPoint()
1925 self.depotPath = settings['depot-paths'][0]
1926 if len(self.origin) == 0:
1927 self.origin = upstream
1929 if self.preserveUser:
1930 if not self.canChangeChangelists():
1931 die("Cannot preserve user names without p4 super-user or admin permissions")
1933 # if not set from the command line, try the config file
1934 if self.conflict_behavior is None:
1935 val = gitConfig("git-p4.conflict")
1936 if val:
1937 if val not in self.conflict_behavior_choices:
1938 die("Invalid value '%s' for config git-p4.conflict" % val)
1939 else:
1940 val = "ask"
1941 self.conflict_behavior = val
1943 if self.verbose:
1944 print "Origin branch is " + self.origin
1946 if len(self.depotPath) == 0:
1947 print "Internal error: cannot locate perforce depot path from existing branches"
1948 sys.exit(128)
1950 self.useClientSpec = False
1951 if gitConfigBool("git-p4.useclientspec"):
1952 self.useClientSpec = True
1953 if self.useClientSpec:
1954 self.clientSpecDirs = getClientSpec()
1956 # Check for the existence of P4 branches
1957 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1959 if self.useClientSpec and not branchesDetected:
1960 # all files are relative to the client spec
1961 self.clientPath = getClientRoot()
1962 else:
1963 self.clientPath = p4Where(self.depotPath)
1965 if self.clientPath == "":
1966 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1968 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1969 self.oldWorkingDirectory = os.getcwd()
1971 # ensure the clientPath exists
1972 new_client_dir = False
1973 if not os.path.exists(self.clientPath):
1974 new_client_dir = True
1975 os.makedirs(self.clientPath)
1977 chdir(self.clientPath, is_client_path=True)
1978 if self.dry_run:
1979 print "Would synchronize p4 checkout in %s" % self.clientPath
1980 else:
1981 print "Synchronizing p4 checkout..."
1982 if new_client_dir:
1983 # old one was destroyed, and maybe nobody told p4
1984 p4_sync("...", "-f")
1985 else:
1986 p4_sync("...")
1987 self.check()
1989 commits = []
1990 if self.master:
1991 commitish = self.master
1992 else:
1993 commitish = 'HEAD'
1995 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1996 commits.append(line.strip())
1997 commits.reverse()
1999 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2000 self.checkAuthorship = False
2001 else:
2002 self.checkAuthorship = True
2004 if self.preserveUser:
2005 self.checkValidP4Users(commits)
2008 # Build up a set of options to be passed to diff when
2009 # submitting each commit to p4.
2011 if self.detectRenames:
2012 # command-line -M arg
2013 self.diffOpts = "-M"
2014 else:
2015 # If not explicitly set check the config variable
2016 detectRenames = gitConfig("git-p4.detectRenames")
2018 if detectRenames.lower() == "false" or detectRenames == "":
2019 self.diffOpts = ""
2020 elif detectRenames.lower() == "true":
2021 self.diffOpts = "-M"
2022 else:
2023 self.diffOpts = "-M%s" % detectRenames
2025 # no command-line arg for -C or --find-copies-harder, just
2026 # config variables
2027 detectCopies = gitConfig("git-p4.detectCopies")
2028 if detectCopies.lower() == "false" or detectCopies == "":
2029 pass
2030 elif detectCopies.lower() == "true":
2031 self.diffOpts += " -C"
2032 else:
2033 self.diffOpts += " -C%s" % detectCopies
2035 if gitConfigBool("git-p4.detectCopiesHarder"):
2036 self.diffOpts += " --find-copies-harder"
2039 # Apply the commits, one at a time. On failure, ask if should
2040 # continue to try the rest of the patches, or quit.
2042 if self.dry_run:
2043 print "Would apply"
2044 applied = []
2045 last = len(commits) - 1
2046 for i, commit in enumerate(commits):
2047 if self.dry_run:
2048 print " ", read_pipe(["git", "show", "-s",
2049 "--format=format:%h %s", commit])
2050 ok = True
2051 else:
2052 ok = self.applyCommit(commit)
2053 if ok:
2054 applied.append(commit)
2055 else:
2056 if self.prepare_p4_only and i < last:
2057 print "Processing only the first commit due to option" \
2058 " --prepare-p4-only"
2059 break
2060 if i < last:
2061 quit = False
2062 while True:
2063 # prompt for what to do, or use the option/variable
2064 if self.conflict_behavior == "ask":
2065 print "What do you want to do?"
2066 response = raw_input("[s]kip this commit but apply"
2067 " the rest, or [q]uit? ")
2068 if not response:
2069 continue
2070 elif self.conflict_behavior == "skip":
2071 response = "s"
2072 elif self.conflict_behavior == "quit":
2073 response = "q"
2074 else:
2075 die("Unknown conflict_behavior '%s'" %
2076 self.conflict_behavior)
2078 if response[0] == "s":
2079 print "Skipping this commit, but applying the rest"
2080 break
2081 if response[0] == "q":
2082 print "Quitting"
2083 quit = True
2084 break
2085 if quit:
2086 break
2088 chdir(self.oldWorkingDirectory)
2090 if self.dry_run:
2091 pass
2092 elif self.prepare_p4_only:
2093 pass
2094 elif len(commits) == len(applied):
2095 print "All commits applied!"
2097 sync = P4Sync()
2098 if self.branch:
2099 sync.branch = self.branch
2100 sync.run([])
2102 rebase = P4Rebase()
2103 rebase.rebase()
2105 else:
2106 if len(applied) == 0:
2107 print "No commits applied."
2108 else:
2109 print "Applied only the commits marked with '*':"
2110 for c in commits:
2111 if c in applied:
2112 star = "*"
2113 else:
2114 star = " "
2115 print star, read_pipe(["git", "show", "-s",
2116 "--format=format:%h %s", c])
2117 print "You will have to do 'git p4 sync' and rebase."
2119 if gitConfigBool("git-p4.exportLabels"):
2120 self.exportLabels = True
2122 if self.exportLabels:
2123 p4Labels = getP4Labels(self.depotPath)
2124 gitTags = getGitTags()
2126 missingGitTags = gitTags - p4Labels
2127 self.exportGitTags(missingGitTags)
2129 # exit with error unless everything applied perfectly
2130 if len(commits) != len(applied):
2131 sys.exit(1)
2133 return True
2135 class View(object):
2136 """Represent a p4 view ("p4 help views"), and map files in a
2137 repo according to the view."""
2139 def __init__(self, client_name):
2140 self.mappings = []
2141 self.client_prefix = "//%s/" % client_name
2142 # cache results of "p4 where" to lookup client file locations
2143 self.client_spec_path_cache = {}
2145 def append(self, view_line):
2146 """Parse a view line, splitting it into depot and client
2147 sides. Append to self.mappings, preserving order. This
2148 is only needed for tag creation."""
2150 # Split the view line into exactly two words. P4 enforces
2151 # structure on these lines that simplifies this quite a bit.
2153 # Either or both words may be double-quoted.
2154 # Single quotes do not matter.
2155 # Double-quote marks cannot occur inside the words.
2156 # A + or - prefix is also inside the quotes.
2157 # There are no quotes unless they contain a space.
2158 # The line is already white-space stripped.
2159 # The two words are separated by a single space.
2161 if view_line[0] == '"':
2162 # First word is double quoted. Find its end.
2163 close_quote_index = view_line.find('"', 1)
2164 if close_quote_index <= 0:
2165 die("No first-word closing quote found: %s" % view_line)
2166 depot_side = view_line[1:close_quote_index]
2167 # skip closing quote and space
2168 rhs_index = close_quote_index + 1 + 1
2169 else:
2170 space_index = view_line.find(" ")
2171 if space_index <= 0:
2172 die("No word-splitting space found: %s" % view_line)
2173 depot_side = view_line[0:space_index]
2174 rhs_index = space_index + 1
2176 # prefix + means overlay on previous mapping
2177 if depot_side.startswith("+"):
2178 depot_side = depot_side[1:]
2180 # prefix - means exclude this path, leave out of mappings
2181 exclude = False
2182 if depot_side.startswith("-"):
2183 exclude = True
2184 depot_side = depot_side[1:]
2186 if not exclude:
2187 self.mappings.append(depot_side)
2189 def convert_client_path(self, clientFile):
2190 # chop off //client/ part to make it relative
2191 if not clientFile.startswith(self.client_prefix):
2192 die("No prefix '%s' on clientFile '%s'" %
2193 (self.client_prefix, clientFile))
2194 return clientFile[len(self.client_prefix):]
2196 def update_client_spec_path_cache(self, files):
2197 """ Caching file paths by "p4 where" batch query """
2199 # List depot file paths exclude that already cached
2200 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2202 if len(fileArgs) == 0:
2203 return # All files in cache
2205 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2206 for res in where_result:
2207 if "code" in res and res["code"] == "error":
2208 # assume error is "... file(s) not in client view"
2209 continue
2210 if "clientFile" not in res:
2211 die("No clientFile in 'p4 where' output")
2212 if "unmap" in res:
2213 # it will list all of them, but only one not unmap-ped
2214 continue
2215 if gitConfigBool("core.ignorecase"):
2216 res['depotFile'] = res['depotFile'].lower()
2217 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2219 # not found files or unmap files set to ""
2220 for depotFile in fileArgs:
2221 if gitConfigBool("core.ignorecase"):
2222 depotFile = depotFile.lower()
2223 if depotFile not in self.client_spec_path_cache:
2224 self.client_spec_path_cache[depotFile] = ""
2226 def map_in_client(self, depot_path):
2227 """Return the relative location in the client where this
2228 depot file should live. Returns "" if the file should
2229 not be mapped in the client."""
2231 if gitConfigBool("core.ignorecase"):
2232 depot_path = depot_path.lower()
2234 if depot_path in self.client_spec_path_cache:
2235 return self.client_spec_path_cache[depot_path]
2237 die( "Error: %s is not found in client spec path" % depot_path )
2238 return ""
2240 class P4Sync(Command, P4UserMap):
2241 delete_actions = ( "delete", "move/delete", "purge" )
2243 def __init__(self):
2244 Command.__init__(self)
2245 P4UserMap.__init__(self)
2246 self.options = [
2247 optparse.make_option("--branch", dest="branch"),
2248 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2249 optparse.make_option("--changesfile", dest="changesFile"),
2250 optparse.make_option("--silent", dest="silent", action="store_true"),
2251 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2252 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2253 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2254 help="Import into refs/heads/ , not refs/remotes"),
2255 optparse.make_option("--max-changes", dest="maxChanges",
2256 help="Maximum number of changes to import"),
2257 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2258 help="Internal block size to use when iteratively calling p4 changes"),
2259 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2260 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2261 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2262 help="Only sync files that are included in the Perforce Client Spec"),
2263 optparse.make_option("-/", dest="cloneExclude",
2264 action="append", type="string",
2265 help="exclude depot path"),
2267 self.description = """Imports from Perforce into a git repository.\n
2268 example:
2269 //depot/my/project/ -- to import the current head
2270 //depot/my/project/@all -- to import everything
2271 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2273 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2275 self.usage += " //depot/path[@revRange]"
2276 self.silent = False
2277 self.createdBranches = set()
2278 self.committedChanges = set()
2279 self.branch = ""
2280 self.detectBranches = False
2281 self.detectLabels = False
2282 self.importLabels = False
2283 self.changesFile = ""
2284 self.syncWithOrigin = True
2285 self.importIntoRemotes = True
2286 self.maxChanges = ""
2287 self.changes_block_size = None
2288 self.keepRepoPath = False
2289 self.depotPaths = None
2290 self.p4BranchesInGit = []
2291 self.cloneExclude = []
2292 self.useClientSpec = False
2293 self.useClientSpec_from_options = False
2294 self.clientSpecDirs = None
2295 self.tempBranches = []
2296 self.tempBranchLocation = "refs/git-p4-tmp"
2297 self.largeFileSystem = None
2299 if gitConfig('git-p4.largeFileSystem'):
2300 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2301 self.largeFileSystem = largeFileSystemConstructor(
2302 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2305 if gitConfig("git-p4.syncFromOrigin") == "false":
2306 self.syncWithOrigin = False
2308 # This is required for the "append" cloneExclude action
2309 def ensure_value(self, attr, value):
2310 if not hasattr(self, attr) or getattr(self, attr) is None:
2311 setattr(self, attr, value)
2312 return getattr(self, attr)
2314 # Force a checkpoint in fast-import and wait for it to finish
2315 def checkpoint(self):
2316 self.gitStream.write("checkpoint\n\n")
2317 self.gitStream.write("progress checkpoint\n\n")
2318 out = self.gitOutput.readline()
2319 if self.verbose:
2320 print "checkpoint finished: " + out
2322 def extractFilesFromCommit(self, commit):
2323 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2324 for path in self.cloneExclude]
2325 files = []
2326 fnum = 0
2327 while commit.has_key("depotFile%s" % fnum):
2328 path = commit["depotFile%s" % fnum]
2330 if [p for p in self.cloneExclude
2331 if p4PathStartsWith(path, p)]:
2332 found = False
2333 else:
2334 found = [p for p in self.depotPaths
2335 if p4PathStartsWith(path, p)]
2336 if not found:
2337 fnum = fnum + 1
2338 continue
2340 file = {}
2341 file["path"] = path
2342 file["rev"] = commit["rev%s" % fnum]
2343 file["action"] = commit["action%s" % fnum]
2344 file["type"] = commit["type%s" % fnum]
2345 files.append(file)
2346 fnum = fnum + 1
2347 return files
2349 def extractJobsFromCommit(self, commit):
2350 jobs = []
2351 jnum = 0
2352 while commit.has_key("job%s" % jnum):
2353 job = commit["job%s" % jnum]
2354 jobs.append(job)
2355 jnum = jnum + 1
2356 return jobs
2358 def stripRepoPath(self, path, prefixes):
2359 """When streaming files, this is called to map a p4 depot path
2360 to where it should go in git. The prefixes are either
2361 self.depotPaths, or self.branchPrefixes in the case of
2362 branch detection."""
2364 if self.useClientSpec:
2365 # branch detection moves files up a level (the branch name)
2366 # from what client spec interpretation gives
2367 path = self.clientSpecDirs.map_in_client(path)
2368 if self.detectBranches:
2369 for b in self.knownBranches:
2370 if path.startswith(b + "/"):
2371 path = path[len(b)+1:]
2373 elif self.keepRepoPath:
2374 # Preserve everything in relative path name except leading
2375 # //depot/; just look at first prefix as they all should
2376 # be in the same depot.
2377 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2378 if p4PathStartsWith(path, depot):
2379 path = path[len(depot):]
2381 else:
2382 for p in prefixes:
2383 if p4PathStartsWith(path, p):
2384 path = path[len(p):]
2385 break
2387 path = wildcard_decode(path)
2388 return path
2390 def splitFilesIntoBranches(self, commit):
2391 """Look at each depotFile in the commit to figure out to what
2392 branch it belongs."""
2394 if self.clientSpecDirs:
2395 files = self.extractFilesFromCommit(commit)
2396 self.clientSpecDirs.update_client_spec_path_cache(files)
2398 branches = {}
2399 fnum = 0
2400 while commit.has_key("depotFile%s" % fnum):
2401 path = commit["depotFile%s" % fnum]
2402 found = [p for p in self.depotPaths
2403 if p4PathStartsWith(path, p)]
2404 if not found:
2405 fnum = fnum + 1
2406 continue
2408 file = {}
2409 file["path"] = path
2410 file["rev"] = commit["rev%s" % fnum]
2411 file["action"] = commit["action%s" % fnum]
2412 file["type"] = commit["type%s" % fnum]
2413 fnum = fnum + 1
2415 # start with the full relative path where this file would
2416 # go in a p4 client
2417 if self.useClientSpec:
2418 relPath = self.clientSpecDirs.map_in_client(path)
2419 else:
2420 relPath = self.stripRepoPath(path, self.depotPaths)
2422 for branch in self.knownBranches.keys():
2423 # add a trailing slash so that a commit into qt/4.2foo
2424 # doesn't end up in qt/4.2, e.g.
2425 if relPath.startswith(branch + "/"):
2426 if branch not in branches:
2427 branches[branch] = []
2428 branches[branch].append(file)
2429 break
2431 return branches
2433 def writeToGitStream(self, gitMode, relPath, contents):
2434 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2435 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2436 for d in contents:
2437 self.gitStream.write(d)
2438 self.gitStream.write('\n')
2440 # output one file from the P4 stream
2441 # - helper for streamP4Files
2443 def streamOneP4File(self, file, contents):
2444 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2445 if verbose:
2446 size = int(self.stream_file['fileSize'])
2447 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2448 sys.stdout.flush()
2450 (type_base, type_mods) = split_p4_type(file["type"])
2452 git_mode = "100644"
2453 if "x" in type_mods:
2454 git_mode = "100755"
2455 if type_base == "symlink":
2456 git_mode = "120000"
2457 # p4 print on a symlink sometimes contains "target\n";
2458 # if it does, remove the newline
2459 data = ''.join(contents)
2460 if not data:
2461 # Some version of p4 allowed creating a symlink that pointed
2462 # to nothing. This causes p4 errors when checking out such
2463 # a change, and errors here too. Work around it by ignoring
2464 # the bad symlink; hopefully a future change fixes it.
2465 print "\nIgnoring empty symlink in %s" % file['depotFile']
2466 return
2467 elif data[-1] == '\n':
2468 contents = [data[:-1]]
2469 else:
2470 contents = [data]
2472 if type_base == "utf16":
2473 # p4 delivers different text in the python output to -G
2474 # than it does when using "print -o", or normal p4 client
2475 # operations. utf16 is converted to ascii or utf8, perhaps.
2476 # But ascii text saved as -t utf16 is completely mangled.
2477 # Invoke print -o to get the real contents.
2479 # On windows, the newlines will always be mangled by print, so put
2480 # them back too. This is not needed to the cygwin windows version,
2481 # just the native "NT" type.
2483 try:
2484 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2485 except Exception as e:
2486 if 'Translation of file content failed' in str(e):
2487 type_base = 'binary'
2488 else:
2489 raise e
2490 else:
2491 if p4_version_string().find('/NT') >= 0:
2492 text = text.replace('\r\n', '\n')
2493 contents = [ text ]
2495 if type_base == "apple":
2496 # Apple filetype files will be streamed as a concatenation of
2497 # its appledouble header and the contents. This is useless
2498 # on both macs and non-macs. If using "print -q -o xx", it
2499 # will create "xx" with the data, and "%xx" with the header.
2500 # This is also not very useful.
2502 # Ideally, someday, this script can learn how to generate
2503 # appledouble files directly and import those to git, but
2504 # non-mac machines can never find a use for apple filetype.
2505 print "\nIgnoring apple filetype file %s" % file['depotFile']
2506 return
2508 # Note that we do not try to de-mangle keywords on utf16 files,
2509 # even though in theory somebody may want that.
2510 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2511 if pattern:
2512 regexp = re.compile(pattern, re.VERBOSE)
2513 text = ''.join(contents)
2514 text = regexp.sub(r'$\1$', text)
2515 contents = [ text ]
2517 try:
2518 relPath.decode('ascii')
2519 except:
2520 encoding = 'utf8'
2521 if gitConfig('git-p4.pathEncoding'):
2522 encoding = gitConfig('git-p4.pathEncoding')
2523 relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2524 if self.verbose:
2525 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2527 if self.largeFileSystem:
2528 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2530 self.writeToGitStream(git_mode, relPath, contents)
2532 def streamOneP4Deletion(self, file):
2533 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2534 if verbose:
2535 sys.stdout.write("delete %s\n" % relPath)
2536 sys.stdout.flush()
2537 self.gitStream.write("D %s\n" % relPath)
2539 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2540 self.largeFileSystem.removeLargeFile(relPath)
2542 # handle another chunk of streaming data
2543 def streamP4FilesCb(self, marshalled):
2545 # catch p4 errors and complain
2546 err = None
2547 if "code" in marshalled:
2548 if marshalled["code"] == "error":
2549 if "data" in marshalled:
2550 err = marshalled["data"].rstrip()
2552 if not err and 'fileSize' in self.stream_file:
2553 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2554 if required_bytes > 0:
2555 err = 'Not enough space left on %s! Free at least %i MB.' % (
2556 os.getcwd(), required_bytes/1024/1024
2559 if err:
2560 f = None
2561 if self.stream_have_file_info:
2562 if "depotFile" in self.stream_file:
2563 f = self.stream_file["depotFile"]
2564 # force a failure in fast-import, else an empty
2565 # commit will be made
2566 self.gitStream.write("\n")
2567 self.gitStream.write("die-now\n")
2568 self.gitStream.close()
2569 # ignore errors, but make sure it exits first
2570 self.importProcess.wait()
2571 if f:
2572 die("Error from p4 print for %s: %s" % (f, err))
2573 else:
2574 die("Error from p4 print: %s" % err)
2576 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2577 # start of a new file - output the old one first
2578 self.streamOneP4File(self.stream_file, self.stream_contents)
2579 self.stream_file = {}
2580 self.stream_contents = []
2581 self.stream_have_file_info = False
2583 # pick up the new file information... for the
2584 # 'data' field we need to append to our array
2585 for k in marshalled.keys():
2586 if k == 'data':
2587 if 'streamContentSize' not in self.stream_file:
2588 self.stream_file['streamContentSize'] = 0
2589 self.stream_file['streamContentSize'] += len(marshalled['data'])
2590 self.stream_contents.append(marshalled['data'])
2591 else:
2592 self.stream_file[k] = marshalled[k]
2594 if (verbose and
2595 'streamContentSize' in self.stream_file and
2596 'fileSize' in self.stream_file and
2597 'depotFile' in self.stream_file):
2598 size = int(self.stream_file["fileSize"])
2599 if size > 0:
2600 progress = 100*self.stream_file['streamContentSize']/size
2601 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2602 sys.stdout.flush()
2604 self.stream_have_file_info = True
2606 # Stream directly from "p4 files" into "git fast-import"
2607 def streamP4Files(self, files):
2608 filesForCommit = []
2609 filesToRead = []
2610 filesToDelete = []
2612 for f in files:
2613 filesForCommit.append(f)
2614 if f['action'] in self.delete_actions:
2615 filesToDelete.append(f)
2616 else:
2617 filesToRead.append(f)
2619 # deleted files...
2620 for f in filesToDelete:
2621 self.streamOneP4Deletion(f)
2623 if len(filesToRead) > 0:
2624 self.stream_file = {}
2625 self.stream_contents = []
2626 self.stream_have_file_info = False
2628 # curry self argument
2629 def streamP4FilesCbSelf(entry):
2630 self.streamP4FilesCb(entry)
2632 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2634 p4CmdList(["-x", "-", "print"],
2635 stdin=fileArgs,
2636 cb=streamP4FilesCbSelf)
2638 # do the last chunk
2639 if self.stream_file.has_key('depotFile'):
2640 self.streamOneP4File(self.stream_file, self.stream_contents)
2642 def make_email(self, userid):
2643 if userid in self.users:
2644 return self.users[userid]
2645 else:
2646 return "%s <a@b>" % userid
2648 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2649 """ Stream a p4 tag.
2650 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2653 if verbose:
2654 print "writing tag %s for commit %s" % (labelName, commit)
2655 gitStream.write("tag %s\n" % labelName)
2656 gitStream.write("from %s\n" % commit)
2658 if labelDetails.has_key('Owner'):
2659 owner = labelDetails["Owner"]
2660 else:
2661 owner = None
2663 # Try to use the owner of the p4 label, or failing that,
2664 # the current p4 user id.
2665 if owner:
2666 email = self.make_email(owner)
2667 else:
2668 email = self.make_email(self.p4UserId())
2669 tagger = "%s %s %s" % (email, epoch, self.tz)
2671 gitStream.write("tagger %s\n" % tagger)
2673 print "labelDetails=",labelDetails
2674 if labelDetails.has_key('Description'):
2675 description = labelDetails['Description']
2676 else:
2677 description = 'Label from git p4'
2679 gitStream.write("data %d\n" % len(description))
2680 gitStream.write(description)
2681 gitStream.write("\n")
2683 def inClientSpec(self, path):
2684 if not self.clientSpecDirs:
2685 return True
2686 inClientSpec = self.clientSpecDirs.map_in_client(path)
2687 if not inClientSpec and self.verbose:
2688 print('Ignoring file outside of client spec: {0}'.format(path))
2689 return inClientSpec
2691 def hasBranchPrefix(self, path):
2692 if not self.branchPrefixes:
2693 return True
2694 hasPrefix = [p for p in self.branchPrefixes
2695 if p4PathStartsWith(path, p)]
2696 if not hasPrefix and self.verbose:
2697 print('Ignoring file outside of prefix: {0}'.format(path))
2698 return hasPrefix
2700 def commit(self, details, files, branch, parent = ""):
2701 epoch = details["time"]
2702 author = details["user"]
2703 jobs = self.extractJobsFromCommit(details)
2705 if self.verbose:
2706 print('commit into {0}'.format(branch))
2708 if self.clientSpecDirs:
2709 self.clientSpecDirs.update_client_spec_path_cache(files)
2711 files = [f for f in files
2712 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2714 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2715 print('Ignoring revision {0} as it would produce an empty commit.'
2716 .format(details['change']))
2717 return
2719 self.gitStream.write("commit %s\n" % branch)
2720 self.gitStream.write("mark :%s\n" % details["change"])
2721 self.committedChanges.add(int(details["change"]))
2722 committer = ""
2723 if author not in self.users:
2724 self.getUserMapFromPerforceServer()
2725 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2727 self.gitStream.write("committer %s\n" % committer)
2729 self.gitStream.write("data <<EOT\n")
2730 self.gitStream.write(details["desc"])
2731 if len(jobs) > 0:
2732 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2733 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2734 (','.join(self.branchPrefixes), details["change"]))
2735 if len(details['options']) > 0:
2736 self.gitStream.write(": options = %s" % details['options'])
2737 self.gitStream.write("]\nEOT\n\n")
2739 if len(parent) > 0:
2740 if self.verbose:
2741 print "parent %s" % parent
2742 self.gitStream.write("from %s\n" % parent)
2744 self.streamP4Files(files)
2745 self.gitStream.write("\n")
2747 change = int(details["change"])
2749 if self.labels.has_key(change):
2750 label = self.labels[change]
2751 labelDetails = label[0]
2752 labelRevisions = label[1]
2753 if self.verbose:
2754 print "Change %s is labelled %s" % (change, labelDetails)
2756 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2757 for p in self.branchPrefixes])
2759 if len(files) == len(labelRevisions):
2761 cleanedFiles = {}
2762 for info in files:
2763 if info["action"] in self.delete_actions:
2764 continue
2765 cleanedFiles[info["depotFile"]] = info["rev"]
2767 if cleanedFiles == labelRevisions:
2768 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2770 else:
2771 if not self.silent:
2772 print ("Tag %s does not match with change %s: files do not match."
2773 % (labelDetails["label"], change))
2775 else:
2776 if not self.silent:
2777 print ("Tag %s does not match with change %s: file count is different."
2778 % (labelDetails["label"], change))
2780 # Build a dictionary of changelists and labels, for "detect-labels" option.
2781 def getLabels(self):
2782 self.labels = {}
2784 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2785 if len(l) > 0 and not self.silent:
2786 print "Finding files belonging to labels in %s" % `self.depotPaths`
2788 for output in l:
2789 label = output["label"]
2790 revisions = {}
2791 newestChange = 0
2792 if self.verbose:
2793 print "Querying files for label %s" % label
2794 for file in p4CmdList(["files"] +
2795 ["%s...@%s" % (p, label)
2796 for p in self.depotPaths]):
2797 revisions[file["depotFile"]] = file["rev"]
2798 change = int(file["change"])
2799 if change > newestChange:
2800 newestChange = change
2802 self.labels[newestChange] = [output, revisions]
2804 if self.verbose:
2805 print "Label changes: %s" % self.labels.keys()
2807 # Import p4 labels as git tags. A direct mapping does not
2808 # exist, so assume that if all the files are at the same revision
2809 # then we can use that, or it's something more complicated we should
2810 # just ignore.
2811 def importP4Labels(self, stream, p4Labels):
2812 if verbose:
2813 print "import p4 labels: " + ' '.join(p4Labels)
2815 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2816 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2817 if len(validLabelRegexp) == 0:
2818 validLabelRegexp = defaultLabelRegexp
2819 m = re.compile(validLabelRegexp)
2821 for name in p4Labels:
2822 commitFound = False
2824 if not m.match(name):
2825 if verbose:
2826 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2827 continue
2829 if name in ignoredP4Labels:
2830 continue
2832 labelDetails = p4CmdList(['label', "-o", name])[0]
2834 # get the most recent changelist for each file in this label
2835 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2836 for p in self.depotPaths])
2838 if change.has_key('change'):
2839 # find the corresponding git commit; take the oldest commit
2840 changelist = int(change['change'])
2841 if changelist in self.committedChanges:
2842 gitCommit = ":%d" % changelist # use a fast-import mark
2843 commitFound = True
2844 else:
2845 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2846 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2847 if len(gitCommit) == 0:
2848 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2849 else:
2850 commitFound = True
2851 gitCommit = gitCommit.strip()
2853 if commitFound:
2854 # Convert from p4 time format
2855 try:
2856 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2857 except ValueError:
2858 print "Could not convert label time %s" % labelDetails['Update']
2859 tmwhen = 1
2861 when = int(time.mktime(tmwhen))
2862 self.streamTag(stream, name, labelDetails, gitCommit, when)
2863 if verbose:
2864 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2865 else:
2866 if verbose:
2867 print "Label %s has no changelists - possibly deleted?" % name
2869 if not commitFound:
2870 # We can't import this label; don't try again as it will get very
2871 # expensive repeatedly fetching all the files for labels that will
2872 # never be imported. If the label is moved in the future, the
2873 # ignore will need to be removed manually.
2874 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2876 def guessProjectName(self):
2877 for p in self.depotPaths:
2878 if p.endswith("/"):
2879 p = p[:-1]
2880 p = p[p.strip().rfind("/") + 1:]
2881 if not p.endswith("/"):
2882 p += "/"
2883 return p
2885 def getBranchMapping(self):
2886 lostAndFoundBranches = set()
2888 user = gitConfig("git-p4.branchUser")
2889 if len(user) > 0:
2890 command = "branches -u %s" % user
2891 else:
2892 command = "branches"
2894 for info in p4CmdList(command):
2895 details = p4Cmd(["branch", "-o", info["branch"]])
2896 viewIdx = 0
2897 while details.has_key("View%s" % viewIdx):
2898 paths = details["View%s" % viewIdx].split(" ")
2899 viewIdx = viewIdx + 1
2900 # require standard //depot/foo/... //depot/bar/... mapping
2901 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2902 continue
2903 source = paths[0]
2904 destination = paths[1]
2905 ## HACK
2906 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2907 source = source[len(self.depotPaths[0]):-4]
2908 destination = destination[len(self.depotPaths[0]):-4]
2910 if destination in self.knownBranches:
2911 if not self.silent:
2912 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2913 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2914 continue
2916 self.knownBranches[destination] = source
2918 lostAndFoundBranches.discard(destination)
2920 if source not in self.knownBranches:
2921 lostAndFoundBranches.add(source)
2923 # Perforce does not strictly require branches to be defined, so we also
2924 # check git config for a branch list.
2926 # Example of branch definition in git config file:
2927 # [git-p4]
2928 # branchList=main:branchA
2929 # branchList=main:branchB
2930 # branchList=branchA:branchC
2931 configBranches = gitConfigList("git-p4.branchList")
2932 for branch in configBranches:
2933 if branch:
2934 (source, destination) = branch.split(":")
2935 self.knownBranches[destination] = source
2937 lostAndFoundBranches.discard(destination)
2939 if source not in self.knownBranches:
2940 lostAndFoundBranches.add(source)
2943 for branch in lostAndFoundBranches:
2944 self.knownBranches[branch] = branch
2946 def getBranchMappingFromGitBranches(self):
2947 branches = p4BranchesInGit(self.importIntoRemotes)
2948 for branch in branches.keys():
2949 if branch == "master":
2950 branch = "main"
2951 else:
2952 branch = branch[len(self.projectName):]
2953 self.knownBranches[branch] = branch
2955 def updateOptionDict(self, d):
2956 option_keys = {}
2957 if self.keepRepoPath:
2958 option_keys['keepRepoPath'] = 1
2960 d["options"] = ' '.join(sorted(option_keys.keys()))
2962 def readOptions(self, d):
2963 self.keepRepoPath = (d.has_key('options')
2964 and ('keepRepoPath' in d['options']))
2966 def gitRefForBranch(self, branch):
2967 if branch == "main":
2968 return self.refPrefix + "master"
2970 if len(branch) <= 0:
2971 return branch
2973 return self.refPrefix + self.projectName + branch
2975 def gitCommitByP4Change(self, ref, change):
2976 if self.verbose:
2977 print "looking in ref " + ref + " for change %s using bisect..." % change
2979 earliestCommit = ""
2980 latestCommit = parseRevision(ref)
2982 while True:
2983 if self.verbose:
2984 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2985 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2986 if len(next) == 0:
2987 if self.verbose:
2988 print "argh"
2989 return ""
2990 log = extractLogMessageFromGitCommit(next)
2991 settings = extractSettingsGitLog(log)
2992 currentChange = int(settings['change'])
2993 if self.verbose:
2994 print "current change %s" % currentChange
2996 if currentChange == change:
2997 if self.verbose:
2998 print "found %s" % next
2999 return next
3001 if currentChange < change:
3002 earliestCommit = "^%s" % next
3003 else:
3004 latestCommit = "%s" % next
3006 return ""
3008 def importNewBranch(self, branch, maxChange):
3009 # make fast-import flush all changes to disk and update the refs using the checkpoint
3010 # command so that we can try to find the branch parent in the git history
3011 self.gitStream.write("checkpoint\n\n");
3012 self.gitStream.flush();
3013 branchPrefix = self.depotPaths[0] + branch + "/"
3014 range = "@1,%s" % maxChange
3015 #print "prefix" + branchPrefix
3016 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3017 if len(changes) <= 0:
3018 return False
3019 firstChange = changes[0]
3020 #print "first change in branch: %s" % firstChange
3021 sourceBranch = self.knownBranches[branch]
3022 sourceDepotPath = self.depotPaths[0] + sourceBranch
3023 sourceRef = self.gitRefForBranch(sourceBranch)
3024 #print "source " + sourceBranch
3026 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3027 #print "branch parent: %s" % branchParentChange
3028 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3029 if len(gitParent) > 0:
3030 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3031 #print "parent git commit: %s" % gitParent
3033 self.importChanges(changes)
3034 return True
3036 def searchParent(self, parent, branch, target):
3037 parentFound = False
3038 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3039 "--no-merges", parent]):
3040 blob = blob.strip()
3041 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3042 parentFound = True
3043 if self.verbose:
3044 print "Found parent of %s in commit %s" % (branch, blob)
3045 break
3046 if parentFound:
3047 return blob
3048 else:
3049 return None
3051 def importChanges(self, changes):
3052 cnt = 1
3053 for change in changes:
3054 description = p4_describe(change)
3055 self.updateOptionDict(description)
3057 if not self.silent:
3058 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3059 sys.stdout.flush()
3060 cnt = cnt + 1
3062 try:
3063 if self.detectBranches:
3064 branches = self.splitFilesIntoBranches(description)
3065 for branch in branches.keys():
3066 ## HACK --hwn
3067 branchPrefix = self.depotPaths[0] + branch + "/"
3068 self.branchPrefixes = [ branchPrefix ]
3070 parent = ""
3072 filesForCommit = branches[branch]
3074 if self.verbose:
3075 print "branch is %s" % branch
3077 self.updatedBranches.add(branch)
3079 if branch not in self.createdBranches:
3080 self.createdBranches.add(branch)
3081 parent = self.knownBranches[branch]
3082 if parent == branch:
3083 parent = ""
3084 else:
3085 fullBranch = self.projectName + branch
3086 if fullBranch not in self.p4BranchesInGit:
3087 if not self.silent:
3088 print("\n Importing new branch %s" % fullBranch);
3089 if self.importNewBranch(branch, change - 1):
3090 parent = ""
3091 self.p4BranchesInGit.append(fullBranch)
3092 if not self.silent:
3093 print("\n Resuming with change %s" % change);
3095 if self.verbose:
3096 print "parent determined through known branches: %s" % parent
3098 branch = self.gitRefForBranch(branch)
3099 parent = self.gitRefForBranch(parent)
3101 if self.verbose:
3102 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3104 if len(parent) == 0 and branch in self.initialParents:
3105 parent = self.initialParents[branch]
3106 del self.initialParents[branch]
3108 blob = None
3109 if len(parent) > 0:
3110 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3111 if self.verbose:
3112 print "Creating temporary branch: " + tempBranch
3113 self.commit(description, filesForCommit, tempBranch)
3114 self.tempBranches.append(tempBranch)
3115 self.checkpoint()
3116 blob = self.searchParent(parent, branch, tempBranch)
3117 if blob:
3118 self.commit(description, filesForCommit, branch, blob)
3119 else:
3120 if self.verbose:
3121 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3122 self.commit(description, filesForCommit, branch, parent)
3123 else:
3124 files = self.extractFilesFromCommit(description)
3125 self.commit(description, files, self.branch,
3126 self.initialParent)
3127 # only needed once, to connect to the previous commit
3128 self.initialParent = ""
3129 except IOError:
3130 print self.gitError.read()
3131 sys.exit(1)
3133 def importHeadRevision(self, revision):
3134 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3136 details = {}
3137 details["user"] = "git perforce import user"
3138 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3139 % (' '.join(self.depotPaths), revision))
3140 details["change"] = revision
3141 newestRevision = 0
3143 fileCnt = 0
3144 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3146 for info in p4CmdList(["files"] + fileArgs):
3148 if 'code' in info and info['code'] == 'error':
3149 sys.stderr.write("p4 returned an error: %s\n"
3150 % info['data'])
3151 if info['data'].find("must refer to client") >= 0:
3152 sys.stderr.write("This particular p4 error is misleading.\n")
3153 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3154 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3155 sys.exit(1)
3156 if 'p4ExitCode' in info:
3157 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3158 sys.exit(1)
3161 change = int(info["change"])
3162 if change > newestRevision:
3163 newestRevision = change
3165 if info["action"] in self.delete_actions:
3166 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3167 #fileCnt = fileCnt + 1
3168 continue
3170 for prop in ["depotFile", "rev", "action", "type" ]:
3171 details["%s%s" % (prop, fileCnt)] = info[prop]
3173 fileCnt = fileCnt + 1
3175 details["change"] = newestRevision
3177 # Use time from top-most change so that all git p4 clones of
3178 # the same p4 repo have the same commit SHA1s.
3179 res = p4_describe(newestRevision)
3180 details["time"] = res["time"]
3182 self.updateOptionDict(details)
3183 try:
3184 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3185 except IOError:
3186 print "IO error with git fast-import. Is your git version recent enough?"
3187 print self.gitError.read()
3190 def run(self, args):
3191 self.depotPaths = []
3192 self.changeRange = ""
3193 self.previousDepotPaths = []
3194 self.hasOrigin = False
3196 # map from branch depot path to parent branch
3197 self.knownBranches = {}
3198 self.initialParents = {}
3200 if self.importIntoRemotes:
3201 self.refPrefix = "refs/remotes/p4/"
3202 else:
3203 self.refPrefix = "refs/heads/p4/"
3205 if self.syncWithOrigin:
3206 self.hasOrigin = originP4BranchesExist()
3207 if self.hasOrigin:
3208 if not self.silent:
3209 print 'Syncing with origin first, using "git fetch origin"'
3210 system("git fetch origin")
3212 branch_arg_given = bool(self.branch)
3213 if len(self.branch) == 0:
3214 self.branch = self.refPrefix + "master"
3215 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3216 system("git update-ref %s refs/heads/p4" % self.branch)
3217 system("git branch -D p4")
3219 # accept either the command-line option, or the configuration variable
3220 if self.useClientSpec:
3221 # will use this after clone to set the variable
3222 self.useClientSpec_from_options = True
3223 else:
3224 if gitConfigBool("git-p4.useclientspec"):
3225 self.useClientSpec = True
3226 if self.useClientSpec:
3227 self.clientSpecDirs = getClientSpec()
3229 # TODO: should always look at previous commits,
3230 # merge with previous imports, if possible.
3231 if args == []:
3232 if self.hasOrigin:
3233 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3235 # branches holds mapping from branch name to sha1
3236 branches = p4BranchesInGit(self.importIntoRemotes)
3238 # restrict to just this one, disabling detect-branches
3239 if branch_arg_given:
3240 short = self.branch.split("/")[-1]
3241 if short in branches:
3242 self.p4BranchesInGit = [ short ]
3243 else:
3244 self.p4BranchesInGit = branches.keys()
3246 if len(self.p4BranchesInGit) > 1:
3247 if not self.silent:
3248 print "Importing from/into multiple branches"
3249 self.detectBranches = True
3250 for branch in branches.keys():
3251 self.initialParents[self.refPrefix + branch] = \
3252 branches[branch]
3254 if self.verbose:
3255 print "branches: %s" % self.p4BranchesInGit
3257 p4Change = 0
3258 for branch in self.p4BranchesInGit:
3259 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3261 settings = extractSettingsGitLog(logMsg)
3263 self.readOptions(settings)
3264 if (settings.has_key('depot-paths')
3265 and settings.has_key ('change')):
3266 change = int(settings['change']) + 1
3267 p4Change = max(p4Change, change)
3269 depotPaths = sorted(settings['depot-paths'])
3270 if self.previousDepotPaths == []:
3271 self.previousDepotPaths = depotPaths
3272 else:
3273 paths = []
3274 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3275 prev_list = prev.split("/")
3276 cur_list = cur.split("/")
3277 for i in range(0, min(len(cur_list), len(prev_list))):
3278 if cur_list[i] <> prev_list[i]:
3279 i = i - 1
3280 break
3282 paths.append ("/".join(cur_list[:i + 1]))
3284 self.previousDepotPaths = paths
3286 if p4Change > 0:
3287 self.depotPaths = sorted(self.previousDepotPaths)
3288 self.changeRange = "@%s,#head" % p4Change
3289 if not self.silent and not self.detectBranches:
3290 print "Performing incremental import into %s git branch" % self.branch
3292 # accept multiple ref name abbreviations:
3293 # refs/foo/bar/branch -> use it exactly
3294 # p4/branch -> prepend refs/remotes/ or refs/heads/
3295 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3296 if not self.branch.startswith("refs/"):
3297 if self.importIntoRemotes:
3298 prepend = "refs/remotes/"
3299 else:
3300 prepend = "refs/heads/"
3301 if not self.branch.startswith("p4/"):
3302 prepend += "p4/"
3303 self.branch = prepend + self.branch
3305 if len(args) == 0 and self.depotPaths:
3306 if not self.silent:
3307 print "Depot paths: %s" % ' '.join(self.depotPaths)
3308 else:
3309 if self.depotPaths and self.depotPaths != args:
3310 print ("previous import used depot path %s and now %s was specified. "
3311 "This doesn't work!" % (' '.join (self.depotPaths),
3312 ' '.join (args)))
3313 sys.exit(1)
3315 self.depotPaths = sorted(args)
3317 revision = ""
3318 self.users = {}
3320 # Make sure no revision specifiers are used when --changesfile
3321 # is specified.
3322 bad_changesfile = False
3323 if len(self.changesFile) > 0:
3324 for p in self.depotPaths:
3325 if p.find("@") >= 0 or p.find("#") >= 0:
3326 bad_changesfile = True
3327 break
3328 if bad_changesfile:
3329 die("Option --changesfile is incompatible with revision specifiers")
3331 newPaths = []
3332 for p in self.depotPaths:
3333 if p.find("@") != -1:
3334 atIdx = p.index("@")
3335 self.changeRange = p[atIdx:]
3336 if self.changeRange == "@all":
3337 self.changeRange = ""
3338 elif ',' not in self.changeRange:
3339 revision = self.changeRange
3340 self.changeRange = ""
3341 p = p[:atIdx]
3342 elif p.find("#") != -1:
3343 hashIdx = p.index("#")
3344 revision = p[hashIdx:]
3345 p = p[:hashIdx]
3346 elif self.previousDepotPaths == []:
3347 # pay attention to changesfile, if given, else import
3348 # the entire p4 tree at the head revision
3349 if len(self.changesFile) == 0:
3350 revision = "#head"
3352 p = re.sub ("\.\.\.$", "", p)
3353 if not p.endswith("/"):
3354 p += "/"
3356 newPaths.append(p)
3358 self.depotPaths = newPaths
3360 # --detect-branches may change this for each branch
3361 self.branchPrefixes = self.depotPaths
3363 self.loadUserMapFromCache()
3364 self.labels = {}
3365 if self.detectLabels:
3366 self.getLabels();
3368 if self.detectBranches:
3369 ## FIXME - what's a P4 projectName ?
3370 self.projectName = self.guessProjectName()
3372 if self.hasOrigin:
3373 self.getBranchMappingFromGitBranches()
3374 else:
3375 self.getBranchMapping()
3376 if self.verbose:
3377 print "p4-git branches: %s" % self.p4BranchesInGit
3378 print "initial parents: %s" % self.initialParents
3379 for b in self.p4BranchesInGit:
3380 if b != "master":
3382 ## FIXME
3383 b = b[len(self.projectName):]
3384 self.createdBranches.add(b)
3386 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3388 self.importProcess = subprocess.Popen(["git", "fast-import"],
3389 stdin=subprocess.PIPE,
3390 stdout=subprocess.PIPE,
3391 stderr=subprocess.PIPE);
3392 self.gitOutput = self.importProcess.stdout
3393 self.gitStream = self.importProcess.stdin
3394 self.gitError = self.importProcess.stderr
3396 if revision:
3397 self.importHeadRevision(revision)
3398 else:
3399 changes = []
3401 if len(self.changesFile) > 0:
3402 output = open(self.changesFile).readlines()
3403 changeSet = set()
3404 for line in output:
3405 changeSet.add(int(line))
3407 for change in changeSet:
3408 changes.append(change)
3410 changes.sort()
3411 else:
3412 # catch "git p4 sync" with no new branches, in a repo that
3413 # does not have any existing p4 branches
3414 if len(args) == 0:
3415 if not self.p4BranchesInGit:
3416 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3418 # The default branch is master, unless --branch is used to
3419 # specify something else. Make sure it exists, or complain
3420 # nicely about how to use --branch.
3421 if not self.detectBranches:
3422 if not branch_exists(self.branch):
3423 if branch_arg_given:
3424 die("Error: branch %s does not exist." % self.branch)
3425 else:
3426 die("Error: no branch %s; perhaps specify one with --branch." %
3427 self.branch)
3429 if self.verbose:
3430 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3431 self.changeRange)
3432 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3434 if len(self.maxChanges) > 0:
3435 changes = changes[:min(int(self.maxChanges), len(changes))]
3437 if len(changes) == 0:
3438 if not self.silent:
3439 print "No changes to import!"
3440 else:
3441 if not self.silent and not self.detectBranches:
3442 print "Import destination: %s" % self.branch
3444 self.updatedBranches = set()
3446 if not self.detectBranches:
3447 if args:
3448 # start a new branch
3449 self.initialParent = ""
3450 else:
3451 # build on a previous revision
3452 self.initialParent = parseRevision(self.branch)
3454 self.importChanges(changes)
3456 if not self.silent:
3457 print ""
3458 if len(self.updatedBranches) > 0:
3459 sys.stdout.write("Updated branches: ")
3460 for b in self.updatedBranches:
3461 sys.stdout.write("%s " % b)
3462 sys.stdout.write("\n")
3464 if gitConfigBool("git-p4.importLabels"):
3465 self.importLabels = True
3467 if self.importLabels:
3468 p4Labels = getP4Labels(self.depotPaths)
3469 gitTags = getGitTags()
3471 missingP4Labels = p4Labels - gitTags
3472 self.importP4Labels(self.gitStream, missingP4Labels)
3474 self.gitStream.close()
3475 if self.importProcess.wait() != 0:
3476 die("fast-import failed: %s" % self.gitError.read())
3477 self.gitOutput.close()
3478 self.gitError.close()
3480 # Cleanup temporary branches created during import
3481 if self.tempBranches != []:
3482 for branch in self.tempBranches:
3483 read_pipe("git update-ref -d %s" % branch)
3484 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3486 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3487 # a convenient shortcut refname "p4".
3488 if self.importIntoRemotes:
3489 head_ref = self.refPrefix + "HEAD"
3490 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3491 system(["git", "symbolic-ref", head_ref, self.branch])
3493 return True
3495 class P4Rebase(Command):
3496 def __init__(self):
3497 Command.__init__(self)
3498 self.options = [
3499 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3501 self.importLabels = False
3502 self.description = ("Fetches the latest revision from perforce and "
3503 + "rebases the current work (branch) against it")
3505 def run(self, args):
3506 sync = P4Sync()
3507 sync.importLabels = self.importLabels
3508 sync.run([])
3510 return self.rebase()
3512 def rebase(self):
3513 if os.system("git update-index --refresh") != 0:
3514 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3515 if len(read_pipe("git diff-index HEAD --")) > 0:
3516 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3518 [upstream, settings] = findUpstreamBranchPoint()
3519 if len(upstream) == 0:
3520 die("Cannot find upstream branchpoint for rebase")
3522 # the branchpoint may be p4/foo~3, so strip off the parent
3523 upstream = re.sub("~[0-9]+$", "", upstream)
3525 print "Rebasing the current branch onto %s" % upstream
3526 oldHead = read_pipe("git rev-parse HEAD").strip()
3527 system("git rebase %s" % upstream)
3528 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3529 return True
3531 class P4Clone(P4Sync):
3532 def __init__(self):
3533 P4Sync.__init__(self)
3534 self.description = "Creates a new git repository and imports from Perforce into it"
3535 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3536 self.options += [
3537 optparse.make_option("--destination", dest="cloneDestination",
3538 action='store', default=None,
3539 help="where to leave result of the clone"),
3540 optparse.make_option("--bare", dest="cloneBare",
3541 action="store_true", default=False),
3543 self.cloneDestination = None
3544 self.needsGit = False
3545 self.cloneBare = False
3547 def defaultDestination(self, args):
3548 ## TODO: use common prefix of args?
3549 depotPath = args[0]
3550 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3551 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3552 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3553 depotDir = re.sub(r"/$", "", depotDir)
3554 return os.path.split(depotDir)[1]
3556 def run(self, args):
3557 if len(args) < 1:
3558 return False
3560 if self.keepRepoPath and not self.cloneDestination:
3561 sys.stderr.write("Must specify destination for --keep-path\n")
3562 sys.exit(1)
3564 depotPaths = args
3566 if not self.cloneDestination and len(depotPaths) > 1:
3567 self.cloneDestination = depotPaths[-1]
3568 depotPaths = depotPaths[:-1]
3570 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3571 for p in depotPaths:
3572 if not p.startswith("//"):
3573 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3574 return False
3576 if not self.cloneDestination:
3577 self.cloneDestination = self.defaultDestination(args)
3579 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3581 if not os.path.exists(self.cloneDestination):
3582 os.makedirs(self.cloneDestination)
3583 chdir(self.cloneDestination)
3585 init_cmd = [ "git", "init" ]
3586 if self.cloneBare:
3587 init_cmd.append("--bare")
3588 retcode = subprocess.call(init_cmd)
3589 if retcode:
3590 raise CalledProcessError(retcode, init_cmd)
3592 if not P4Sync.run(self, depotPaths):
3593 return False
3595 # create a master branch and check out a work tree
3596 if gitBranchExists(self.branch):
3597 system([ "git", "branch", "master", self.branch ])
3598 if not self.cloneBare:
3599 system([ "git", "checkout", "-f" ])
3600 else:
3601 print 'Not checking out any branch, use ' \
3602 '"git checkout -q -b master <branch>"'
3604 # auto-set this variable if invoked with --use-client-spec
3605 if self.useClientSpec_from_options:
3606 system("git config --bool git-p4.useclientspec true")
3608 return True
3610 class P4Branches(Command):
3611 def __init__(self):
3612 Command.__init__(self)
3613 self.options = [ ]
3614 self.description = ("Shows the git branches that hold imports and their "
3615 + "corresponding perforce depot paths")
3616 self.verbose = False
3618 def run(self, args):
3619 if originP4BranchesExist():
3620 createOrUpdateBranchesFromOrigin()
3622 cmdline = "git rev-parse --symbolic "
3623 cmdline += " --remotes"
3625 for line in read_pipe_lines(cmdline):
3626 line = line.strip()
3628 if not line.startswith('p4/') or line == "p4/HEAD":
3629 continue
3630 branch = line
3632 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3633 settings = extractSettingsGitLog(log)
3635 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3636 return True
3638 class HelpFormatter(optparse.IndentedHelpFormatter):
3639 def __init__(self):
3640 optparse.IndentedHelpFormatter.__init__(self)
3642 def format_description(self, description):
3643 if description:
3644 return description + "\n"
3645 else:
3646 return ""
3648 def printUsage(commands):
3649 print "usage: %s <command> [options]" % sys.argv[0]
3650 print ""
3651 print "valid commands: %s" % ", ".join(commands)
3652 print ""
3653 print "Try %s <command> --help for command specific help." % sys.argv[0]
3654 print ""
3656 commands = {
3657 "debug" : P4Debug,
3658 "submit" : P4Submit,
3659 "commit" : P4Submit,
3660 "sync" : P4Sync,
3661 "rebase" : P4Rebase,
3662 "clone" : P4Clone,
3663 "rollback" : P4RollBack,
3664 "branches" : P4Branches
3668 def main():
3669 if len(sys.argv[1:]) == 0:
3670 printUsage(commands.keys())
3671 sys.exit(2)
3673 cmdName = sys.argv[1]
3674 try:
3675 klass = commands[cmdName]
3676 cmd = klass()
3677 except KeyError:
3678 print "unknown command %s" % cmdName
3679 print ""
3680 printUsage(commands.keys())
3681 sys.exit(2)
3683 options = cmd.options
3684 cmd.gitdir = os.environ.get("GIT_DIR", None)
3686 args = sys.argv[2:]
3688 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3689 if cmd.needsGit:
3690 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3692 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3693 options,
3694 description = cmd.description,
3695 formatter = HelpFormatter())
3697 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3698 global verbose
3699 verbose = cmd.verbose
3700 if cmd.needsGit:
3701 if cmd.gitdir == None:
3702 cmd.gitdir = os.path.abspath(".git")
3703 if not isValidGitDir(cmd.gitdir):
3704 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3705 if os.path.exists(cmd.gitdir):
3706 cdup = read_pipe("git rev-parse --show-cdup").strip()
3707 if len(cdup) > 0:
3708 chdir(cdup);
3710 if not isValidGitDir(cmd.gitdir):
3711 if isValidGitDir(cmd.gitdir + "/.git"):
3712 cmd.gitdir += "/.git"
3713 else:
3714 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3716 os.environ["GIT_DIR"] = cmd.gitdir
3718 if not cmd.run(args):
3719 parser.print_help()
3720 sys.exit(2)
3723 if __name__ == '__main__':
3724 main()