Merge tag 'v2.14.1'
[git/raj.git] / git-p4.py
blob2fa581789c5d9d4dd967e1783c513d64660f8a9c
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
45 verbose = False
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
94 return real_cmd
96 def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
106 def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
128 def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
138 def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
145 def write_pipe(c, stdin):
146 if verbose:
147 sys.stderr.write('Writing pipe: %s\n' % str(c))
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
157 return val
159 def p4_write_pipe(c, stdin):
160 real_cmd = p4_build_cmd(c)
161 return write_pipe(real_cmd, stdin)
163 def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
168 if verbose:
169 sys.stderr.write('Reading pipe: %s\n' % str(c))
171 expand = isinstance(c,basestring)
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
174 return (p.returncode, out, err)
176 def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
187 return out
189 def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
199 def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
203 def read_pipe_lines(c):
204 if verbose:
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
210 val = pipe.readlines()
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
214 return val
216 def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd = p4_build_cmd(c)
219 return read_pipe_lines(real_cmd)
221 def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
249 def system(cmd, ignore_error=False):
250 expand = isinstance(cmd,basestring)
251 if verbose:
252 sys.stderr.write("executing %s\n" % str(cmd))
253 retcode = subprocess.call(cmd, shell=expand)
254 if retcode and not ignore_error:
255 raise CalledProcessError(retcode, cmd)
257 return retcode
259 def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
261 real_cmd = p4_build_cmd(cmd)
262 expand = isinstance(real_cmd, basestring)
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
267 _p4_version_string = None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
283 def p4_integrate(src, dest):
284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
286 def p4_sync(f, *options):
287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
289 def p4_add(f):
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
296 def p4_delete(f):
297 p4_system(["delete", wildcard_encode(f)])
299 def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
302 def p4_revert(f):
303 p4_system(["revert", wildcard_encode(f)])
305 def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
308 def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
312 def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
315 def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317 return int(results[0]['change'])
319 def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
324 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
328 d = ds[0]
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
340 return d
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers. See "p4 help filetypes"
345 # for a list and explanation.
347 def split_p4_type(p4type):
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
378 # return the raw p4 type of a file (text, text+ko, etc)
380 def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382 return results[0]['headType']
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
388 def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
400 (:[^$\n]+)? # possibly an old expansion, followed by...
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
412 def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
419 def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
423 p4Type = "+x"
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
432 p4_reopen(p4Type, file)
434 def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
437 result = p4_read_pipe(["opened", wildcard_encode(file)])
438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439 if match:
440 return match.group(1)
441 else:
442 die("Could not determine file type for %s (result: '%s')" % (file, result))
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
454 return labels
456 # Return the set of all git tags
457 def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
464 def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
471 def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
488 If the pattern is not matched, None is returned."""
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
502 return None
504 def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
509 def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
521 cmd = p4_build_cmd(cmd)
522 if verbose:
523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
536 stdin_file.flush()
537 stdin_file.seek(0)
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
544 result = []
545 try:
546 while True:
547 entry = marshal.load(p4.stdout)
548 if skip_info:
549 if 'code' in entry and entry['code'] == 'info':
550 continue
551 if cb is not None:
552 cb(entry)
553 else:
554 result.append(entry)
555 except EOFError:
556 pass
557 exitCode = p4.wait()
558 if exitCode != 0:
559 entry = {}
560 entry["p4ExitCode"] = exitCode
561 result.append(entry)
563 return result
565 def p4Cmd(cmd):
566 list = p4CmdList(cmd)
567 result = {}
568 for entry in list:
569 result.update(entry)
570 return result;
572 def p4Where(depotPath):
573 if not depotPath.endswith("/"):
574 depotPath += "/"
575 depotPathLong = depotPath + "..."
576 outputList = p4CmdList(["where", depotPathLong])
577 output = None
578 for entry in outputList:
579 if "depotFile" in entry:
580 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581 # The base path always ends with "/...".
582 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
583 output = entry
584 break
585 elif "data" in entry:
586 data = entry.get("data")
587 space = data.find(" ")
588 if data[:space] == depotPath:
589 output = entry
590 break
591 if output == None:
592 return ""
593 if output["code"] == "error":
594 return ""
595 clientPath = ""
596 if "path" in output:
597 clientPath = output.get("path")
598 elif "data" in output:
599 data = output.get("data")
600 lastSpace = data.rfind(" ")
601 clientPath = data[lastSpace + 1:]
603 if clientPath.endswith("..."):
604 clientPath = clientPath[:-3]
605 return clientPath
607 def currentGitBranch():
608 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
610 def isValidGitDir(path):
611 return git_dir(path) != None
613 def parseRevision(ref):
614 return read_pipe("git rev-parse %s" % ref).strip()
616 def branchExists(ref):
617 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618 ignore_error=True)
619 return len(rev) > 0
621 def extractLogMessageFromGitCommit(commit):
622 logMessage = ""
624 ## fixme: title is first line of commit, not 1st paragraph.
625 foundTitle = False
626 for log in read_pipe_lines("git cat-file commit %s" % commit):
627 if not foundTitle:
628 if len(log) == 1:
629 foundTitle = True
630 continue
632 logMessage += log
633 return logMessage
635 def extractSettingsGitLog(log):
636 values = {}
637 for line in log.split("\n"):
638 line = line.strip()
639 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640 if not m:
641 continue
643 assignments = m.group(1).split (':')
644 for a in assignments:
645 vals = a.split ('=')
646 key = vals[0].strip()
647 val = ('='.join (vals[1:])).strip()
648 if val.endswith ('\"') and val.startswith('"'):
649 val = val[1:-1]
651 values[key] = val
653 paths = values.get("depot-paths")
654 if not paths:
655 paths = values.get("depot-path")
656 if paths:
657 values['depot-paths'] = paths.split(',')
658 return values
660 def gitBranchExists(branch):
661 proc = subprocess.Popen(["git", "rev-parse", branch],
662 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
663 return proc.wait() == 0;
665 _gitConfig = {}
667 def gitConfig(key, typeSpecifier=None):
668 if not _gitConfig.has_key(key):
669 cmd = [ "git", "config" ]
670 if typeSpecifier:
671 cmd += [ typeSpecifier ]
672 cmd += [ key ]
673 s = read_pipe(cmd, ignore_error=True)
674 _gitConfig[key] = s.strip()
675 return _gitConfig[key]
677 def gitConfigBool(key):
678 """Return a bool, using git config --bool. It is True only if the
679 variable is set to true, and False if set to false or not present
680 in the config."""
682 if not _gitConfig.has_key(key):
683 _gitConfig[key] = gitConfig(key, '--bool') == "true"
684 return _gitConfig[key]
686 def gitConfigInt(key):
687 if not _gitConfig.has_key(key):
688 cmd = [ "git", "config", "--int", key ]
689 s = read_pipe(cmd, ignore_error=True)
690 v = s.strip()
691 try:
692 _gitConfig[key] = int(gitConfig(key, '--int'))
693 except ValueError:
694 _gitConfig[key] = None
695 return _gitConfig[key]
697 def gitConfigList(key):
698 if not _gitConfig.has_key(key):
699 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
700 _gitConfig[key] = s.strip().splitlines()
701 if _gitConfig[key] == ['']:
702 _gitConfig[key] = []
703 return _gitConfig[key]
705 def p4BranchesInGit(branchesAreInRemotes=True):
706 """Find all the branches whose names start with "p4/", looking
707 in remotes or heads as specified by the argument. Return
708 a dictionary of { branch: revision } for each one found.
709 The branch names are the short names, without any
710 "p4/" prefix."""
712 branches = {}
714 cmdline = "git rev-parse --symbolic "
715 if branchesAreInRemotes:
716 cmdline += "--remotes"
717 else:
718 cmdline += "--branches"
720 for line in read_pipe_lines(cmdline):
721 line = line.strip()
723 # only import to p4/
724 if not line.startswith('p4/'):
725 continue
726 # special symbolic ref to p4/master
727 if line == "p4/HEAD":
728 continue
730 # strip off p4/ prefix
731 branch = line[len("p4/"):]
733 branches[branch] = parseRevision(line)
735 return branches
737 def branch_exists(branch):
738 """Make sure that the given ref name really exists."""
740 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742 out, _ = p.communicate()
743 if p.returncode:
744 return False
745 # expect exactly one line of output: the branch name
746 return out.rstrip() == branch
748 def findUpstreamBranchPoint(head = "HEAD"):
749 branches = p4BranchesInGit()
750 # map from depot-path to branch name
751 branchByDepotPath = {}
752 for branch in branches.keys():
753 tip = branches[branch]
754 log = extractLogMessageFromGitCommit(tip)
755 settings = extractSettingsGitLog(log)
756 if settings.has_key("depot-paths"):
757 paths = ",".join(settings["depot-paths"])
758 branchByDepotPath[paths] = "remotes/p4/" + branch
760 settings = None
761 parent = 0
762 while parent < 65535:
763 commit = head + "~%s" % parent
764 log = extractLogMessageFromGitCommit(commit)
765 settings = extractSettingsGitLog(log)
766 if settings.has_key("depot-paths"):
767 paths = ",".join(settings["depot-paths"])
768 if branchByDepotPath.has_key(paths):
769 return [branchByDepotPath[paths], settings]
771 parent = parent + 1
773 return ["", settings]
775 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776 if not silent:
777 print ("Creating/updating branch(es) in %s based on origin branch(es)"
778 % localRefPrefix)
780 originPrefix = "origin/p4/"
782 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783 line = line.strip()
784 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785 continue
787 headName = line[len(originPrefix):]
788 remoteHead = localRefPrefix + headName
789 originHead = line
791 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792 if (not original.has_key('depot-paths')
793 or not original.has_key('change')):
794 continue
796 update = False
797 if not gitBranchExists(remoteHead):
798 if verbose:
799 print "creating %s" % remoteHead
800 update = True
801 else:
802 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803 if settings.has_key('change') > 0:
804 if settings['depot-paths'] == original['depot-paths']:
805 originP4Change = int(original['change'])
806 p4Change = int(settings['change'])
807 if originP4Change > p4Change:
808 print ("%s (%s) is newer than %s (%s). "
809 "Updating p4 branch from origin."
810 % (originHead, originP4Change,
811 remoteHead, p4Change))
812 update = True
813 else:
814 print ("Ignoring: %s was imported from %s while "
815 "%s was imported from %s"
816 % (originHead, ','.join(original['depot-paths']),
817 remoteHead, ','.join(settings['depot-paths'])))
819 if update:
820 system("git update-ref %s %s" % (remoteHead, originHead))
822 def originP4BranchesExist():
823 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
826 def p4ParseNumericChangeRange(parts):
827 changeStart = int(parts[0][1:])
828 if parts[1] == '#head':
829 changeEnd = p4_last_change()
830 else:
831 changeEnd = int(parts[1])
833 return (changeStart, changeEnd)
835 def chooseBlockSize(blockSize):
836 if blockSize:
837 return blockSize
838 else:
839 return defaultBlockSize
841 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
842 assert depotPaths
844 # Parse the change range into start and end. Try to find integer
845 # revision ranges as these can be broken up into blocks to avoid
846 # hitting server-side limits (maxrows, maxscanresults). But if
847 # that doesn't work, fall back to using the raw revision specifier
848 # strings, without using block mode.
850 if changeRange is None or changeRange == '':
851 changeStart = 1
852 changeEnd = p4_last_change()
853 block_size = chooseBlockSize(requestedBlockSize)
854 else:
855 parts = changeRange.split(',')
856 assert len(parts) == 2
857 try:
858 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859 block_size = chooseBlockSize(requestedBlockSize)
860 except:
861 changeStart = parts[0][1:]
862 changeEnd = parts[1]
863 if requestedBlockSize:
864 die("cannot use --changes-block-size with non-numeric revisions")
865 block_size = None
867 changes = set()
869 # Retrieve changes a block at a time, to prevent running
870 # into a MaxResults/MaxScanRows error from the server.
872 while True:
873 cmd = ['changes']
875 if block_size:
876 end = min(changeEnd, changeStart + block_size)
877 revisionRange = "%d,%d" % (changeStart, end)
878 else:
879 revisionRange = "%s,%s" % (changeStart, changeEnd)
881 for p in depotPaths:
882 cmd += ["%s...@%s" % (p, revisionRange)]
884 # Insert changes in chronological order
885 for entry in reversed(p4CmdList(cmd)):
886 if entry.has_key('p4ExitCode'):
887 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888 if not entry.has_key('change'):
889 continue
890 changes.add(int(entry['change']))
892 if not block_size:
893 break
895 if end >= changeEnd:
896 break
898 changeStart = end + 1
900 changes = sorted(changes)
901 return changes
903 def p4PathStartsWith(path, prefix):
904 # This method tries to remedy a potential mixed-case issue:
906 # If UserA adds //depot/DirA/file1
907 # and UserB adds //depot/dira/file2
909 # we may or may not have a problem. If you have core.ignorecase=true,
910 # we treat DirA and dira as the same directory
911 if gitConfigBool("core.ignorecase"):
912 return path.lower().startswith(prefix.lower())
913 return path.startswith(prefix)
915 def getClientSpec():
916 """Look at the p4 client spec, create a View() object that contains
917 all the mappings, and return it."""
919 specList = p4CmdList("client -o")
920 if len(specList) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' %
922 len(specList))
924 # dictionary of all client parameters
925 entry = specList[0]
927 # the //client/ name
928 client_name = entry["Client"]
930 # just the keys that start with "View"
931 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
933 # hold this new View
934 view = View(client_name)
936 # append the lines, in order, to the view
937 for view_num in range(len(view_keys)):
938 k = "View%d" % view_num
939 if k not in view_keys:
940 die("Expected view key %s missing" % k)
941 view.append(entry[k])
943 return view
945 def getClientRoot():
946 """Grab the client directory."""
948 output = p4CmdList("client -o")
949 if len(output) != 1:
950 die('Output from "client -o" is %d lines, expecting 1' % len(output))
952 entry = output[0]
953 if "Root" not in entry:
954 die('Client has no "Root"')
956 return entry["Root"]
959 # P4 wildcards are not allowed in filenames. P4 complains
960 # if you simply add them, but you can force it with "-f", in
961 # which case it translates them into %xx encoding internally.
963 def wildcard_decode(path):
964 # Search for and fix just these four characters. Do % last so
965 # that fixing it does not inadvertently create new %-escapes.
966 # Cannot have * in a filename in windows; untested as to
967 # what p4 would do in such a case.
968 if not platform.system() == "Windows":
969 path = path.replace("%2A", "*")
970 path = path.replace("%23", "#") \
971 .replace("%40", "@") \
972 .replace("%25", "%")
973 return path
975 def wildcard_encode(path):
976 # do % first to avoid double-encoding the %s introduced here
977 path = path.replace("%", "%25") \
978 .replace("*", "%2A") \
979 .replace("#", "%23") \
980 .replace("@", "%40")
981 return path
983 def wildcard_present(path):
984 m = re.search("[*#@%]", path)
985 return m is not None
987 class LargeFileSystem(object):
988 """Base class for large file system support."""
990 def __init__(self, writeToGitStream):
991 self.largeFiles = set()
992 self.writeToGitStream = writeToGitStream
994 def generatePointer(self, cloneDestination, contentFile):
995 """Return the content of a pointer file that is stored in Git instead of
996 the actual content."""
997 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
999 def pushFile(self, localLargeFile):
1000 """Push the actual content which is not stored in the Git repository to
1001 a server."""
1002 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1004 def hasLargeFileExtension(self, relPath):
1005 return reduce(
1006 lambda a, b: a or b,
1007 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008 False
1011 def generateTempFile(self, contents):
1012 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013 for d in contents:
1014 contentFile.write(d)
1015 contentFile.close()
1016 return contentFile.name
1018 def exceedsLargeFileThreshold(self, relPath, contents):
1019 if gitConfigInt('git-p4.largeFileThreshold'):
1020 contentsSize = sum(len(d) for d in contents)
1021 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022 return True
1023 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024 contentsSize = sum(len(d) for d in contents)
1025 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026 return False
1027 contentTempFile = self.generateTempFile(contents)
1028 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031 zf.close()
1032 compressedContentsSize = zf.infolist()[0].compress_size
1033 os.remove(contentTempFile)
1034 os.remove(compressedContentFile.name)
1035 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036 return True
1037 return False
1039 def addLargeFile(self, relPath):
1040 self.largeFiles.add(relPath)
1042 def removeLargeFile(self, relPath):
1043 self.largeFiles.remove(relPath)
1045 def isLargeFile(self, relPath):
1046 return relPath in self.largeFiles
1048 def processContent(self, git_mode, relPath, contents):
1049 """Processes the content of git fast import. This method decides if a
1050 file is stored in the large file system and handles all necessary
1051 steps."""
1052 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053 contentTempFile = self.generateTempFile(contents)
1054 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055 if pointer_git_mode:
1056 git_mode = pointer_git_mode
1057 if localLargeFile:
1058 # Move temp file to final location in large file system
1059 largeFileDir = os.path.dirname(localLargeFile)
1060 if not os.path.isdir(largeFileDir):
1061 os.makedirs(largeFileDir)
1062 shutil.move(contentTempFile, localLargeFile)
1063 self.addLargeFile(relPath)
1064 if gitConfigBool('git-p4.largeFilePush'):
1065 self.pushFile(localLargeFile)
1066 if verbose:
1067 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1068 return (git_mode, contents)
1070 class MockLFS(LargeFileSystem):
1071 """Mock large file system for testing."""
1073 def generatePointer(self, contentFile):
1074 """The pointer content is the original content prefixed with "pointer-".
1075 The local filename of the large file storage is derived from the file content.
1077 with open(contentFile, 'r') as f:
1078 content = next(f)
1079 gitMode = '100644'
1080 pointerContents = 'pointer-' + content
1081 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082 return (gitMode, pointerContents, localLargeFile)
1084 def pushFile(self, localLargeFile):
1085 """The remote filename of the large file storage is the same as the local
1086 one but in a different directory.
1088 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089 if not os.path.exists(remotePath):
1090 os.makedirs(remotePath)
1091 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1093 class GitLFS(LargeFileSystem):
1094 """Git LFS as backend for the git-p4 large file system.
1095 See https://git-lfs.github.com/ for details."""
1097 def __init__(self, *args):
1098 LargeFileSystem.__init__(self, *args)
1099 self.baseGitAttributes = []
1101 def generatePointer(self, contentFile):
1102 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103 mode and content which is stored in the Git repository instead of
1104 the actual content. Return also the new location of the actual
1105 content.
1107 if os.path.getsize(contentFile) == 0:
1108 return (None, '', None)
1110 pointerProcess = subprocess.Popen(
1111 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112 stdout=subprocess.PIPE
1114 pointerFile = pointerProcess.stdout.read()
1115 if pointerProcess.wait():
1116 os.remove(contentFile)
1117 die('git-lfs pointer command failed. Did you install the extension?')
1119 # Git LFS removed the preamble in the output of the 'pointer' command
1120 # starting from version 1.2.0. Check for the preamble here to support
1121 # earlier versions.
1122 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123 if pointerFile.startswith('Git LFS pointer for'):
1124 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1126 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1127 localLargeFile = os.path.join(
1128 os.getcwd(),
1129 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130 oid,
1132 # LFS Spec states that pointer files should not have the executable bit set.
1133 gitMode = '100644'
1134 return (gitMode, pointerFile, localLargeFile)
1136 def pushFile(self, localLargeFile):
1137 uploadProcess = subprocess.Popen(
1138 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1140 if uploadProcess.wait():
1141 die('git-lfs push command failed. Did you define a remote?')
1143 def generateGitAttributes(self):
1144 return (
1145 self.baseGitAttributes +
1147 '\n',
1148 '#\n',
1149 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 '#\n',
1152 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1153 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1155 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1156 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1160 def addLargeFile(self, relPath):
1161 LargeFileSystem.addLargeFile(self, relPath)
1162 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1164 def removeLargeFile(self, relPath):
1165 LargeFileSystem.removeLargeFile(self, relPath)
1166 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1168 def processContent(self, git_mode, relPath, contents):
1169 if relPath == '.gitattributes':
1170 self.baseGitAttributes = contents
1171 return (git_mode, self.generateGitAttributes())
1172 else:
1173 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1175 class Command:
1176 def __init__(self):
1177 self.usage = "usage: %prog [options]"
1178 self.needsGit = True
1179 self.verbose = False
1181 class P4UserMap:
1182 def __init__(self):
1183 self.userMapFromPerforceServer = False
1184 self.myP4UserId = None
1186 def p4UserId(self):
1187 if self.myP4UserId:
1188 return self.myP4UserId
1190 results = p4CmdList("user -o")
1191 for r in results:
1192 if r.has_key('User'):
1193 self.myP4UserId = r['User']
1194 return r['User']
1195 die("Could not find your p4 user id")
1197 def p4UserIsMe(self, p4User):
1198 # return True if the given p4 user is actually me
1199 me = self.p4UserId()
1200 if not p4User or p4User != me:
1201 return False
1202 else:
1203 return True
1205 def getUserCacheFilename(self):
1206 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1207 return home + "/.gitp4-usercache.txt"
1209 def getUserMapFromPerforceServer(self):
1210 if self.userMapFromPerforceServer:
1211 return
1212 self.users = {}
1213 self.emails = {}
1215 for output in p4CmdList("users"):
1216 if not output.has_key("User"):
1217 continue
1218 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1219 self.emails[output["Email"]] = output["User"]
1221 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1222 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1223 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1224 if mapUser and len(mapUser[0]) == 3:
1225 user = mapUser[0][0]
1226 fullname = mapUser[0][1]
1227 email = mapUser[0][2]
1228 self.users[user] = fullname + " <" + email + ">"
1229 self.emails[email] = user
1231 s = ''
1232 for (key, val) in self.users.items():
1233 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1235 open(self.getUserCacheFilename(), "wb").write(s)
1236 self.userMapFromPerforceServer = True
1238 def loadUserMapFromCache(self):
1239 self.users = {}
1240 self.userMapFromPerforceServer = False
1241 try:
1242 cache = open(self.getUserCacheFilename(), "rb")
1243 lines = cache.readlines()
1244 cache.close()
1245 for line in lines:
1246 entry = line.strip().split("\t")
1247 self.users[entry[0]] = entry[1]
1248 except IOError:
1249 self.getUserMapFromPerforceServer()
1251 class P4Debug(Command):
1252 def __init__(self):
1253 Command.__init__(self)
1254 self.options = []
1255 self.description = "A tool to debug the output of p4 -G."
1256 self.needsGit = False
1258 def run(self, args):
1259 j = 0
1260 for output in p4CmdList(args):
1261 print 'Element: %d' % j
1262 j += 1
1263 print output
1264 return True
1266 class P4RollBack(Command):
1267 def __init__(self):
1268 Command.__init__(self)
1269 self.options = [
1270 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1272 self.description = "A tool to debug the multi-branch import. Don't use :)"
1273 self.rollbackLocalBranches = False
1275 def run(self, args):
1276 if len(args) != 1:
1277 return False
1278 maxChange = int(args[0])
1280 if "p4ExitCode" in p4Cmd("changes -m 1"):
1281 die("Problems executing p4");
1283 if self.rollbackLocalBranches:
1284 refPrefix = "refs/heads/"
1285 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1286 else:
1287 refPrefix = "refs/remotes/"
1288 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1290 for line in lines:
1291 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1292 line = line.strip()
1293 ref = refPrefix + line
1294 log = extractLogMessageFromGitCommit(ref)
1295 settings = extractSettingsGitLog(log)
1297 depotPaths = settings['depot-paths']
1298 change = settings['change']
1300 changed = False
1302 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1303 for p in depotPaths]))) == 0:
1304 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1305 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1306 continue
1308 while change and int(change) > maxChange:
1309 changed = True
1310 if self.verbose:
1311 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1312 system("git update-ref %s \"%s^\"" % (ref, ref))
1313 log = extractLogMessageFromGitCommit(ref)
1314 settings = extractSettingsGitLog(log)
1317 depotPaths = settings['depot-paths']
1318 change = settings['change']
1320 if changed:
1321 print "%s rewound to %s" % (ref, change)
1323 return True
1325 class P4Submit(Command, P4UserMap):
1327 conflict_behavior_choices = ("ask", "skip", "quit")
1329 def __init__(self):
1330 Command.__init__(self)
1331 P4UserMap.__init__(self)
1332 self.options = [
1333 optparse.make_option("--origin", dest="origin"),
1334 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1335 # preserve the user, requires relevant p4 permissions
1336 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1337 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1338 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1339 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1340 optparse.make_option("--conflict", dest="conflict_behavior",
1341 choices=self.conflict_behavior_choices),
1342 optparse.make_option("--branch", dest="branch"),
1343 optparse.make_option("--shelve", dest="shelve", action="store_true",
1344 help="Shelve instead of submit. Shelved files are reverted, "
1345 "restoring the workspace to the state before the shelve"),
1346 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1347 metavar="CHANGELIST",
1348 help="update an existing shelved changelist, implies --shelve")
1350 self.description = "Submit changes from git to the perforce depot."
1351 self.usage += " [name of git branch to submit into perforce depot]"
1352 self.origin = ""
1353 self.detectRenames = False
1354 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1355 self.dry_run = False
1356 self.shelve = False
1357 self.update_shelve = None
1358 self.prepare_p4_only = False
1359 self.conflict_behavior = None
1360 self.isWindows = (platform.system() == "Windows")
1361 self.exportLabels = False
1362 self.p4HasMoveCommand = p4_has_move_command()
1363 self.branch = None
1365 if gitConfig('git-p4.largeFileSystem'):
1366 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1368 def check(self):
1369 if len(p4CmdList("opened ...")) > 0:
1370 die("You have files opened with perforce! Close them before starting the sync.")
1372 def separate_jobs_from_description(self, message):
1373 """Extract and return a possible Jobs field in the commit
1374 message. It goes into a separate section in the p4 change
1375 specification.
1377 A jobs line starts with "Jobs:" and looks like a new field
1378 in a form. Values are white-space separated on the same
1379 line or on following lines that start with a tab.
1381 This does not parse and extract the full git commit message
1382 like a p4 form. It just sees the Jobs: line as a marker
1383 to pass everything from then on directly into the p4 form,
1384 but outside the description section.
1386 Return a tuple (stripped log message, jobs string)."""
1388 m = re.search(r'^Jobs:', message, re.MULTILINE)
1389 if m is None:
1390 return (message, None)
1392 jobtext = message[m.start():]
1393 stripped_message = message[:m.start()].rstrip()
1394 return (stripped_message, jobtext)
1396 def prepareLogMessage(self, template, message, jobs):
1397 """Edits the template returned from "p4 change -o" to insert
1398 the message in the Description field, and the jobs text in
1399 the Jobs field."""
1400 result = ""
1402 inDescriptionSection = False
1404 for line in template.split("\n"):
1405 if line.startswith("#"):
1406 result += line + "\n"
1407 continue
1409 if inDescriptionSection:
1410 if line.startswith("Files:") or line.startswith("Jobs:"):
1411 inDescriptionSection = False
1412 # insert Jobs section
1413 if jobs:
1414 result += jobs + "\n"
1415 else:
1416 continue
1417 else:
1418 if line.startswith("Description:"):
1419 inDescriptionSection = True
1420 line += "\n"
1421 for messageLine in message.split("\n"):
1422 line += "\t" + messageLine + "\n"
1424 result += line + "\n"
1426 return result
1428 def patchRCSKeywords(self, file, pattern):
1429 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1430 (handle, outFileName) = tempfile.mkstemp(dir='.')
1431 try:
1432 outFile = os.fdopen(handle, "w+")
1433 inFile = open(file, "r")
1434 regexp = re.compile(pattern, re.VERBOSE)
1435 for line in inFile.readlines():
1436 line = regexp.sub(r'$\1$', line)
1437 outFile.write(line)
1438 inFile.close()
1439 outFile.close()
1440 # Forcibly overwrite the original file
1441 os.unlink(file)
1442 shutil.move(outFileName, file)
1443 except:
1444 # cleanup our temporary file
1445 os.unlink(outFileName)
1446 print "Failed to strip RCS keywords in %s" % file
1447 raise
1449 print "Patched up RCS keywords in %s" % file
1451 def p4UserForCommit(self,id):
1452 # Return the tuple (perforce user,git email) for a given git commit id
1453 self.getUserMapFromPerforceServer()
1454 gitEmail = read_pipe(["git", "log", "--max-count=1",
1455 "--format=%ae", id])
1456 gitEmail = gitEmail.strip()
1457 if not self.emails.has_key(gitEmail):
1458 return (None,gitEmail)
1459 else:
1460 return (self.emails[gitEmail],gitEmail)
1462 def checkValidP4Users(self,commits):
1463 # check if any git authors cannot be mapped to p4 users
1464 for id in commits:
1465 (user,email) = self.p4UserForCommit(id)
1466 if not user:
1467 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1468 if gitConfigBool("git-p4.allowMissingP4Users"):
1469 print "%s" % msg
1470 else:
1471 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1473 def lastP4Changelist(self):
1474 # Get back the last changelist number submitted in this client spec. This
1475 # then gets used to patch up the username in the change. If the same
1476 # client spec is being used by multiple processes then this might go
1477 # wrong.
1478 results = p4CmdList("client -o") # find the current client
1479 client = None
1480 for r in results:
1481 if r.has_key('Client'):
1482 client = r['Client']
1483 break
1484 if not client:
1485 die("could not get client spec")
1486 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1487 for r in results:
1488 if r.has_key('change'):
1489 return r['change']
1490 die("Could not get changelist number for last submit - cannot patch up user details")
1492 def modifyChangelistUser(self, changelist, newUser):
1493 # fixup the user field of a changelist after it has been submitted.
1494 changes = p4CmdList("change -o %s" % changelist)
1495 if len(changes) != 1:
1496 die("Bad output from p4 change modifying %s to user %s" %
1497 (changelist, newUser))
1499 c = changes[0]
1500 if c['User'] == newUser: return # nothing to do
1501 c['User'] = newUser
1502 input = marshal.dumps(c)
1504 result = p4CmdList("change -f -i", stdin=input)
1505 for r in result:
1506 if r.has_key('code'):
1507 if r['code'] == 'error':
1508 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1509 if r.has_key('data'):
1510 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1511 return
1512 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1514 def canChangeChangelists(self):
1515 # check to see if we have p4 admin or super-user permissions, either of
1516 # which are required to modify changelists.
1517 results = p4CmdList(["protects", self.depotPath])
1518 for r in results:
1519 if r.has_key('perm'):
1520 if r['perm'] == 'admin':
1521 return 1
1522 if r['perm'] == 'super':
1523 return 1
1524 return 0
1526 def prepareSubmitTemplate(self, changelist=None):
1527 """Run "p4 change -o" to grab a change specification template.
1528 This does not use "p4 -G", as it is nice to keep the submission
1529 template in original order, since a human might edit it.
1531 Remove lines in the Files section that show changes to files
1532 outside the depot path we're committing into."""
1534 [upstream, settings] = findUpstreamBranchPoint()
1536 template = """\
1537 # A Perforce Change Specification.
1539 # Change: The change number. 'new' on a new changelist.
1540 # Date: The date this specification was last modified.
1541 # Client: The client on which the changelist was created. Read-only.
1542 # User: The user who created the changelist.
1543 # Status: Either 'pending' or 'submitted'. Read-only.
1544 # Type: Either 'public' or 'restricted'. Default is 'public'.
1545 # Description: Comments about the changelist. Required.
1546 # Jobs: What opened jobs are to be closed by this changelist.
1547 # You may delete jobs from this list. (New changelists only.)
1548 # Files: What opened files from the default changelist are to be added
1549 # to this changelist. You may delete files from this list.
1550 # (New changelists only.)
1552 files_list = []
1553 inFilesSection = False
1554 change_entry = None
1555 args = ['change', '-o']
1556 if changelist:
1557 args.append(str(changelist))
1558 for entry in p4CmdList(args):
1559 if not entry.has_key('code'):
1560 continue
1561 if entry['code'] == 'stat':
1562 change_entry = entry
1563 break
1564 if not change_entry:
1565 die('Failed to decode output of p4 change -o')
1566 for key, value in change_entry.iteritems():
1567 if key.startswith('File'):
1568 if settings.has_key('depot-paths'):
1569 if not [p for p in settings['depot-paths']
1570 if p4PathStartsWith(value, p)]:
1571 continue
1572 else:
1573 if not p4PathStartsWith(value, self.depotPath):
1574 continue
1575 files_list.append(value)
1576 continue
1577 # Output in the order expected by prepareLogMessage
1578 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1579 if not change_entry.has_key(key):
1580 continue
1581 template += '\n'
1582 template += key + ':'
1583 if key == 'Description':
1584 template += '\n'
1585 for field_line in change_entry[key].splitlines():
1586 template += '\t'+field_line+'\n'
1587 if len(files_list) > 0:
1588 template += '\n'
1589 template += 'Files:\n'
1590 for path in files_list:
1591 template += '\t'+path+'\n'
1592 return template
1594 def edit_template(self, template_file):
1595 """Invoke the editor to let the user change the submission
1596 message. Return true if okay to continue with the submit."""
1598 # if configured to skip the editing part, just submit
1599 if gitConfigBool("git-p4.skipSubmitEdit"):
1600 return True
1602 # look at the modification time, to check later if the user saved
1603 # the file
1604 mtime = os.stat(template_file).st_mtime
1606 # invoke the editor
1607 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1608 editor = os.environ.get("P4EDITOR")
1609 else:
1610 editor = read_pipe("git var GIT_EDITOR").strip()
1611 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1613 # If the file was not saved, prompt to see if this patch should
1614 # be skipped. But skip this verification step if configured so.
1615 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1616 return True
1618 # modification time updated means user saved the file
1619 if os.stat(template_file).st_mtime > mtime:
1620 return True
1622 while True:
1623 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1624 if response == 'y':
1625 return True
1626 if response == 'n':
1627 return False
1629 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1630 # diff
1631 if os.environ.has_key("P4DIFF"):
1632 del(os.environ["P4DIFF"])
1633 diff = ""
1634 for editedFile in editedFiles:
1635 diff += p4_read_pipe(['diff', '-du',
1636 wildcard_encode(editedFile)])
1638 # new file diff
1639 newdiff = ""
1640 for newFile in filesToAdd:
1641 newdiff += "==== new file ====\n"
1642 newdiff += "--- /dev/null\n"
1643 newdiff += "+++ %s\n" % newFile
1645 is_link = os.path.islink(newFile)
1646 expect_link = newFile in symlinks
1648 if is_link and expect_link:
1649 newdiff += "+%s\n" % os.readlink(newFile)
1650 else:
1651 f = open(newFile, "r")
1652 for line in f.readlines():
1653 newdiff += "+" + line
1654 f.close()
1656 return (diff + newdiff).replace('\r\n', '\n')
1658 def applyCommit(self, id):
1659 """Apply one commit, return True if it succeeded."""
1661 print "Applying", read_pipe(["git", "show", "-s",
1662 "--format=format:%h %s", id])
1664 (p4User, gitEmail) = self.p4UserForCommit(id)
1666 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1667 filesToAdd = set()
1668 filesToChangeType = set()
1669 filesToDelete = set()
1670 editedFiles = set()
1671 pureRenameCopy = set()
1672 symlinks = set()
1673 filesToChangeExecBit = {}
1674 all_files = list()
1676 for line in diff:
1677 diff = parseDiffTreeEntry(line)
1678 modifier = diff['status']
1679 path = diff['src']
1680 all_files.append(path)
1682 if modifier == "M":
1683 p4_edit(path)
1684 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1685 filesToChangeExecBit[path] = diff['dst_mode']
1686 editedFiles.add(path)
1687 elif modifier == "A":
1688 filesToAdd.add(path)
1689 filesToChangeExecBit[path] = diff['dst_mode']
1690 if path in filesToDelete:
1691 filesToDelete.remove(path)
1693 dst_mode = int(diff['dst_mode'], 8)
1694 if dst_mode == 0120000:
1695 symlinks.add(path)
1697 elif modifier == "D":
1698 filesToDelete.add(path)
1699 if path in filesToAdd:
1700 filesToAdd.remove(path)
1701 elif modifier == "C":
1702 src, dest = diff['src'], diff['dst']
1703 p4_integrate(src, dest)
1704 pureRenameCopy.add(dest)
1705 if diff['src_sha1'] != diff['dst_sha1']:
1706 p4_edit(dest)
1707 pureRenameCopy.discard(dest)
1708 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1709 p4_edit(dest)
1710 pureRenameCopy.discard(dest)
1711 filesToChangeExecBit[dest] = diff['dst_mode']
1712 if self.isWindows:
1713 # turn off read-only attribute
1714 os.chmod(dest, stat.S_IWRITE)
1715 os.unlink(dest)
1716 editedFiles.add(dest)
1717 elif modifier == "R":
1718 src, dest = diff['src'], diff['dst']
1719 if self.p4HasMoveCommand:
1720 p4_edit(src) # src must be open before move
1721 p4_move(src, dest) # opens for (move/delete, move/add)
1722 else:
1723 p4_integrate(src, dest)
1724 if diff['src_sha1'] != diff['dst_sha1']:
1725 p4_edit(dest)
1726 else:
1727 pureRenameCopy.add(dest)
1728 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1729 if not self.p4HasMoveCommand:
1730 p4_edit(dest) # with move: already open, writable
1731 filesToChangeExecBit[dest] = diff['dst_mode']
1732 if not self.p4HasMoveCommand:
1733 if self.isWindows:
1734 os.chmod(dest, stat.S_IWRITE)
1735 os.unlink(dest)
1736 filesToDelete.add(src)
1737 editedFiles.add(dest)
1738 elif modifier == "T":
1739 filesToChangeType.add(path)
1740 else:
1741 die("unknown modifier %s for %s" % (modifier, path))
1743 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1744 patchcmd = diffcmd + " | git apply "
1745 tryPatchCmd = patchcmd + "--check -"
1746 applyPatchCmd = patchcmd + "--check --apply -"
1747 patch_succeeded = True
1749 if os.system(tryPatchCmd) != 0:
1750 fixed_rcs_keywords = False
1751 patch_succeeded = False
1752 print "Unfortunately applying the change failed!"
1754 # Patch failed, maybe it's just RCS keyword woes. Look through
1755 # the patch to see if that's possible.
1756 if gitConfigBool("git-p4.attemptRCSCleanup"):
1757 file = None
1758 pattern = None
1759 kwfiles = {}
1760 for file in editedFiles | filesToDelete:
1761 # did this file's delta contain RCS keywords?
1762 pattern = p4_keywords_regexp_for_file(file)
1764 if pattern:
1765 # this file is a possibility...look for RCS keywords.
1766 regexp = re.compile(pattern, re.VERBOSE)
1767 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1768 if regexp.search(line):
1769 if verbose:
1770 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1771 kwfiles[file] = pattern
1772 break
1774 for file in kwfiles:
1775 if verbose:
1776 print "zapping %s with %s" % (line,pattern)
1777 # File is being deleted, so not open in p4. Must
1778 # disable the read-only bit on windows.
1779 if self.isWindows and file not in editedFiles:
1780 os.chmod(file, stat.S_IWRITE)
1781 self.patchRCSKeywords(file, kwfiles[file])
1782 fixed_rcs_keywords = True
1784 if fixed_rcs_keywords:
1785 print "Retrying the patch with RCS keywords cleaned up"
1786 if os.system(tryPatchCmd) == 0:
1787 patch_succeeded = True
1789 if not patch_succeeded:
1790 for f in editedFiles:
1791 p4_revert(f)
1792 return False
1795 # Apply the patch for real, and do add/delete/+x handling.
1797 system(applyPatchCmd)
1799 for f in filesToChangeType:
1800 p4_edit(f, "-t", "auto")
1801 for f in filesToAdd:
1802 p4_add(f)
1803 for f in filesToDelete:
1804 p4_revert(f)
1805 p4_delete(f)
1807 # Set/clear executable bits
1808 for f in filesToChangeExecBit.keys():
1809 mode = filesToChangeExecBit[f]
1810 setP4ExecBit(f, mode)
1812 if self.update_shelve:
1813 print("all_files = %s" % str(all_files))
1814 p4_reopen_in_change(self.update_shelve, all_files)
1817 # Build p4 change description, starting with the contents
1818 # of the git commit message.
1820 logMessage = extractLogMessageFromGitCommit(id)
1821 logMessage = logMessage.strip()
1822 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1824 template = self.prepareSubmitTemplate(self.update_shelve)
1825 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1827 if self.preserveUser:
1828 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1830 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1831 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1832 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1833 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1835 separatorLine = "######## everything below this line is just the diff #######\n"
1836 if not self.prepare_p4_only:
1837 submitTemplate += separatorLine
1838 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1840 (handle, fileName) = tempfile.mkstemp()
1841 tmpFile = os.fdopen(handle, "w+b")
1842 if self.isWindows:
1843 submitTemplate = submitTemplate.replace("\n", "\r\n")
1844 tmpFile.write(submitTemplate)
1845 tmpFile.close()
1847 if self.prepare_p4_only:
1849 # Leave the p4 tree prepared, and the submit template around
1850 # and let the user decide what to do next
1852 print
1853 print "P4 workspace prepared for submission."
1854 print "To submit or revert, go to client workspace"
1855 print " " + self.clientPath
1856 print
1857 print "To submit, use \"p4 submit\" to write a new description,"
1858 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1859 " \"git p4\"." % fileName
1860 print "You can delete the file \"%s\" when finished." % fileName
1862 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1863 print "To preserve change ownership by user %s, you must\n" \
1864 "do \"p4 change -f <change>\" after submitting and\n" \
1865 "edit the User field."
1866 if pureRenameCopy:
1867 print "After submitting, renamed files must be re-synced."
1868 print "Invoke \"p4 sync -f\" on each of these files:"
1869 for f in pureRenameCopy:
1870 print " " + f
1872 print
1873 print "To revert the changes, use \"p4 revert ...\", and delete"
1874 print "the submit template file \"%s\"" % fileName
1875 if filesToAdd:
1876 print "Since the commit adds new files, they must be deleted:"
1877 for f in filesToAdd:
1878 print " " + f
1879 print
1880 return True
1883 # Let the user edit the change description, then submit it.
1885 submitted = False
1887 try:
1888 if self.edit_template(fileName):
1889 # read the edited message and submit
1890 tmpFile = open(fileName, "rb")
1891 message = tmpFile.read()
1892 tmpFile.close()
1893 if self.isWindows:
1894 message = message.replace("\r\n", "\n")
1895 submitTemplate = message[:message.index(separatorLine)]
1897 if self.update_shelve:
1898 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1899 elif self.shelve:
1900 p4_write_pipe(['shelve', '-i'], submitTemplate)
1901 else:
1902 p4_write_pipe(['submit', '-i'], submitTemplate)
1903 # The rename/copy happened by applying a patch that created a
1904 # new file. This leaves it writable, which confuses p4.
1905 for f in pureRenameCopy:
1906 p4_sync(f, "-f")
1908 if self.preserveUser:
1909 if p4User:
1910 # Get last changelist number. Cannot easily get it from
1911 # the submit command output as the output is
1912 # unmarshalled.
1913 changelist = self.lastP4Changelist()
1914 self.modifyChangelistUser(changelist, p4User)
1916 submitted = True
1918 finally:
1919 # skip this patch
1920 if not submitted or self.shelve:
1921 if self.shelve:
1922 print ("Reverting shelved files.")
1923 else:
1924 print ("Submission cancelled, undoing p4 changes.")
1925 for f in editedFiles | filesToDelete:
1926 p4_revert(f)
1927 for f in filesToAdd:
1928 p4_revert(f)
1929 os.remove(f)
1931 os.remove(fileName)
1932 return submitted
1934 # Export git tags as p4 labels. Create a p4 label and then tag
1935 # with that.
1936 def exportGitTags(self, gitTags):
1937 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1938 if len(validLabelRegexp) == 0:
1939 validLabelRegexp = defaultLabelRegexp
1940 m = re.compile(validLabelRegexp)
1942 for name in gitTags:
1944 if not m.match(name):
1945 if verbose:
1946 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1947 continue
1949 # Get the p4 commit this corresponds to
1950 logMessage = extractLogMessageFromGitCommit(name)
1951 values = extractSettingsGitLog(logMessage)
1953 if not values.has_key('change'):
1954 # a tag pointing to something not sent to p4; ignore
1955 if verbose:
1956 print "git tag %s does not give a p4 commit" % name
1957 continue
1958 else:
1959 changelist = values['change']
1961 # Get the tag details.
1962 inHeader = True
1963 isAnnotated = False
1964 body = []
1965 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1966 l = l.strip()
1967 if inHeader:
1968 if re.match(r'tag\s+', l):
1969 isAnnotated = True
1970 elif re.match(r'\s*$', l):
1971 inHeader = False
1972 continue
1973 else:
1974 body.append(l)
1976 if not isAnnotated:
1977 body = ["lightweight tag imported by git p4\n"]
1979 # Create the label - use the same view as the client spec we are using
1980 clientSpec = getClientSpec()
1982 labelTemplate = "Label: %s\n" % name
1983 labelTemplate += "Description:\n"
1984 for b in body:
1985 labelTemplate += "\t" + b + "\n"
1986 labelTemplate += "View:\n"
1987 for depot_side in clientSpec.mappings:
1988 labelTemplate += "\t%s\n" % depot_side
1990 if self.dry_run:
1991 print "Would create p4 label %s for tag" % name
1992 elif self.prepare_p4_only:
1993 print "Not creating p4 label %s for tag due to option" \
1994 " --prepare-p4-only" % name
1995 else:
1996 p4_write_pipe(["label", "-i"], labelTemplate)
1998 # Use the label
1999 p4_system(["tag", "-l", name] +
2000 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
2002 if verbose:
2003 print "created p4 label for tag %s" % name
2005 def run(self, args):
2006 if len(args) == 0:
2007 self.master = currentGitBranch()
2008 elif len(args) == 1:
2009 self.master = args[0]
2010 if not branchExists(self.master):
2011 die("Branch %s does not exist" % self.master)
2012 else:
2013 return False
2015 if self.master:
2016 allowSubmit = gitConfig("git-p4.allowSubmit")
2017 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2018 die("%s is not in git-p4.allowSubmit" % self.master)
2020 [upstream, settings] = findUpstreamBranchPoint()
2021 self.depotPath = settings['depot-paths'][0]
2022 if len(self.origin) == 0:
2023 self.origin = upstream
2025 if self.update_shelve:
2026 self.shelve = True
2028 if self.preserveUser:
2029 if not self.canChangeChangelists():
2030 die("Cannot preserve user names without p4 super-user or admin permissions")
2032 # if not set from the command line, try the config file
2033 if self.conflict_behavior is None:
2034 val = gitConfig("git-p4.conflict")
2035 if val:
2036 if val not in self.conflict_behavior_choices:
2037 die("Invalid value '%s' for config git-p4.conflict" % val)
2038 else:
2039 val = "ask"
2040 self.conflict_behavior = val
2042 if self.verbose:
2043 print "Origin branch is " + self.origin
2045 if len(self.depotPath) == 0:
2046 print "Internal error: cannot locate perforce depot path from existing branches"
2047 sys.exit(128)
2049 self.useClientSpec = False
2050 if gitConfigBool("git-p4.useclientspec"):
2051 self.useClientSpec = True
2052 if self.useClientSpec:
2053 self.clientSpecDirs = getClientSpec()
2055 # Check for the existence of P4 branches
2056 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2058 if self.useClientSpec and not branchesDetected:
2059 # all files are relative to the client spec
2060 self.clientPath = getClientRoot()
2061 else:
2062 self.clientPath = p4Where(self.depotPath)
2064 if self.clientPath == "":
2065 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2067 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2068 self.oldWorkingDirectory = os.getcwd()
2070 # ensure the clientPath exists
2071 new_client_dir = False
2072 if not os.path.exists(self.clientPath):
2073 new_client_dir = True
2074 os.makedirs(self.clientPath)
2076 chdir(self.clientPath, is_client_path=True)
2077 if self.dry_run:
2078 print "Would synchronize p4 checkout in %s" % self.clientPath
2079 else:
2080 print "Synchronizing p4 checkout..."
2081 if new_client_dir:
2082 # old one was destroyed, and maybe nobody told p4
2083 p4_sync("...", "-f")
2084 else:
2085 p4_sync("...")
2086 self.check()
2088 commits = []
2089 if self.master:
2090 commitish = self.master
2091 else:
2092 commitish = 'HEAD'
2094 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2095 commits.append(line.strip())
2096 commits.reverse()
2098 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2099 self.checkAuthorship = False
2100 else:
2101 self.checkAuthorship = True
2103 if self.preserveUser:
2104 self.checkValidP4Users(commits)
2107 # Build up a set of options to be passed to diff when
2108 # submitting each commit to p4.
2110 if self.detectRenames:
2111 # command-line -M arg
2112 self.diffOpts = "-M"
2113 else:
2114 # If not explicitly set check the config variable
2115 detectRenames = gitConfig("git-p4.detectRenames")
2117 if detectRenames.lower() == "false" or detectRenames == "":
2118 self.diffOpts = ""
2119 elif detectRenames.lower() == "true":
2120 self.diffOpts = "-M"
2121 else:
2122 self.diffOpts = "-M%s" % detectRenames
2124 # no command-line arg for -C or --find-copies-harder, just
2125 # config variables
2126 detectCopies = gitConfig("git-p4.detectCopies")
2127 if detectCopies.lower() == "false" or detectCopies == "":
2128 pass
2129 elif detectCopies.lower() == "true":
2130 self.diffOpts += " -C"
2131 else:
2132 self.diffOpts += " -C%s" % detectCopies
2134 if gitConfigBool("git-p4.detectCopiesHarder"):
2135 self.diffOpts += " --find-copies-harder"
2138 # Apply the commits, one at a time. On failure, ask if should
2139 # continue to try the rest of the patches, or quit.
2141 if self.dry_run:
2142 print "Would apply"
2143 applied = []
2144 last = len(commits) - 1
2145 for i, commit in enumerate(commits):
2146 if self.dry_run:
2147 print " ", read_pipe(["git", "show", "-s",
2148 "--format=format:%h %s", commit])
2149 ok = True
2150 else:
2151 ok = self.applyCommit(commit)
2152 if ok:
2153 applied.append(commit)
2154 else:
2155 if self.prepare_p4_only and i < last:
2156 print "Processing only the first commit due to option" \
2157 " --prepare-p4-only"
2158 break
2159 if i < last:
2160 quit = False
2161 while True:
2162 # prompt for what to do, or use the option/variable
2163 if self.conflict_behavior == "ask":
2164 print "What do you want to do?"
2165 response = raw_input("[s]kip this commit but apply"
2166 " the rest, or [q]uit? ")
2167 if not response:
2168 continue
2169 elif self.conflict_behavior == "skip":
2170 response = "s"
2171 elif self.conflict_behavior == "quit":
2172 response = "q"
2173 else:
2174 die("Unknown conflict_behavior '%s'" %
2175 self.conflict_behavior)
2177 if response[0] == "s":
2178 print "Skipping this commit, but applying the rest"
2179 break
2180 if response[0] == "q":
2181 print "Quitting"
2182 quit = True
2183 break
2184 if quit:
2185 break
2187 chdir(self.oldWorkingDirectory)
2188 shelved_applied = "shelved" if self.shelve else "applied"
2189 if self.dry_run:
2190 pass
2191 elif self.prepare_p4_only:
2192 pass
2193 elif len(commits) == len(applied):
2194 print ("All commits {0}!".format(shelved_applied))
2196 sync = P4Sync()
2197 if self.branch:
2198 sync.branch = self.branch
2199 sync.run([])
2201 rebase = P4Rebase()
2202 rebase.rebase()
2204 else:
2205 if len(applied) == 0:
2206 print ("No commits {0}.".format(shelved_applied))
2207 else:
2208 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2209 for c in commits:
2210 if c in applied:
2211 star = "*"
2212 else:
2213 star = " "
2214 print star, read_pipe(["git", "show", "-s",
2215 "--format=format:%h %s", c])
2216 print "You will have to do 'git p4 sync' and rebase."
2218 if gitConfigBool("git-p4.exportLabels"):
2219 self.exportLabels = True
2221 if self.exportLabels:
2222 p4Labels = getP4Labels(self.depotPath)
2223 gitTags = getGitTags()
2225 missingGitTags = gitTags - p4Labels
2226 self.exportGitTags(missingGitTags)
2228 # exit with error unless everything applied perfectly
2229 if len(commits) != len(applied):
2230 sys.exit(1)
2232 return True
2234 class View(object):
2235 """Represent a p4 view ("p4 help views"), and map files in a
2236 repo according to the view."""
2238 def __init__(self, client_name):
2239 self.mappings = []
2240 self.client_prefix = "//%s/" % client_name
2241 # cache results of "p4 where" to lookup client file locations
2242 self.client_spec_path_cache = {}
2244 def append(self, view_line):
2245 """Parse a view line, splitting it into depot and client
2246 sides. Append to self.mappings, preserving order. This
2247 is only needed for tag creation."""
2249 # Split the view line into exactly two words. P4 enforces
2250 # structure on these lines that simplifies this quite a bit.
2252 # Either or both words may be double-quoted.
2253 # Single quotes do not matter.
2254 # Double-quote marks cannot occur inside the words.
2255 # A + or - prefix is also inside the quotes.
2256 # There are no quotes unless they contain a space.
2257 # The line is already white-space stripped.
2258 # The two words are separated by a single space.
2260 if view_line[0] == '"':
2261 # First word is double quoted. Find its end.
2262 close_quote_index = view_line.find('"', 1)
2263 if close_quote_index <= 0:
2264 die("No first-word closing quote found: %s" % view_line)
2265 depot_side = view_line[1:close_quote_index]
2266 # skip closing quote and space
2267 rhs_index = close_quote_index + 1 + 1
2268 else:
2269 space_index = view_line.find(" ")
2270 if space_index <= 0:
2271 die("No word-splitting space found: %s" % view_line)
2272 depot_side = view_line[0:space_index]
2273 rhs_index = space_index + 1
2275 # prefix + means overlay on previous mapping
2276 if depot_side.startswith("+"):
2277 depot_side = depot_side[1:]
2279 # prefix - means exclude this path, leave out of mappings
2280 exclude = False
2281 if depot_side.startswith("-"):
2282 exclude = True
2283 depot_side = depot_side[1:]
2285 if not exclude:
2286 self.mappings.append(depot_side)
2288 def convert_client_path(self, clientFile):
2289 # chop off //client/ part to make it relative
2290 if not clientFile.startswith(self.client_prefix):
2291 die("No prefix '%s' on clientFile '%s'" %
2292 (self.client_prefix, clientFile))
2293 return clientFile[len(self.client_prefix):]
2295 def update_client_spec_path_cache(self, files):
2296 """ Caching file paths by "p4 where" batch query """
2298 # List depot file paths exclude that already cached
2299 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2301 if len(fileArgs) == 0:
2302 return # All files in cache
2304 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2305 for res in where_result:
2306 if "code" in res and res["code"] == "error":
2307 # assume error is "... file(s) not in client view"
2308 continue
2309 if "clientFile" not in res:
2310 die("No clientFile in 'p4 where' output")
2311 if "unmap" in res:
2312 # it will list all of them, but only one not unmap-ped
2313 continue
2314 if gitConfigBool("core.ignorecase"):
2315 res['depotFile'] = res['depotFile'].lower()
2316 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2318 # not found files or unmap files set to ""
2319 for depotFile in fileArgs:
2320 if gitConfigBool("core.ignorecase"):
2321 depotFile = depotFile.lower()
2322 if depotFile not in self.client_spec_path_cache:
2323 self.client_spec_path_cache[depotFile] = ""
2325 def map_in_client(self, depot_path):
2326 """Return the relative location in the client where this
2327 depot file should live. Returns "" if the file should
2328 not be mapped in the client."""
2330 if gitConfigBool("core.ignorecase"):
2331 depot_path = depot_path.lower()
2333 if depot_path in self.client_spec_path_cache:
2334 return self.client_spec_path_cache[depot_path]
2336 die( "Error: %s is not found in client spec path" % depot_path )
2337 return ""
2339 class P4Sync(Command, P4UserMap):
2340 delete_actions = ( "delete", "move/delete", "purge" )
2342 def __init__(self):
2343 Command.__init__(self)
2344 P4UserMap.__init__(self)
2345 self.options = [
2346 optparse.make_option("--branch", dest="branch"),
2347 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2348 optparse.make_option("--changesfile", dest="changesFile"),
2349 optparse.make_option("--silent", dest="silent", action="store_true"),
2350 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2351 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2352 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2353 help="Import into refs/heads/ , not refs/remotes"),
2354 optparse.make_option("--max-changes", dest="maxChanges",
2355 help="Maximum number of changes to import"),
2356 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2357 help="Internal block size to use when iteratively calling p4 changes"),
2358 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2359 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2360 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2361 help="Only sync files that are included in the Perforce Client Spec"),
2362 optparse.make_option("-/", dest="cloneExclude",
2363 action="append", type="string",
2364 help="exclude depot path"),
2366 self.description = """Imports from Perforce into a git repository.\n
2367 example:
2368 //depot/my/project/ -- to import the current head
2369 //depot/my/project/@all -- to import everything
2370 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2372 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2374 self.usage += " //depot/path[@revRange]"
2375 self.silent = False
2376 self.createdBranches = set()
2377 self.committedChanges = set()
2378 self.branch = ""
2379 self.detectBranches = False
2380 self.detectLabels = False
2381 self.importLabels = False
2382 self.changesFile = ""
2383 self.syncWithOrigin = True
2384 self.importIntoRemotes = True
2385 self.maxChanges = ""
2386 self.changes_block_size = None
2387 self.keepRepoPath = False
2388 self.depotPaths = None
2389 self.p4BranchesInGit = []
2390 self.cloneExclude = []
2391 self.useClientSpec = False
2392 self.useClientSpec_from_options = False
2393 self.clientSpecDirs = None
2394 self.tempBranches = []
2395 self.tempBranchLocation = "refs/git-p4-tmp"
2396 self.largeFileSystem = None
2398 if gitConfig('git-p4.largeFileSystem'):
2399 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2400 self.largeFileSystem = largeFileSystemConstructor(
2401 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2404 if gitConfig("git-p4.syncFromOrigin") == "false":
2405 self.syncWithOrigin = False
2407 # This is required for the "append" cloneExclude action
2408 def ensure_value(self, attr, value):
2409 if not hasattr(self, attr) or getattr(self, attr) is None:
2410 setattr(self, attr, value)
2411 return getattr(self, attr)
2413 # Force a checkpoint in fast-import and wait for it to finish
2414 def checkpoint(self):
2415 self.gitStream.write("checkpoint\n\n")
2416 self.gitStream.write("progress checkpoint\n\n")
2417 out = self.gitOutput.readline()
2418 if self.verbose:
2419 print "checkpoint finished: " + out
2421 def extractFilesFromCommit(self, commit):
2422 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2423 for path in self.cloneExclude]
2424 files = []
2425 fnum = 0
2426 while commit.has_key("depotFile%s" % fnum):
2427 path = commit["depotFile%s" % fnum]
2429 if [p for p in self.cloneExclude
2430 if p4PathStartsWith(path, p)]:
2431 found = False
2432 else:
2433 found = [p for p in self.depotPaths
2434 if p4PathStartsWith(path, p)]
2435 if not found:
2436 fnum = fnum + 1
2437 continue
2439 file = {}
2440 file["path"] = path
2441 file["rev"] = commit["rev%s" % fnum]
2442 file["action"] = commit["action%s" % fnum]
2443 file["type"] = commit["type%s" % fnum]
2444 files.append(file)
2445 fnum = fnum + 1
2446 return files
2448 def extractJobsFromCommit(self, commit):
2449 jobs = []
2450 jnum = 0
2451 while commit.has_key("job%s" % jnum):
2452 job = commit["job%s" % jnum]
2453 jobs.append(job)
2454 jnum = jnum + 1
2455 return jobs
2457 def stripRepoPath(self, path, prefixes):
2458 """When streaming files, this is called to map a p4 depot path
2459 to where it should go in git. The prefixes are either
2460 self.depotPaths, or self.branchPrefixes in the case of
2461 branch detection."""
2463 if self.useClientSpec:
2464 # branch detection moves files up a level (the branch name)
2465 # from what client spec interpretation gives
2466 path = self.clientSpecDirs.map_in_client(path)
2467 if self.detectBranches:
2468 for b in self.knownBranches:
2469 if path.startswith(b + "/"):
2470 path = path[len(b)+1:]
2472 elif self.keepRepoPath:
2473 # Preserve everything in relative path name except leading
2474 # //depot/; just look at first prefix as they all should
2475 # be in the same depot.
2476 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2477 if p4PathStartsWith(path, depot):
2478 path = path[len(depot):]
2480 else:
2481 for p in prefixes:
2482 if p4PathStartsWith(path, p):
2483 path = path[len(p):]
2484 break
2486 path = wildcard_decode(path)
2487 return path
2489 def splitFilesIntoBranches(self, commit):
2490 """Look at each depotFile in the commit to figure out to what
2491 branch it belongs."""
2493 if self.clientSpecDirs:
2494 files = self.extractFilesFromCommit(commit)
2495 self.clientSpecDirs.update_client_spec_path_cache(files)
2497 branches = {}
2498 fnum = 0
2499 while commit.has_key("depotFile%s" % fnum):
2500 path = commit["depotFile%s" % fnum]
2501 found = [p for p in self.depotPaths
2502 if p4PathStartsWith(path, p)]
2503 if not found:
2504 fnum = fnum + 1
2505 continue
2507 file = {}
2508 file["path"] = path
2509 file["rev"] = commit["rev%s" % fnum]
2510 file["action"] = commit["action%s" % fnum]
2511 file["type"] = commit["type%s" % fnum]
2512 fnum = fnum + 1
2514 # start with the full relative path where this file would
2515 # go in a p4 client
2516 if self.useClientSpec:
2517 relPath = self.clientSpecDirs.map_in_client(path)
2518 else:
2519 relPath = self.stripRepoPath(path, self.depotPaths)
2521 for branch in self.knownBranches.keys():
2522 # add a trailing slash so that a commit into qt/4.2foo
2523 # doesn't end up in qt/4.2, e.g.
2524 if relPath.startswith(branch + "/"):
2525 if branch not in branches:
2526 branches[branch] = []
2527 branches[branch].append(file)
2528 break
2530 return branches
2532 def writeToGitStream(self, gitMode, relPath, contents):
2533 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2534 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2535 for d in contents:
2536 self.gitStream.write(d)
2537 self.gitStream.write('\n')
2539 def encodeWithUTF8(self, path):
2540 try:
2541 path.decode('ascii')
2542 except:
2543 encoding = 'utf8'
2544 if gitConfig('git-p4.pathEncoding'):
2545 encoding = gitConfig('git-p4.pathEncoding')
2546 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2547 if self.verbose:
2548 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2549 return path
2551 # output one file from the P4 stream
2552 # - helper for streamP4Files
2554 def streamOneP4File(self, file, contents):
2555 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2556 relPath = self.encodeWithUTF8(relPath)
2557 if verbose:
2558 size = int(self.stream_file['fileSize'])
2559 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2560 sys.stdout.flush()
2562 (type_base, type_mods) = split_p4_type(file["type"])
2564 git_mode = "100644"
2565 if "x" in type_mods:
2566 git_mode = "100755"
2567 if type_base == "symlink":
2568 git_mode = "120000"
2569 # p4 print on a symlink sometimes contains "target\n";
2570 # if it does, remove the newline
2571 data = ''.join(contents)
2572 if not data:
2573 # Some version of p4 allowed creating a symlink that pointed
2574 # to nothing. This causes p4 errors when checking out such
2575 # a change, and errors here too. Work around it by ignoring
2576 # the bad symlink; hopefully a future change fixes it.
2577 print "\nIgnoring empty symlink in %s" % file['depotFile']
2578 return
2579 elif data[-1] == '\n':
2580 contents = [data[:-1]]
2581 else:
2582 contents = [data]
2584 if type_base == "utf16":
2585 # p4 delivers different text in the python output to -G
2586 # than it does when using "print -o", or normal p4 client
2587 # operations. utf16 is converted to ascii or utf8, perhaps.
2588 # But ascii text saved as -t utf16 is completely mangled.
2589 # Invoke print -o to get the real contents.
2591 # On windows, the newlines will always be mangled by print, so put
2592 # them back too. This is not needed to the cygwin windows version,
2593 # just the native "NT" type.
2595 try:
2596 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2597 except Exception as e:
2598 if 'Translation of file content failed' in str(e):
2599 type_base = 'binary'
2600 else:
2601 raise e
2602 else:
2603 if p4_version_string().find('/NT') >= 0:
2604 text = text.replace('\r\n', '\n')
2605 contents = [ text ]
2607 if type_base == "apple":
2608 # Apple filetype files will be streamed as a concatenation of
2609 # its appledouble header and the contents. This is useless
2610 # on both macs and non-macs. If using "print -q -o xx", it
2611 # will create "xx" with the data, and "%xx" with the header.
2612 # This is also not very useful.
2614 # Ideally, someday, this script can learn how to generate
2615 # appledouble files directly and import those to git, but
2616 # non-mac machines can never find a use for apple filetype.
2617 print "\nIgnoring apple filetype file %s" % file['depotFile']
2618 return
2620 # Note that we do not try to de-mangle keywords on utf16 files,
2621 # even though in theory somebody may want that.
2622 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2623 if pattern:
2624 regexp = re.compile(pattern, re.VERBOSE)
2625 text = ''.join(contents)
2626 text = regexp.sub(r'$\1$', text)
2627 contents = [ text ]
2629 if self.largeFileSystem:
2630 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2632 self.writeToGitStream(git_mode, relPath, contents)
2634 def streamOneP4Deletion(self, file):
2635 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2636 relPath = self.encodeWithUTF8(relPath)
2637 if verbose:
2638 sys.stdout.write("delete %s\n" % relPath)
2639 sys.stdout.flush()
2640 self.gitStream.write("D %s\n" % relPath)
2642 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2643 self.largeFileSystem.removeLargeFile(relPath)
2645 # handle another chunk of streaming data
2646 def streamP4FilesCb(self, marshalled):
2648 # catch p4 errors and complain
2649 err = None
2650 if "code" in marshalled:
2651 if marshalled["code"] == "error":
2652 if "data" in marshalled:
2653 err = marshalled["data"].rstrip()
2655 if not err and 'fileSize' in self.stream_file:
2656 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2657 if required_bytes > 0:
2658 err = 'Not enough space left on %s! Free at least %i MB.' % (
2659 os.getcwd(), required_bytes/1024/1024
2662 if err:
2663 f = None
2664 if self.stream_have_file_info:
2665 if "depotFile" in self.stream_file:
2666 f = self.stream_file["depotFile"]
2667 # force a failure in fast-import, else an empty
2668 # commit will be made
2669 self.gitStream.write("\n")
2670 self.gitStream.write("die-now\n")
2671 self.gitStream.close()
2672 # ignore errors, but make sure it exits first
2673 self.importProcess.wait()
2674 if f:
2675 die("Error from p4 print for %s: %s" % (f, err))
2676 else:
2677 die("Error from p4 print: %s" % err)
2679 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2680 # start of a new file - output the old one first
2681 self.streamOneP4File(self.stream_file, self.stream_contents)
2682 self.stream_file = {}
2683 self.stream_contents = []
2684 self.stream_have_file_info = False
2686 # pick up the new file information... for the
2687 # 'data' field we need to append to our array
2688 for k in marshalled.keys():
2689 if k == 'data':
2690 if 'streamContentSize' not in self.stream_file:
2691 self.stream_file['streamContentSize'] = 0
2692 self.stream_file['streamContentSize'] += len(marshalled['data'])
2693 self.stream_contents.append(marshalled['data'])
2694 else:
2695 self.stream_file[k] = marshalled[k]
2697 if (verbose and
2698 'streamContentSize' in self.stream_file and
2699 'fileSize' in self.stream_file and
2700 'depotFile' in self.stream_file):
2701 size = int(self.stream_file["fileSize"])
2702 if size > 0:
2703 progress = 100*self.stream_file['streamContentSize']/size
2704 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2705 sys.stdout.flush()
2707 self.stream_have_file_info = True
2709 # Stream directly from "p4 files" into "git fast-import"
2710 def streamP4Files(self, files):
2711 filesForCommit = []
2712 filesToRead = []
2713 filesToDelete = []
2715 for f in files:
2716 filesForCommit.append(f)
2717 if f['action'] in self.delete_actions:
2718 filesToDelete.append(f)
2719 else:
2720 filesToRead.append(f)
2722 # deleted files...
2723 for f in filesToDelete:
2724 self.streamOneP4Deletion(f)
2726 if len(filesToRead) > 0:
2727 self.stream_file = {}
2728 self.stream_contents = []
2729 self.stream_have_file_info = False
2731 # curry self argument
2732 def streamP4FilesCbSelf(entry):
2733 self.streamP4FilesCb(entry)
2735 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2737 p4CmdList(["-x", "-", "print"],
2738 stdin=fileArgs,
2739 cb=streamP4FilesCbSelf)
2741 # do the last chunk
2742 if self.stream_file.has_key('depotFile'):
2743 self.streamOneP4File(self.stream_file, self.stream_contents)
2745 def make_email(self, userid):
2746 if userid in self.users:
2747 return self.users[userid]
2748 else:
2749 return "%s <a@b>" % userid
2751 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2752 """ Stream a p4 tag.
2753 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2756 if verbose:
2757 print "writing tag %s for commit %s" % (labelName, commit)
2758 gitStream.write("tag %s\n" % labelName)
2759 gitStream.write("from %s\n" % commit)
2761 if labelDetails.has_key('Owner'):
2762 owner = labelDetails["Owner"]
2763 else:
2764 owner = None
2766 # Try to use the owner of the p4 label, or failing that,
2767 # the current p4 user id.
2768 if owner:
2769 email = self.make_email(owner)
2770 else:
2771 email = self.make_email(self.p4UserId())
2772 tagger = "%s %s %s" % (email, epoch, self.tz)
2774 gitStream.write("tagger %s\n" % tagger)
2776 print "labelDetails=",labelDetails
2777 if labelDetails.has_key('Description'):
2778 description = labelDetails['Description']
2779 else:
2780 description = 'Label from git p4'
2782 gitStream.write("data %d\n" % len(description))
2783 gitStream.write(description)
2784 gitStream.write("\n")
2786 def inClientSpec(self, path):
2787 if not self.clientSpecDirs:
2788 return True
2789 inClientSpec = self.clientSpecDirs.map_in_client(path)
2790 if not inClientSpec and self.verbose:
2791 print('Ignoring file outside of client spec: {0}'.format(path))
2792 return inClientSpec
2794 def hasBranchPrefix(self, path):
2795 if not self.branchPrefixes:
2796 return True
2797 hasPrefix = [p for p in self.branchPrefixes
2798 if p4PathStartsWith(path, p)]
2799 if not hasPrefix and self.verbose:
2800 print('Ignoring file outside of prefix: {0}'.format(path))
2801 return hasPrefix
2803 def commit(self, details, files, branch, parent = ""):
2804 epoch = details["time"]
2805 author = details["user"]
2806 jobs = self.extractJobsFromCommit(details)
2808 if self.verbose:
2809 print('commit into {0}'.format(branch))
2811 if self.clientSpecDirs:
2812 self.clientSpecDirs.update_client_spec_path_cache(files)
2814 files = [f for f in files
2815 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2817 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2818 print('Ignoring revision {0} as it would produce an empty commit.'
2819 .format(details['change']))
2820 return
2822 self.gitStream.write("commit %s\n" % branch)
2823 self.gitStream.write("mark :%s\n" % details["change"])
2824 self.committedChanges.add(int(details["change"]))
2825 committer = ""
2826 if author not in self.users:
2827 self.getUserMapFromPerforceServer()
2828 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2830 self.gitStream.write("committer %s\n" % committer)
2832 self.gitStream.write("data <<EOT\n")
2833 self.gitStream.write(details["desc"])
2834 if len(jobs) > 0:
2835 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2836 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2837 (','.join(self.branchPrefixes), details["change"]))
2838 if len(details['options']) > 0:
2839 self.gitStream.write(": options = %s" % details['options'])
2840 self.gitStream.write("]\nEOT\n\n")
2842 if len(parent) > 0:
2843 if self.verbose:
2844 print "parent %s" % parent
2845 self.gitStream.write("from %s\n" % parent)
2847 self.streamP4Files(files)
2848 self.gitStream.write("\n")
2850 change = int(details["change"])
2852 if self.labels.has_key(change):
2853 label = self.labels[change]
2854 labelDetails = label[0]
2855 labelRevisions = label[1]
2856 if self.verbose:
2857 print "Change %s is labelled %s" % (change, labelDetails)
2859 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2860 for p in self.branchPrefixes])
2862 if len(files) == len(labelRevisions):
2864 cleanedFiles = {}
2865 for info in files:
2866 if info["action"] in self.delete_actions:
2867 continue
2868 cleanedFiles[info["depotFile"]] = info["rev"]
2870 if cleanedFiles == labelRevisions:
2871 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2873 else:
2874 if not self.silent:
2875 print ("Tag %s does not match with change %s: files do not match."
2876 % (labelDetails["label"], change))
2878 else:
2879 if not self.silent:
2880 print ("Tag %s does not match with change %s: file count is different."
2881 % (labelDetails["label"], change))
2883 # Build a dictionary of changelists and labels, for "detect-labels" option.
2884 def getLabels(self):
2885 self.labels = {}
2887 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2888 if len(l) > 0 and not self.silent:
2889 print "Finding files belonging to labels in %s" % `self.depotPaths`
2891 for output in l:
2892 label = output["label"]
2893 revisions = {}
2894 newestChange = 0
2895 if self.verbose:
2896 print "Querying files for label %s" % label
2897 for file in p4CmdList(["files"] +
2898 ["%s...@%s" % (p, label)
2899 for p in self.depotPaths]):
2900 revisions[file["depotFile"]] = file["rev"]
2901 change = int(file["change"])
2902 if change > newestChange:
2903 newestChange = change
2905 self.labels[newestChange] = [output, revisions]
2907 if self.verbose:
2908 print "Label changes: %s" % self.labels.keys()
2910 # Import p4 labels as git tags. A direct mapping does not
2911 # exist, so assume that if all the files are at the same revision
2912 # then we can use that, or it's something more complicated we should
2913 # just ignore.
2914 def importP4Labels(self, stream, p4Labels):
2915 if verbose:
2916 print "import p4 labels: " + ' '.join(p4Labels)
2918 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2919 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2920 if len(validLabelRegexp) == 0:
2921 validLabelRegexp = defaultLabelRegexp
2922 m = re.compile(validLabelRegexp)
2924 for name in p4Labels:
2925 commitFound = False
2927 if not m.match(name):
2928 if verbose:
2929 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2930 continue
2932 if name in ignoredP4Labels:
2933 continue
2935 labelDetails = p4CmdList(['label', "-o", name])[0]
2937 # get the most recent changelist for each file in this label
2938 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2939 for p in self.depotPaths])
2941 if change.has_key('change'):
2942 # find the corresponding git commit; take the oldest commit
2943 changelist = int(change['change'])
2944 if changelist in self.committedChanges:
2945 gitCommit = ":%d" % changelist # use a fast-import mark
2946 commitFound = True
2947 else:
2948 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2949 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2950 if len(gitCommit) == 0:
2951 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2952 else:
2953 commitFound = True
2954 gitCommit = gitCommit.strip()
2956 if commitFound:
2957 # Convert from p4 time format
2958 try:
2959 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2960 except ValueError:
2961 print "Could not convert label time %s" % labelDetails['Update']
2962 tmwhen = 1
2964 when = int(time.mktime(tmwhen))
2965 self.streamTag(stream, name, labelDetails, gitCommit, when)
2966 if verbose:
2967 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2968 else:
2969 if verbose:
2970 print "Label %s has no changelists - possibly deleted?" % name
2972 if not commitFound:
2973 # We can't import this label; don't try again as it will get very
2974 # expensive repeatedly fetching all the files for labels that will
2975 # never be imported. If the label is moved in the future, the
2976 # ignore will need to be removed manually.
2977 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2979 def guessProjectName(self):
2980 for p in self.depotPaths:
2981 if p.endswith("/"):
2982 p = p[:-1]
2983 p = p[p.strip().rfind("/") + 1:]
2984 if not p.endswith("/"):
2985 p += "/"
2986 return p
2988 def getBranchMapping(self):
2989 lostAndFoundBranches = set()
2991 user = gitConfig("git-p4.branchUser")
2992 if len(user) > 0:
2993 command = "branches -u %s" % user
2994 else:
2995 command = "branches"
2997 for info in p4CmdList(command):
2998 details = p4Cmd(["branch", "-o", info["branch"]])
2999 viewIdx = 0
3000 while details.has_key("View%s" % viewIdx):
3001 paths = details["View%s" % viewIdx].split(" ")
3002 viewIdx = viewIdx + 1
3003 # require standard //depot/foo/... //depot/bar/... mapping
3004 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3005 continue
3006 source = paths[0]
3007 destination = paths[1]
3008 ## HACK
3009 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
3010 source = source[len(self.depotPaths[0]):-4]
3011 destination = destination[len(self.depotPaths[0]):-4]
3013 if destination in self.knownBranches:
3014 if not self.silent:
3015 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3016 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3017 continue
3019 self.knownBranches[destination] = source
3021 lostAndFoundBranches.discard(destination)
3023 if source not in self.knownBranches:
3024 lostAndFoundBranches.add(source)
3026 # Perforce does not strictly require branches to be defined, so we also
3027 # check git config for a branch list.
3029 # Example of branch definition in git config file:
3030 # [git-p4]
3031 # branchList=main:branchA
3032 # branchList=main:branchB
3033 # branchList=branchA:branchC
3034 configBranches = gitConfigList("git-p4.branchList")
3035 for branch in configBranches:
3036 if branch:
3037 (source, destination) = branch.split(":")
3038 self.knownBranches[destination] = source
3040 lostAndFoundBranches.discard(destination)
3042 if source not in self.knownBranches:
3043 lostAndFoundBranches.add(source)
3046 for branch in lostAndFoundBranches:
3047 self.knownBranches[branch] = branch
3049 def getBranchMappingFromGitBranches(self):
3050 branches = p4BranchesInGit(self.importIntoRemotes)
3051 for branch in branches.keys():
3052 if branch == "master":
3053 branch = "main"
3054 else:
3055 branch = branch[len(self.projectName):]
3056 self.knownBranches[branch] = branch
3058 def updateOptionDict(self, d):
3059 option_keys = {}
3060 if self.keepRepoPath:
3061 option_keys['keepRepoPath'] = 1
3063 d["options"] = ' '.join(sorted(option_keys.keys()))
3065 def readOptions(self, d):
3066 self.keepRepoPath = (d.has_key('options')
3067 and ('keepRepoPath' in d['options']))
3069 def gitRefForBranch(self, branch):
3070 if branch == "main":
3071 return self.refPrefix + "master"
3073 if len(branch) <= 0:
3074 return branch
3076 return self.refPrefix + self.projectName + branch
3078 def gitCommitByP4Change(self, ref, change):
3079 if self.verbose:
3080 print "looking in ref " + ref + " for change %s using bisect..." % change
3082 earliestCommit = ""
3083 latestCommit = parseRevision(ref)
3085 while True:
3086 if self.verbose:
3087 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3088 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3089 if len(next) == 0:
3090 if self.verbose:
3091 print "argh"
3092 return ""
3093 log = extractLogMessageFromGitCommit(next)
3094 settings = extractSettingsGitLog(log)
3095 currentChange = int(settings['change'])
3096 if self.verbose:
3097 print "current change %s" % currentChange
3099 if currentChange == change:
3100 if self.verbose:
3101 print "found %s" % next
3102 return next
3104 if currentChange < change:
3105 earliestCommit = "^%s" % next
3106 else:
3107 latestCommit = "%s" % next
3109 return ""
3111 def importNewBranch(self, branch, maxChange):
3112 # make fast-import flush all changes to disk and update the refs using the checkpoint
3113 # command so that we can try to find the branch parent in the git history
3114 self.gitStream.write("checkpoint\n\n");
3115 self.gitStream.flush();
3116 branchPrefix = self.depotPaths[0] + branch + "/"
3117 range = "@1,%s" % maxChange
3118 #print "prefix" + branchPrefix
3119 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3120 if len(changes) <= 0:
3121 return False
3122 firstChange = changes[0]
3123 #print "first change in branch: %s" % firstChange
3124 sourceBranch = self.knownBranches[branch]
3125 sourceDepotPath = self.depotPaths[0] + sourceBranch
3126 sourceRef = self.gitRefForBranch(sourceBranch)
3127 #print "source " + sourceBranch
3129 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3130 #print "branch parent: %s" % branchParentChange
3131 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3132 if len(gitParent) > 0:
3133 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3134 #print "parent git commit: %s" % gitParent
3136 self.importChanges(changes)
3137 return True
3139 def searchParent(self, parent, branch, target):
3140 parentFound = False
3141 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3142 "--no-merges", parent]):
3143 blob = blob.strip()
3144 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3145 parentFound = True
3146 if self.verbose:
3147 print "Found parent of %s in commit %s" % (branch, blob)
3148 break
3149 if parentFound:
3150 return blob
3151 else:
3152 return None
3154 def importChanges(self, changes):
3155 cnt = 1
3156 for change in changes:
3157 description = p4_describe(change)
3158 self.updateOptionDict(description)
3160 if not self.silent:
3161 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3162 sys.stdout.flush()
3163 cnt = cnt + 1
3165 try:
3166 if self.detectBranches:
3167 branches = self.splitFilesIntoBranches(description)
3168 for branch in branches.keys():
3169 ## HACK --hwn
3170 branchPrefix = self.depotPaths[0] + branch + "/"
3171 self.branchPrefixes = [ branchPrefix ]
3173 parent = ""
3175 filesForCommit = branches[branch]
3177 if self.verbose:
3178 print "branch is %s" % branch
3180 self.updatedBranches.add(branch)
3182 if branch not in self.createdBranches:
3183 self.createdBranches.add(branch)
3184 parent = self.knownBranches[branch]
3185 if parent == branch:
3186 parent = ""
3187 else:
3188 fullBranch = self.projectName + branch
3189 if fullBranch not in self.p4BranchesInGit:
3190 if not self.silent:
3191 print("\n Importing new branch %s" % fullBranch);
3192 if self.importNewBranch(branch, change - 1):
3193 parent = ""
3194 self.p4BranchesInGit.append(fullBranch)
3195 if not self.silent:
3196 print("\n Resuming with change %s" % change);
3198 if self.verbose:
3199 print "parent determined through known branches: %s" % parent
3201 branch = self.gitRefForBranch(branch)
3202 parent = self.gitRefForBranch(parent)
3204 if self.verbose:
3205 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3207 if len(parent) == 0 and branch in self.initialParents:
3208 parent = self.initialParents[branch]
3209 del self.initialParents[branch]
3211 blob = None
3212 if len(parent) > 0:
3213 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3214 if self.verbose:
3215 print "Creating temporary branch: " + tempBranch
3216 self.commit(description, filesForCommit, tempBranch)
3217 self.tempBranches.append(tempBranch)
3218 self.checkpoint()
3219 blob = self.searchParent(parent, branch, tempBranch)
3220 if blob:
3221 self.commit(description, filesForCommit, branch, blob)
3222 else:
3223 if self.verbose:
3224 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3225 self.commit(description, filesForCommit, branch, parent)
3226 else:
3227 files = self.extractFilesFromCommit(description)
3228 self.commit(description, files, self.branch,
3229 self.initialParent)
3230 # only needed once, to connect to the previous commit
3231 self.initialParent = ""
3232 except IOError:
3233 print self.gitError.read()
3234 sys.exit(1)
3236 def importHeadRevision(self, revision):
3237 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3239 details = {}
3240 details["user"] = "git perforce import user"
3241 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3242 % (' '.join(self.depotPaths), revision))
3243 details["change"] = revision
3244 newestRevision = 0
3246 fileCnt = 0
3247 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3249 for info in p4CmdList(["files"] + fileArgs):
3251 if 'code' in info and info['code'] == 'error':
3252 sys.stderr.write("p4 returned an error: %s\n"
3253 % info['data'])
3254 if info['data'].find("must refer to client") >= 0:
3255 sys.stderr.write("This particular p4 error is misleading.\n")
3256 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3257 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3258 sys.exit(1)
3259 if 'p4ExitCode' in info:
3260 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3261 sys.exit(1)
3264 change = int(info["change"])
3265 if change > newestRevision:
3266 newestRevision = change
3268 if info["action"] in self.delete_actions:
3269 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3270 #fileCnt = fileCnt + 1
3271 continue
3273 for prop in ["depotFile", "rev", "action", "type" ]:
3274 details["%s%s" % (prop, fileCnt)] = info[prop]
3276 fileCnt = fileCnt + 1
3278 details["change"] = newestRevision
3280 # Use time from top-most change so that all git p4 clones of
3281 # the same p4 repo have the same commit SHA1s.
3282 res = p4_describe(newestRevision)
3283 details["time"] = res["time"]
3285 self.updateOptionDict(details)
3286 try:
3287 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3288 except IOError:
3289 print "IO error with git fast-import. Is your git version recent enough?"
3290 print self.gitError.read()
3293 def run(self, args):
3294 self.depotPaths = []
3295 self.changeRange = ""
3296 self.previousDepotPaths = []
3297 self.hasOrigin = False
3299 # map from branch depot path to parent branch
3300 self.knownBranches = {}
3301 self.initialParents = {}
3303 if self.importIntoRemotes:
3304 self.refPrefix = "refs/remotes/p4/"
3305 else:
3306 self.refPrefix = "refs/heads/p4/"
3308 if self.syncWithOrigin:
3309 self.hasOrigin = originP4BranchesExist()
3310 if self.hasOrigin:
3311 if not self.silent:
3312 print 'Syncing with origin first, using "git fetch origin"'
3313 system("git fetch origin")
3315 branch_arg_given = bool(self.branch)
3316 if len(self.branch) == 0:
3317 self.branch = self.refPrefix + "master"
3318 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3319 system("git update-ref %s refs/heads/p4" % self.branch)
3320 system("git branch -D p4")
3322 # accept either the command-line option, or the configuration variable
3323 if self.useClientSpec:
3324 # will use this after clone to set the variable
3325 self.useClientSpec_from_options = True
3326 else:
3327 if gitConfigBool("git-p4.useclientspec"):
3328 self.useClientSpec = True
3329 if self.useClientSpec:
3330 self.clientSpecDirs = getClientSpec()
3332 # TODO: should always look at previous commits,
3333 # merge with previous imports, if possible.
3334 if args == []:
3335 if self.hasOrigin:
3336 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3338 # branches holds mapping from branch name to sha1
3339 branches = p4BranchesInGit(self.importIntoRemotes)
3341 # restrict to just this one, disabling detect-branches
3342 if branch_arg_given:
3343 short = self.branch.split("/")[-1]
3344 if short in branches:
3345 self.p4BranchesInGit = [ short ]
3346 else:
3347 self.p4BranchesInGit = branches.keys()
3349 if len(self.p4BranchesInGit) > 1:
3350 if not self.silent:
3351 print "Importing from/into multiple branches"
3352 self.detectBranches = True
3353 for branch in branches.keys():
3354 self.initialParents[self.refPrefix + branch] = \
3355 branches[branch]
3357 if self.verbose:
3358 print "branches: %s" % self.p4BranchesInGit
3360 p4Change = 0
3361 for branch in self.p4BranchesInGit:
3362 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3364 settings = extractSettingsGitLog(logMsg)
3366 self.readOptions(settings)
3367 if (settings.has_key('depot-paths')
3368 and settings.has_key ('change')):
3369 change = int(settings['change']) + 1
3370 p4Change = max(p4Change, change)
3372 depotPaths = sorted(settings['depot-paths'])
3373 if self.previousDepotPaths == []:
3374 self.previousDepotPaths = depotPaths
3375 else:
3376 paths = []
3377 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3378 prev_list = prev.split("/")
3379 cur_list = cur.split("/")
3380 for i in range(0, min(len(cur_list), len(prev_list))):
3381 if cur_list[i] <> prev_list[i]:
3382 i = i - 1
3383 break
3385 paths.append ("/".join(cur_list[:i + 1]))
3387 self.previousDepotPaths = paths
3389 if p4Change > 0:
3390 self.depotPaths = sorted(self.previousDepotPaths)
3391 self.changeRange = "@%s,#head" % p4Change
3392 if not self.silent and not self.detectBranches:
3393 print "Performing incremental import into %s git branch" % self.branch
3395 # accept multiple ref name abbreviations:
3396 # refs/foo/bar/branch -> use it exactly
3397 # p4/branch -> prepend refs/remotes/ or refs/heads/
3398 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3399 if not self.branch.startswith("refs/"):
3400 if self.importIntoRemotes:
3401 prepend = "refs/remotes/"
3402 else:
3403 prepend = "refs/heads/"
3404 if not self.branch.startswith("p4/"):
3405 prepend += "p4/"
3406 self.branch = prepend + self.branch
3408 if len(args) == 0 and self.depotPaths:
3409 if not self.silent:
3410 print "Depot paths: %s" % ' '.join(self.depotPaths)
3411 else:
3412 if self.depotPaths and self.depotPaths != args:
3413 print ("previous import used depot path %s and now %s was specified. "
3414 "This doesn't work!" % (' '.join (self.depotPaths),
3415 ' '.join (args)))
3416 sys.exit(1)
3418 self.depotPaths = sorted(args)
3420 revision = ""
3421 self.users = {}
3423 # Make sure no revision specifiers are used when --changesfile
3424 # is specified.
3425 bad_changesfile = False
3426 if len(self.changesFile) > 0:
3427 for p in self.depotPaths:
3428 if p.find("@") >= 0 or p.find("#") >= 0:
3429 bad_changesfile = True
3430 break
3431 if bad_changesfile:
3432 die("Option --changesfile is incompatible with revision specifiers")
3434 newPaths = []
3435 for p in self.depotPaths:
3436 if p.find("@") != -1:
3437 atIdx = p.index("@")
3438 self.changeRange = p[atIdx:]
3439 if self.changeRange == "@all":
3440 self.changeRange = ""
3441 elif ',' not in self.changeRange:
3442 revision = self.changeRange
3443 self.changeRange = ""
3444 p = p[:atIdx]
3445 elif p.find("#") != -1:
3446 hashIdx = p.index("#")
3447 revision = p[hashIdx:]
3448 p = p[:hashIdx]
3449 elif self.previousDepotPaths == []:
3450 # pay attention to changesfile, if given, else import
3451 # the entire p4 tree at the head revision
3452 if len(self.changesFile) == 0:
3453 revision = "#head"
3455 p = re.sub ("\.\.\.$", "", p)
3456 if not p.endswith("/"):
3457 p += "/"
3459 newPaths.append(p)
3461 self.depotPaths = newPaths
3463 # --detect-branches may change this for each branch
3464 self.branchPrefixes = self.depotPaths
3466 self.loadUserMapFromCache()
3467 self.labels = {}
3468 if self.detectLabels:
3469 self.getLabels();
3471 if self.detectBranches:
3472 ## FIXME - what's a P4 projectName ?
3473 self.projectName = self.guessProjectName()
3475 if self.hasOrigin:
3476 self.getBranchMappingFromGitBranches()
3477 else:
3478 self.getBranchMapping()
3479 if self.verbose:
3480 print "p4-git branches: %s" % self.p4BranchesInGit
3481 print "initial parents: %s" % self.initialParents
3482 for b in self.p4BranchesInGit:
3483 if b != "master":
3485 ## FIXME
3486 b = b[len(self.projectName):]
3487 self.createdBranches.add(b)
3489 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3491 self.importProcess = subprocess.Popen(["git", "fast-import"],
3492 stdin=subprocess.PIPE,
3493 stdout=subprocess.PIPE,
3494 stderr=subprocess.PIPE);
3495 self.gitOutput = self.importProcess.stdout
3496 self.gitStream = self.importProcess.stdin
3497 self.gitError = self.importProcess.stderr
3499 if revision:
3500 self.importHeadRevision(revision)
3501 else:
3502 changes = []
3504 if len(self.changesFile) > 0:
3505 output = open(self.changesFile).readlines()
3506 changeSet = set()
3507 for line in output:
3508 changeSet.add(int(line))
3510 for change in changeSet:
3511 changes.append(change)
3513 changes.sort()
3514 else:
3515 # catch "git p4 sync" with no new branches, in a repo that
3516 # does not have any existing p4 branches
3517 if len(args) == 0:
3518 if not self.p4BranchesInGit:
3519 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3521 # The default branch is master, unless --branch is used to
3522 # specify something else. Make sure it exists, or complain
3523 # nicely about how to use --branch.
3524 if not self.detectBranches:
3525 if not branch_exists(self.branch):
3526 if branch_arg_given:
3527 die("Error: branch %s does not exist." % self.branch)
3528 else:
3529 die("Error: no branch %s; perhaps specify one with --branch." %
3530 self.branch)
3532 if self.verbose:
3533 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3534 self.changeRange)
3535 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3537 if len(self.maxChanges) > 0:
3538 changes = changes[:min(int(self.maxChanges), len(changes))]
3540 if len(changes) == 0:
3541 if not self.silent:
3542 print "No changes to import!"
3543 else:
3544 if not self.silent and not self.detectBranches:
3545 print "Import destination: %s" % self.branch
3547 self.updatedBranches = set()
3549 if not self.detectBranches:
3550 if args:
3551 # start a new branch
3552 self.initialParent = ""
3553 else:
3554 # build on a previous revision
3555 self.initialParent = parseRevision(self.branch)
3557 self.importChanges(changes)
3559 if not self.silent:
3560 print ""
3561 if len(self.updatedBranches) > 0:
3562 sys.stdout.write("Updated branches: ")
3563 for b in self.updatedBranches:
3564 sys.stdout.write("%s " % b)
3565 sys.stdout.write("\n")
3567 if gitConfigBool("git-p4.importLabels"):
3568 self.importLabels = True
3570 if self.importLabels:
3571 p4Labels = getP4Labels(self.depotPaths)
3572 gitTags = getGitTags()
3574 missingP4Labels = p4Labels - gitTags
3575 self.importP4Labels(self.gitStream, missingP4Labels)
3577 self.gitStream.close()
3578 if self.importProcess.wait() != 0:
3579 die("fast-import failed: %s" % self.gitError.read())
3580 self.gitOutput.close()
3581 self.gitError.close()
3583 # Cleanup temporary branches created during import
3584 if self.tempBranches != []:
3585 for branch in self.tempBranches:
3586 read_pipe("git update-ref -d %s" % branch)
3587 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3589 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3590 # a convenient shortcut refname "p4".
3591 if self.importIntoRemotes:
3592 head_ref = self.refPrefix + "HEAD"
3593 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3594 system(["git", "symbolic-ref", head_ref, self.branch])
3596 return True
3598 class P4Rebase(Command):
3599 def __init__(self):
3600 Command.__init__(self)
3601 self.options = [
3602 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3604 self.importLabels = False
3605 self.description = ("Fetches the latest revision from perforce and "
3606 + "rebases the current work (branch) against it")
3608 def run(self, args):
3609 sync = P4Sync()
3610 sync.importLabels = self.importLabels
3611 sync.run([])
3613 return self.rebase()
3615 def rebase(self):
3616 if os.system("git update-index --refresh") != 0:
3617 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3618 if len(read_pipe("git diff-index HEAD --")) > 0:
3619 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3621 [upstream, settings] = findUpstreamBranchPoint()
3622 if len(upstream) == 0:
3623 die("Cannot find upstream branchpoint for rebase")
3625 # the branchpoint may be p4/foo~3, so strip off the parent
3626 upstream = re.sub("~[0-9]+$", "", upstream)
3628 print "Rebasing the current branch onto %s" % upstream
3629 oldHead = read_pipe("git rev-parse HEAD").strip()
3630 system("git rebase %s" % upstream)
3631 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3632 return True
3634 class P4Clone(P4Sync):
3635 def __init__(self):
3636 P4Sync.__init__(self)
3637 self.description = "Creates a new git repository and imports from Perforce into it"
3638 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3639 self.options += [
3640 optparse.make_option("--destination", dest="cloneDestination",
3641 action='store', default=None,
3642 help="where to leave result of the clone"),
3643 optparse.make_option("--bare", dest="cloneBare",
3644 action="store_true", default=False),
3646 self.cloneDestination = None
3647 self.needsGit = False
3648 self.cloneBare = False
3650 def defaultDestination(self, args):
3651 ## TODO: use common prefix of args?
3652 depotPath = args[0]
3653 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3654 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3655 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3656 depotDir = re.sub(r"/$", "", depotDir)
3657 return os.path.split(depotDir)[1]
3659 def run(self, args):
3660 if len(args) < 1:
3661 return False
3663 if self.keepRepoPath and not self.cloneDestination:
3664 sys.stderr.write("Must specify destination for --keep-path\n")
3665 sys.exit(1)
3667 depotPaths = args
3669 if not self.cloneDestination and len(depotPaths) > 1:
3670 self.cloneDestination = depotPaths[-1]
3671 depotPaths = depotPaths[:-1]
3673 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3674 for p in depotPaths:
3675 if not p.startswith("//"):
3676 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3677 return False
3679 if not self.cloneDestination:
3680 self.cloneDestination = self.defaultDestination(args)
3682 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3684 if not os.path.exists(self.cloneDestination):
3685 os.makedirs(self.cloneDestination)
3686 chdir(self.cloneDestination)
3688 init_cmd = [ "git", "init" ]
3689 if self.cloneBare:
3690 init_cmd.append("--bare")
3691 retcode = subprocess.call(init_cmd)
3692 if retcode:
3693 raise CalledProcessError(retcode, init_cmd)
3695 if not P4Sync.run(self, depotPaths):
3696 return False
3698 # create a master branch and check out a work tree
3699 if gitBranchExists(self.branch):
3700 system([ "git", "branch", "master", self.branch ])
3701 if not self.cloneBare:
3702 system([ "git", "checkout", "-f" ])
3703 else:
3704 print 'Not checking out any branch, use ' \
3705 '"git checkout -q -b master <branch>"'
3707 # auto-set this variable if invoked with --use-client-spec
3708 if self.useClientSpec_from_options:
3709 system("git config --bool git-p4.useclientspec true")
3711 return True
3713 class P4Branches(Command):
3714 def __init__(self):
3715 Command.__init__(self)
3716 self.options = [ ]
3717 self.description = ("Shows the git branches that hold imports and their "
3718 + "corresponding perforce depot paths")
3719 self.verbose = False
3721 def run(self, args):
3722 if originP4BranchesExist():
3723 createOrUpdateBranchesFromOrigin()
3725 cmdline = "git rev-parse --symbolic "
3726 cmdline += " --remotes"
3728 for line in read_pipe_lines(cmdline):
3729 line = line.strip()
3731 if not line.startswith('p4/') or line == "p4/HEAD":
3732 continue
3733 branch = line
3735 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3736 settings = extractSettingsGitLog(log)
3738 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3739 return True
3741 class HelpFormatter(optparse.IndentedHelpFormatter):
3742 def __init__(self):
3743 optparse.IndentedHelpFormatter.__init__(self)
3745 def format_description(self, description):
3746 if description:
3747 return description + "\n"
3748 else:
3749 return ""
3751 def printUsage(commands):
3752 print "usage: %s <command> [options]" % sys.argv[0]
3753 print ""
3754 print "valid commands: %s" % ", ".join(commands)
3755 print ""
3756 print "Try %s <command> --help for command specific help." % sys.argv[0]
3757 print ""
3759 commands = {
3760 "debug" : P4Debug,
3761 "submit" : P4Submit,
3762 "commit" : P4Submit,
3763 "sync" : P4Sync,
3764 "rebase" : P4Rebase,
3765 "clone" : P4Clone,
3766 "rollback" : P4RollBack,
3767 "branches" : P4Branches
3771 def main():
3772 if len(sys.argv[1:]) == 0:
3773 printUsage(commands.keys())
3774 sys.exit(2)
3776 cmdName = sys.argv[1]
3777 try:
3778 klass = commands[cmdName]
3779 cmd = klass()
3780 except KeyError:
3781 print "unknown command %s" % cmdName
3782 print ""
3783 printUsage(commands.keys())
3784 sys.exit(2)
3786 options = cmd.options
3787 cmd.gitdir = os.environ.get("GIT_DIR", None)
3789 args = sys.argv[2:]
3791 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3792 if cmd.needsGit:
3793 options.append(optparse.make_option("--git-dir", dest="gitdir"))
3795 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3796 options,
3797 description = cmd.description,
3798 formatter = HelpFormatter())
3800 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3801 global verbose
3802 verbose = cmd.verbose
3803 if cmd.needsGit:
3804 if cmd.gitdir == None:
3805 cmd.gitdir = os.path.abspath(".git")
3806 if not isValidGitDir(cmd.gitdir):
3807 # "rev-parse --git-dir" without arguments will try $PWD/.git
3808 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3809 if os.path.exists(cmd.gitdir):
3810 cdup = read_pipe("git rev-parse --show-cdup").strip()
3811 if len(cdup) > 0:
3812 chdir(cdup);
3814 if not isValidGitDir(cmd.gitdir):
3815 if isValidGitDir(cmd.gitdir + "/.git"):
3816 cmd.gitdir += "/.git"
3817 else:
3818 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3820 # so git commands invoked from the P4 workspace will succeed
3821 os.environ["GIT_DIR"] = cmd.gitdir
3823 if not cmd.run(args):
3824 parser.print_help()
3825 sys.exit(2)
3828 if __name__ == '__main__':
3829 main()