grep: remove redundant REG_NEWLINE when compiling fixed regex
[git.git] / git-p4.py
blob8d151da91b9699e804f4d28b865af7f44138bfc1
1 #!/usr/bin/env python
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 # 2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
10 import sys
11 if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25 import zipfile
26 import zlib
27 import ctypes
28 import errno
30 try:
31 from subprocess import CalledProcessError
32 except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
45 verbose = False
47 # Only labels/tags matching this will be imported/exported
48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
50 # Grab changes in blocks of this many revisions, unless otherwise requested
51 defaultBlockSize = 512
53 def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
60 real_cmd = ["p4"]
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
64 real_cmd += ["-u",user]
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
68 real_cmd += ["-P", password]
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
72 real_cmd += ["-p", port]
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
76 real_cmd += ["-H", host]
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
80 real_cmd += ["-c", client]
82 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
86 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
94 return real_cmd
96 def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
106 def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
128 def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
138 def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
145 def write_pipe(c, stdin):
146 if verbose:
147 sys.stderr.write('Writing pipe: %s\n' % str(c))
149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
157 return val
159 def p4_write_pipe(c, stdin):
160 real_cmd = p4_build_cmd(c)
161 return write_pipe(real_cmd, stdin)
163 def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
168 if verbose:
169 sys.stderr.write('Reading pipe: %s\n' % str(c))
171 expand = isinstance(c,basestring)
172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
174 return (p.returncode, out, err)
176 def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
187 return out
189 def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
199 def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
203 def read_pipe_lines(c):
204 if verbose:
205 sys.stderr.write('Reading pipe: %s\n' % str(c))
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
210 val = pipe.readlines()
211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
214 return val
216 def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
218 real_cmd = p4_build_cmd(c)
219 return read_pipe_lines(real_cmd)
221 def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
230 def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
249 def system(cmd, ignore_error=False):
250 expand = isinstance(cmd,basestring)
251 if verbose:
252 sys.stderr.write("executing %s\n" % str(cmd))
253 retcode = subprocess.call(cmd, shell=expand)
254 if retcode and not ignore_error:
255 raise CalledProcessError(retcode, cmd)
257 return retcode
259 def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
261 real_cmd = p4_build_cmd(cmd)
262 expand = isinstance(real_cmd, basestring)
263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
267 _p4_version_string = None
268 def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
283 def p4_integrate(src, dest):
284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
286 def p4_sync(f, *options):
287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
289 def p4_add(f):
290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
296 def p4_delete(f):
297 p4_system(["delete", wildcard_encode(f)])
299 def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
302 def p4_revert(f):
303 p4_system(["revert", wildcard_encode(f)])
305 def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
308 def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
312 def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
315 def p4_last_change():
316 results = p4CmdList(["changes", "-m", "1"])
317 return int(results[0]['change'])
319 def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
324 ds = p4CmdList(["describe", "-s", str(change)])
325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
328 d = ds[0]
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
340 return d
343 # Canonicalize the p4 type and return a tuple of the
344 # base type, plus any modifiers. See "p4 help filetypes"
345 # for a list and explanation.
347 def split_p4_type(p4type):
349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
378 # return the raw p4 type of a file (text, text+ko, etc)
380 def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382 return results[0]['headType']
385 # Given a type base and modifier, return a regexp matching
386 # the keywords that can be expanded in the file
388 def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
400 (:[^$\n]+)? # possibly an old expansion, followed by...
401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
408 # Given a file, return a regexp matching the possible
409 # RCS keywords that will be expanded, or None for files
410 # with kw expansion turned off.
412 def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
419 def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
423 p4Type = "+x"
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
432 p4_reopen(p4Type, file)
434 def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
437 result = p4_read_pipe(["opened", wildcard_encode(file)])
438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439 if match:
440 return match.group(1)
441 else:
442 die("Could not determine file type for %s (result: '%s')" % (file, result))
444 # Return the set of all p4 labels
445 def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
454 return labels
456 # Return the set of all git tags
457 def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
464 def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
471 def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
488 If the pattern is not matched, None is returned."""
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
502 return None
504 def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
509 def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
521 cmd = p4_build_cmd(cmd)
522 if verbose:
523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
536 stdin_file.flush()
537 stdin_file.seek(0)
539 p4 = subprocess.Popen(cmd,
540 shell=expand,
541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
544 result = []
545 try:
546 while True:
547 entry = marshal.load(p4.stdout)
548 if cb is not None:
549 cb(entry)
550 else:
551 result.append(entry)
552 except EOFError:
553 pass
554 exitCode = p4.wait()
555 if exitCode != 0:
556 entry = {}
557 entry["p4ExitCode"] = exitCode
558 result.append(entry)
560 return result
562 def p4Cmd(cmd):
563 list = p4CmdList(cmd)
564 result = {}
565 for entry in list:
566 result.update(entry)
567 return result;
569 def p4Where(depotPath):
570 if not depotPath.endswith("/"):
571 depotPath += "/"
572 depotPathLong = depotPath + "..."
573 outputList = p4CmdList(["where", depotPathLong])
574 output = None
575 for entry in outputList:
576 if "depotFile" in entry:
577 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
578 # The base path always ends with "/...".
579 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
580 output = entry
581 break
582 elif "data" in entry:
583 data = entry.get("data")
584 space = data.find(" ")
585 if data[:space] == depotPath:
586 output = entry
587 break
588 if output == None:
589 return ""
590 if output["code"] == "error":
591 return ""
592 clientPath = ""
593 if "path" in output:
594 clientPath = output.get("path")
595 elif "data" in output:
596 data = output.get("data")
597 lastSpace = data.rfind(" ")
598 clientPath = data[lastSpace + 1:]
600 if clientPath.endswith("..."):
601 clientPath = clientPath[:-3]
602 return clientPath
604 def currentGitBranch():
605 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
607 def isValidGitDir(path):
608 return git_dir(path) != None
610 def parseRevision(ref):
611 return read_pipe("git rev-parse %s" % ref).strip()
613 def branchExists(ref):
614 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
615 ignore_error=True)
616 return len(rev) > 0
618 def extractLogMessageFromGitCommit(commit):
619 logMessage = ""
621 ## fixme: title is first line of commit, not 1st paragraph.
622 foundTitle = False
623 for log in read_pipe_lines("git cat-file commit %s" % commit):
624 if not foundTitle:
625 if len(log) == 1:
626 foundTitle = True
627 continue
629 logMessage += log
630 return logMessage
632 def extractSettingsGitLog(log):
633 values = {}
634 for line in log.split("\n"):
635 line = line.strip()
636 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
637 if not m:
638 continue
640 assignments = m.group(1).split (':')
641 for a in assignments:
642 vals = a.split ('=')
643 key = vals[0].strip()
644 val = ('='.join (vals[1:])).strip()
645 if val.endswith ('\"') and val.startswith('"'):
646 val = val[1:-1]
648 values[key] = val
650 paths = values.get("depot-paths")
651 if not paths:
652 paths = values.get("depot-path")
653 if paths:
654 values['depot-paths'] = paths.split(',')
655 return values
657 def gitBranchExists(branch):
658 proc = subprocess.Popen(["git", "rev-parse", branch],
659 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
660 return proc.wait() == 0;
662 _gitConfig = {}
664 def gitConfig(key, typeSpecifier=None):
665 if not _gitConfig.has_key(key):
666 cmd = [ "git", "config" ]
667 if typeSpecifier:
668 cmd += [ typeSpecifier ]
669 cmd += [ key ]
670 s = read_pipe(cmd, ignore_error=True)
671 _gitConfig[key] = s.strip()
672 return _gitConfig[key]
674 def gitConfigBool(key):
675 """Return a bool, using git config --bool. It is True only if the
676 variable is set to true, and False if set to false or not present
677 in the config."""
679 if not _gitConfig.has_key(key):
680 _gitConfig[key] = gitConfig(key, '--bool') == "true"
681 return _gitConfig[key]
683 def gitConfigInt(key):
684 if not _gitConfig.has_key(key):
685 cmd = [ "git", "config", "--int", key ]
686 s = read_pipe(cmd, ignore_error=True)
687 v = s.strip()
688 try:
689 _gitConfig[key] = int(gitConfig(key, '--int'))
690 except ValueError:
691 _gitConfig[key] = None
692 return _gitConfig[key]
694 def gitConfigList(key):
695 if not _gitConfig.has_key(key):
696 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
697 _gitConfig[key] = s.strip().splitlines()
698 if _gitConfig[key] == ['']:
699 _gitConfig[key] = []
700 return _gitConfig[key]
702 def p4BranchesInGit(branchesAreInRemotes=True):
703 """Find all the branches whose names start with "p4/", looking
704 in remotes or heads as specified by the argument. Return
705 a dictionary of { branch: revision } for each one found.
706 The branch names are the short names, without any
707 "p4/" prefix."""
709 branches = {}
711 cmdline = "git rev-parse --symbolic "
712 if branchesAreInRemotes:
713 cmdline += "--remotes"
714 else:
715 cmdline += "--branches"
717 for line in read_pipe_lines(cmdline):
718 line = line.strip()
720 # only import to p4/
721 if not line.startswith('p4/'):
722 continue
723 # special symbolic ref to p4/master
724 if line == "p4/HEAD":
725 continue
727 # strip off p4/ prefix
728 branch = line[len("p4/"):]
730 branches[branch] = parseRevision(line)
732 return branches
734 def branch_exists(branch):
735 """Make sure that the given ref name really exists."""
737 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
738 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
739 out, _ = p.communicate()
740 if p.returncode:
741 return False
742 # expect exactly one line of output: the branch name
743 return out.rstrip() == branch
745 def findUpstreamBranchPoint(head = "HEAD"):
746 branches = p4BranchesInGit()
747 # map from depot-path to branch name
748 branchByDepotPath = {}
749 for branch in branches.keys():
750 tip = branches[branch]
751 log = extractLogMessageFromGitCommit(tip)
752 settings = extractSettingsGitLog(log)
753 if settings.has_key("depot-paths"):
754 paths = ",".join(settings["depot-paths"])
755 branchByDepotPath[paths] = "remotes/p4/" + branch
757 settings = None
758 parent = 0
759 while parent < 65535:
760 commit = head + "~%s" % parent
761 log = extractLogMessageFromGitCommit(commit)
762 settings = extractSettingsGitLog(log)
763 if settings.has_key("depot-paths"):
764 paths = ",".join(settings["depot-paths"])
765 if branchByDepotPath.has_key(paths):
766 return [branchByDepotPath[paths], settings]
768 parent = parent + 1
770 return ["", settings]
772 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
773 if not silent:
774 print ("Creating/updating branch(es) in %s based on origin branch(es)"
775 % localRefPrefix)
777 originPrefix = "origin/p4/"
779 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
780 line = line.strip()
781 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
782 continue
784 headName = line[len(originPrefix):]
785 remoteHead = localRefPrefix + headName
786 originHead = line
788 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
789 if (not original.has_key('depot-paths')
790 or not original.has_key('change')):
791 continue
793 update = False
794 if not gitBranchExists(remoteHead):
795 if verbose:
796 print "creating %s" % remoteHead
797 update = True
798 else:
799 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
800 if settings.has_key('change') > 0:
801 if settings['depot-paths'] == original['depot-paths']:
802 originP4Change = int(original['change'])
803 p4Change = int(settings['change'])
804 if originP4Change > p4Change:
805 print ("%s (%s) is newer than %s (%s). "
806 "Updating p4 branch from origin."
807 % (originHead, originP4Change,
808 remoteHead, p4Change))
809 update = True
810 else:
811 print ("Ignoring: %s was imported from %s while "
812 "%s was imported from %s"
813 % (originHead, ','.join(original['depot-paths']),
814 remoteHead, ','.join(settings['depot-paths'])))
816 if update:
817 system("git update-ref %s %s" % (remoteHead, originHead))
819 def originP4BranchesExist():
820 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
823 def p4ParseNumericChangeRange(parts):
824 changeStart = int(parts[0][1:])
825 if parts[1] == '#head':
826 changeEnd = p4_last_change()
827 else:
828 changeEnd = int(parts[1])
830 return (changeStart, changeEnd)
832 def chooseBlockSize(blockSize):
833 if blockSize:
834 return blockSize
835 else:
836 return defaultBlockSize
838 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
839 assert depotPaths
841 # Parse the change range into start and end. Try to find integer
842 # revision ranges as these can be broken up into blocks to avoid
843 # hitting server-side limits (maxrows, maxscanresults). But if
844 # that doesn't work, fall back to using the raw revision specifier
845 # strings, without using block mode.
847 if changeRange is None or changeRange == '':
848 changeStart = 1
849 changeEnd = p4_last_change()
850 block_size = chooseBlockSize(requestedBlockSize)
851 else:
852 parts = changeRange.split(',')
853 assert len(parts) == 2
854 try:
855 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
856 block_size = chooseBlockSize(requestedBlockSize)
857 except:
858 changeStart = parts[0][1:]
859 changeEnd = parts[1]
860 if requestedBlockSize:
861 die("cannot use --changes-block-size with non-numeric revisions")
862 block_size = None
864 changes = set()
866 # Retrieve changes a block at a time, to prevent running
867 # into a MaxResults/MaxScanRows error from the server.
869 while True:
870 cmd = ['changes']
872 if block_size:
873 end = min(changeEnd, changeStart + block_size)
874 revisionRange = "%d,%d" % (changeStart, end)
875 else:
876 revisionRange = "%s,%s" % (changeStart, changeEnd)
878 for p in depotPaths:
879 cmd += ["%s...@%s" % (p, revisionRange)]
881 # Insert changes in chronological order
882 for line in reversed(p4_read_pipe_lines(cmd)):
883 changes.add(int(line.split(" ")[1]))
885 if not block_size:
886 break
888 if end >= changeEnd:
889 break
891 changeStart = end + 1
893 changes = sorted(changes)
894 return changes
896 def p4PathStartsWith(path, prefix):
897 # This method tries to remedy a potential mixed-case issue:
899 # If UserA adds //depot/DirA/file1
900 # and UserB adds //depot/dira/file2
902 # we may or may not have a problem. If you have core.ignorecase=true,
903 # we treat DirA and dira as the same directory
904 if gitConfigBool("core.ignorecase"):
905 return path.lower().startswith(prefix.lower())
906 return path.startswith(prefix)
908 def getClientSpec():
909 """Look at the p4 client spec, create a View() object that contains
910 all the mappings, and return it."""
912 specList = p4CmdList("client -o")
913 if len(specList) != 1:
914 die('Output from "client -o" is %d lines, expecting 1' %
915 len(specList))
917 # dictionary of all client parameters
918 entry = specList[0]
920 # the //client/ name
921 client_name = entry["Client"]
923 # just the keys that start with "View"
924 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
926 # hold this new View
927 view = View(client_name)
929 # append the lines, in order, to the view
930 for view_num in range(len(view_keys)):
931 k = "View%d" % view_num
932 if k not in view_keys:
933 die("Expected view key %s missing" % k)
934 view.append(entry[k])
936 return view
938 def getClientRoot():
939 """Grab the client directory."""
941 output = p4CmdList("client -o")
942 if len(output) != 1:
943 die('Output from "client -o" is %d lines, expecting 1' % len(output))
945 entry = output[0]
946 if "Root" not in entry:
947 die('Client has no "Root"')
949 return entry["Root"]
952 # P4 wildcards are not allowed in filenames. P4 complains
953 # if you simply add them, but you can force it with "-f", in
954 # which case it translates them into %xx encoding internally.
956 def wildcard_decode(path):
957 # Search for and fix just these four characters. Do % last so
958 # that fixing it does not inadvertently create new %-escapes.
959 # Cannot have * in a filename in windows; untested as to
960 # what p4 would do in such a case.
961 if not platform.system() == "Windows":
962 path = path.replace("%2A", "*")
963 path = path.replace("%23", "#") \
964 .replace("%40", "@") \
965 .replace("%25", "%")
966 return path
968 def wildcard_encode(path):
969 # do % first to avoid double-encoding the %s introduced here
970 path = path.replace("%", "%25") \
971 .replace("*", "%2A") \
972 .replace("#", "%23") \
973 .replace("@", "%40")
974 return path
976 def wildcard_present(path):
977 m = re.search("[*#@%]", path)
978 return m is not None
980 class LargeFileSystem(object):
981 """Base class for large file system support."""
983 def __init__(self, writeToGitStream):
984 self.largeFiles = set()
985 self.writeToGitStream = writeToGitStream
987 def generatePointer(self, cloneDestination, contentFile):
988 """Return the content of a pointer file that is stored in Git instead of
989 the actual content."""
990 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
992 def pushFile(self, localLargeFile):
993 """Push the actual content which is not stored in the Git repository to
994 a server."""
995 assert False, "Method 'pushFile' required in " + self.__class__.__name__
997 def hasLargeFileExtension(self, relPath):
998 return reduce(
999 lambda a, b: a or b,
1000 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1001 False
1004 def generateTempFile(self, contents):
1005 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1006 for d in contents:
1007 contentFile.write(d)
1008 contentFile.close()
1009 return contentFile.name
1011 def exceedsLargeFileThreshold(self, relPath, contents):
1012 if gitConfigInt('git-p4.largeFileThreshold'):
1013 contentsSize = sum(len(d) for d in contents)
1014 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1015 return True
1016 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1017 contentsSize = sum(len(d) for d in contents)
1018 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1019 return False
1020 contentTempFile = self.generateTempFile(contents)
1021 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1022 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1023 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1024 zf.close()
1025 compressedContentsSize = zf.infolist()[0].compress_size
1026 os.remove(contentTempFile)
1027 os.remove(compressedContentFile.name)
1028 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1029 return True
1030 return False
1032 def addLargeFile(self, relPath):
1033 self.largeFiles.add(relPath)
1035 def removeLargeFile(self, relPath):
1036 self.largeFiles.remove(relPath)
1038 def isLargeFile(self, relPath):
1039 return relPath in self.largeFiles
1041 def processContent(self, git_mode, relPath, contents):
1042 """Processes the content of git fast import. This method decides if a
1043 file is stored in the large file system and handles all necessary
1044 steps."""
1045 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1046 contentTempFile = self.generateTempFile(contents)
1047 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1048 if pointer_git_mode:
1049 git_mode = pointer_git_mode
1050 if localLargeFile:
1051 # Move temp file to final location in large file system
1052 largeFileDir = os.path.dirname(localLargeFile)
1053 if not os.path.isdir(largeFileDir):
1054 os.makedirs(largeFileDir)
1055 shutil.move(contentTempFile, localLargeFile)
1056 self.addLargeFile(relPath)
1057 if gitConfigBool('git-p4.largeFilePush'):
1058 self.pushFile(localLargeFile)
1059 if verbose:
1060 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1061 return (git_mode, contents)
1063 class MockLFS(LargeFileSystem):
1064 """Mock large file system for testing."""
1066 def generatePointer(self, contentFile):
1067 """The pointer content is the original content prefixed with "pointer-".
1068 The local filename of the large file storage is derived from the file content.
1070 with open(contentFile, 'r') as f:
1071 content = next(f)
1072 gitMode = '100644'
1073 pointerContents = 'pointer-' + content
1074 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1075 return (gitMode, pointerContents, localLargeFile)
1077 def pushFile(self, localLargeFile):
1078 """The remote filename of the large file storage is the same as the local
1079 one but in a different directory.
1081 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1082 if not os.path.exists(remotePath):
1083 os.makedirs(remotePath)
1084 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1086 class GitLFS(LargeFileSystem):
1087 """Git LFS as backend for the git-p4 large file system.
1088 See https://git-lfs.github.com/ for details."""
1090 def __init__(self, *args):
1091 LargeFileSystem.__init__(self, *args)
1092 self.baseGitAttributes = []
1094 def generatePointer(self, contentFile):
1095 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1096 mode and content which is stored in the Git repository instead of
1097 the actual content. Return also the new location of the actual
1098 content.
1100 if os.path.getsize(contentFile) == 0:
1101 return (None, '', None)
1103 pointerProcess = subprocess.Popen(
1104 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1105 stdout=subprocess.PIPE
1107 pointerFile = pointerProcess.stdout.read()
1108 if pointerProcess.wait():
1109 os.remove(contentFile)
1110 die('git-lfs pointer command failed. Did you install the extension?')
1112 # Git LFS removed the preamble in the output of the 'pointer' command
1113 # starting from version 1.2.0. Check for the preamble here to support
1114 # earlier versions.
1115 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1116 if pointerFile.startswith('Git LFS pointer for'):
1117 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1119 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1120 localLargeFile = os.path.join(
1121 os.getcwd(),
1122 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1123 oid,
1125 # LFS Spec states that pointer files should not have the executable bit set.
1126 gitMode = '100644'
1127 return (gitMode, pointerFile, localLargeFile)
1129 def pushFile(self, localLargeFile):
1130 uploadProcess = subprocess.Popen(
1131 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1133 if uploadProcess.wait():
1134 die('git-lfs push command failed. Did you define a remote?')
1136 def generateGitAttributes(self):
1137 return (
1138 self.baseGitAttributes +
1140 '\n',
1141 '#\n',
1142 '# Git LFS (see https://git-lfs.github.com/)\n',
1143 '#\n',
1145 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1146 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1148 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1149 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1153 def addLargeFile(self, relPath):
1154 LargeFileSystem.addLargeFile(self, relPath)
1155 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1157 def removeLargeFile(self, relPath):
1158 LargeFileSystem.removeLargeFile(self, relPath)
1159 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1161 def processContent(self, git_mode, relPath, contents):
1162 if relPath == '.gitattributes':
1163 self.baseGitAttributes = contents
1164 return (git_mode, self.generateGitAttributes())
1165 else:
1166 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1168 class Command:
1169 def __init__(self):
1170 self.usage = "usage: %prog [options]"
1171 self.needsGit = True
1172 self.verbose = False
1174 class P4UserMap:
1175 def __init__(self):
1176 self.userMapFromPerforceServer = False
1177 self.myP4UserId = None
1179 def p4UserId(self):
1180 if self.myP4UserId:
1181 return self.myP4UserId
1183 results = p4CmdList("user -o")
1184 for r in results:
1185 if r.has_key('User'):
1186 self.myP4UserId = r['User']
1187 return r['User']
1188 die("Could not find your p4 user id")
1190 def p4UserIsMe(self, p4User):
1191 # return True if the given p4 user is actually me
1192 me = self.p4UserId()
1193 if not p4User or p4User != me:
1194 return False
1195 else:
1196 return True
1198 def getUserCacheFilename(self):
1199 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1200 return home + "/.gitp4-usercache.txt"
1202 def getUserMapFromPerforceServer(self):
1203 if self.userMapFromPerforceServer:
1204 return
1205 self.users = {}
1206 self.emails = {}
1208 for output in p4CmdList("users"):
1209 if not output.has_key("User"):
1210 continue
1211 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1212 self.emails[output["Email"]] = output["User"]
1214 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1215 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1216 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1217 if mapUser and len(mapUser[0]) == 3:
1218 user = mapUser[0][0]
1219 fullname = mapUser[0][1]
1220 email = mapUser[0][2]
1221 self.users[user] = fullname + " <" + email + ">"
1222 self.emails[email] = user
1224 s = ''
1225 for (key, val) in self.users.items():
1226 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1228 open(self.getUserCacheFilename(), "wb").write(s)
1229 self.userMapFromPerforceServer = True
1231 def loadUserMapFromCache(self):
1232 self.users = {}
1233 self.userMapFromPerforceServer = False
1234 try:
1235 cache = open(self.getUserCacheFilename(), "rb")
1236 lines = cache.readlines()
1237 cache.close()
1238 for line in lines:
1239 entry = line.strip().split("\t")
1240 self.users[entry[0]] = entry[1]
1241 except IOError:
1242 self.getUserMapFromPerforceServer()
1244 class P4Debug(Command):
1245 def __init__(self):
1246 Command.__init__(self)
1247 self.options = []
1248 self.description = "A tool to debug the output of p4 -G."
1249 self.needsGit = False
1251 def run(self, args):
1252 j = 0
1253 for output in p4CmdList(args):
1254 print 'Element: %d' % j
1255 j += 1
1256 print output
1257 return True
1259 class P4RollBack(Command):
1260 def __init__(self):
1261 Command.__init__(self)
1262 self.options = [
1263 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1265 self.description = "A tool to debug the multi-branch import. Don't use :)"
1266 self.rollbackLocalBranches = False
1268 def run(self, args):
1269 if len(args) != 1:
1270 return False
1271 maxChange = int(args[0])
1273 if "p4ExitCode" in p4Cmd("changes -m 1"):
1274 die("Problems executing p4");
1276 if self.rollbackLocalBranches:
1277 refPrefix = "refs/heads/"
1278 lines = read_pipe_lines("git rev-parse --symbolic --branches")
1279 else:
1280 refPrefix = "refs/remotes/"
1281 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1283 for line in lines:
1284 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1285 line = line.strip()
1286 ref = refPrefix + line
1287 log = extractLogMessageFromGitCommit(ref)
1288 settings = extractSettingsGitLog(log)
1290 depotPaths = settings['depot-paths']
1291 change = settings['change']
1293 changed = False
1295 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1296 for p in depotPaths]))) == 0:
1297 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1298 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1299 continue
1301 while change and int(change) > maxChange:
1302 changed = True
1303 if self.verbose:
1304 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1305 system("git update-ref %s \"%s^\"" % (ref, ref))
1306 log = extractLogMessageFromGitCommit(ref)
1307 settings = extractSettingsGitLog(log)
1310 depotPaths = settings['depot-paths']
1311 change = settings['change']
1313 if changed:
1314 print "%s rewound to %s" % (ref, change)
1316 return True
1318 class P4Submit(Command, P4UserMap):
1320 conflict_behavior_choices = ("ask", "skip", "quit")
1322 def __init__(self):
1323 Command.__init__(self)
1324 P4UserMap.__init__(self)
1325 self.options = [
1326 optparse.make_option("--origin", dest="origin"),
1327 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1328 # preserve the user, requires relevant p4 permissions
1329 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1330 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1331 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1332 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1333 optparse.make_option("--conflict", dest="conflict_behavior",
1334 choices=self.conflict_behavior_choices),
1335 optparse.make_option("--branch", dest="branch"),
1336 optparse.make_option("--shelve", dest="shelve", action="store_true",
1337 help="Shelve instead of submit. Shelved files are reverted, "
1338 "restoring the workspace to the state before the shelve"),
1339 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1340 metavar="CHANGELIST",
1341 help="update an existing shelved changelist, implies --shelve")
1343 self.description = "Submit changes from git to the perforce depot."
1344 self.usage += " [name of git branch to submit into perforce depot]"
1345 self.origin = ""
1346 self.detectRenames = False
1347 self.preserveUser = gitConfigBool("git-p4.preserveUser")
1348 self.dry_run = False
1349 self.shelve = False
1350 self.update_shelve = None
1351 self.prepare_p4_only = False
1352 self.conflict_behavior = None
1353 self.isWindows = (platform.system() == "Windows")
1354 self.exportLabels = False
1355 self.p4HasMoveCommand = p4_has_move_command()
1356 self.branch = None
1358 if gitConfig('git-p4.largeFileSystem'):
1359 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1361 def check(self):
1362 if len(p4CmdList("opened ...")) > 0:
1363 die("You have files opened with perforce! Close them before starting the sync.")
1365 def separate_jobs_from_description(self, message):
1366 """Extract and return a possible Jobs field in the commit
1367 message. It goes into a separate section in the p4 change
1368 specification.
1370 A jobs line starts with "Jobs:" and looks like a new field
1371 in a form. Values are white-space separated on the same
1372 line or on following lines that start with a tab.
1374 This does not parse and extract the full git commit message
1375 like a p4 form. It just sees the Jobs: line as a marker
1376 to pass everything from then on directly into the p4 form,
1377 but outside the description section.
1379 Return a tuple (stripped log message, jobs string)."""
1381 m = re.search(r'^Jobs:', message, re.MULTILINE)
1382 if m is None:
1383 return (message, None)
1385 jobtext = message[m.start():]
1386 stripped_message = message[:m.start()].rstrip()
1387 return (stripped_message, jobtext)
1389 def prepareLogMessage(self, template, message, jobs):
1390 """Edits the template returned from "p4 change -o" to insert
1391 the message in the Description field, and the jobs text in
1392 the Jobs field."""
1393 result = ""
1395 inDescriptionSection = False
1397 for line in template.split("\n"):
1398 if line.startswith("#"):
1399 result += line + "\n"
1400 continue
1402 if inDescriptionSection:
1403 if line.startswith("Files:") or line.startswith("Jobs:"):
1404 inDescriptionSection = False
1405 # insert Jobs section
1406 if jobs:
1407 result += jobs + "\n"
1408 else:
1409 continue
1410 else:
1411 if line.startswith("Description:"):
1412 inDescriptionSection = True
1413 line += "\n"
1414 for messageLine in message.split("\n"):
1415 line += "\t" + messageLine + "\n"
1417 result += line + "\n"
1419 return result
1421 def patchRCSKeywords(self, file, pattern):
1422 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1423 (handle, outFileName) = tempfile.mkstemp(dir='.')
1424 try:
1425 outFile = os.fdopen(handle, "w+")
1426 inFile = open(file, "r")
1427 regexp = re.compile(pattern, re.VERBOSE)
1428 for line in inFile.readlines():
1429 line = regexp.sub(r'$\1$', line)
1430 outFile.write(line)
1431 inFile.close()
1432 outFile.close()
1433 # Forcibly overwrite the original file
1434 os.unlink(file)
1435 shutil.move(outFileName, file)
1436 except:
1437 # cleanup our temporary file
1438 os.unlink(outFileName)
1439 print "Failed to strip RCS keywords in %s" % file
1440 raise
1442 print "Patched up RCS keywords in %s" % file
1444 def p4UserForCommit(self,id):
1445 # Return the tuple (perforce user,git email) for a given git commit id
1446 self.getUserMapFromPerforceServer()
1447 gitEmail = read_pipe(["git", "log", "--max-count=1",
1448 "--format=%ae", id])
1449 gitEmail = gitEmail.strip()
1450 if not self.emails.has_key(gitEmail):
1451 return (None,gitEmail)
1452 else:
1453 return (self.emails[gitEmail],gitEmail)
1455 def checkValidP4Users(self,commits):
1456 # check if any git authors cannot be mapped to p4 users
1457 for id in commits:
1458 (user,email) = self.p4UserForCommit(id)
1459 if not user:
1460 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1461 if gitConfigBool("git-p4.allowMissingP4Users"):
1462 print "%s" % msg
1463 else:
1464 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1466 def lastP4Changelist(self):
1467 # Get back the last changelist number submitted in this client spec. This
1468 # then gets used to patch up the username in the change. If the same
1469 # client spec is being used by multiple processes then this might go
1470 # wrong.
1471 results = p4CmdList("client -o") # find the current client
1472 client = None
1473 for r in results:
1474 if r.has_key('Client'):
1475 client = r['Client']
1476 break
1477 if not client:
1478 die("could not get client spec")
1479 results = p4CmdList(["changes", "-c", client, "-m", "1"])
1480 for r in results:
1481 if r.has_key('change'):
1482 return r['change']
1483 die("Could not get changelist number for last submit - cannot patch up user details")
1485 def modifyChangelistUser(self, changelist, newUser):
1486 # fixup the user field of a changelist after it has been submitted.
1487 changes = p4CmdList("change -o %s" % changelist)
1488 if len(changes) != 1:
1489 die("Bad output from p4 change modifying %s to user %s" %
1490 (changelist, newUser))
1492 c = changes[0]
1493 if c['User'] == newUser: return # nothing to do
1494 c['User'] = newUser
1495 input = marshal.dumps(c)
1497 result = p4CmdList("change -f -i", stdin=input)
1498 for r in result:
1499 if r.has_key('code'):
1500 if r['code'] == 'error':
1501 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1502 if r.has_key('data'):
1503 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1504 return
1505 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1507 def canChangeChangelists(self):
1508 # check to see if we have p4 admin or super-user permissions, either of
1509 # which are required to modify changelists.
1510 results = p4CmdList(["protects", self.depotPath])
1511 for r in results:
1512 if r.has_key('perm'):
1513 if r['perm'] == 'admin':
1514 return 1
1515 if r['perm'] == 'super':
1516 return 1
1517 return 0
1519 def prepareSubmitTemplate(self, changelist=None):
1520 """Run "p4 change -o" to grab a change specification template.
1521 This does not use "p4 -G", as it is nice to keep the submission
1522 template in original order, since a human might edit it.
1524 Remove lines in the Files section that show changes to files
1525 outside the depot path we're committing into."""
1527 [upstream, settings] = findUpstreamBranchPoint()
1529 template = ""
1530 inFilesSection = False
1531 args = ['change', '-o']
1532 if changelist:
1533 args.append(str(changelist))
1535 for line in p4_read_pipe_lines(args):
1536 if line.endswith("\r\n"):
1537 line = line[:-2] + "\n"
1538 if inFilesSection:
1539 if line.startswith("\t"):
1540 # path starts and ends with a tab
1541 path = line[1:]
1542 lastTab = path.rfind("\t")
1543 if lastTab != -1:
1544 path = path[:lastTab]
1545 if settings.has_key('depot-paths'):
1546 if not [p for p in settings['depot-paths']
1547 if p4PathStartsWith(path, p)]:
1548 continue
1549 else:
1550 if not p4PathStartsWith(path, self.depotPath):
1551 continue
1552 else:
1553 inFilesSection = False
1554 else:
1555 if line.startswith("Files:"):
1556 inFilesSection = True
1558 template += line
1560 return template
1562 def edit_template(self, template_file):
1563 """Invoke the editor to let the user change the submission
1564 message. Return true if okay to continue with the submit."""
1566 # if configured to skip the editing part, just submit
1567 if gitConfigBool("git-p4.skipSubmitEdit"):
1568 return True
1570 # look at the modification time, to check later if the user saved
1571 # the file
1572 mtime = os.stat(template_file).st_mtime
1574 # invoke the editor
1575 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1576 editor = os.environ.get("P4EDITOR")
1577 else:
1578 editor = read_pipe("git var GIT_EDITOR").strip()
1579 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1581 # If the file was not saved, prompt to see if this patch should
1582 # be skipped. But skip this verification step if configured so.
1583 if gitConfigBool("git-p4.skipSubmitEditCheck"):
1584 return True
1586 # modification time updated means user saved the file
1587 if os.stat(template_file).st_mtime > mtime:
1588 return True
1590 while True:
1591 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1592 if response == 'y':
1593 return True
1594 if response == 'n':
1595 return False
1597 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1598 # diff
1599 if os.environ.has_key("P4DIFF"):
1600 del(os.environ["P4DIFF"])
1601 diff = ""
1602 for editedFile in editedFiles:
1603 diff += p4_read_pipe(['diff', '-du',
1604 wildcard_encode(editedFile)])
1606 # new file diff
1607 newdiff = ""
1608 for newFile in filesToAdd:
1609 newdiff += "==== new file ====\n"
1610 newdiff += "--- /dev/null\n"
1611 newdiff += "+++ %s\n" % newFile
1613 is_link = os.path.islink(newFile)
1614 expect_link = newFile in symlinks
1616 if is_link and expect_link:
1617 newdiff += "+%s\n" % os.readlink(newFile)
1618 else:
1619 f = open(newFile, "r")
1620 for line in f.readlines():
1621 newdiff += "+" + line
1622 f.close()
1624 return (diff + newdiff).replace('\r\n', '\n')
1626 def applyCommit(self, id):
1627 """Apply one commit, return True if it succeeded."""
1629 print "Applying", read_pipe(["git", "show", "-s",
1630 "--format=format:%h %s", id])
1632 (p4User, gitEmail) = self.p4UserForCommit(id)
1634 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1635 filesToAdd = set()
1636 filesToChangeType = set()
1637 filesToDelete = set()
1638 editedFiles = set()
1639 pureRenameCopy = set()
1640 symlinks = set()
1641 filesToChangeExecBit = {}
1642 all_files = list()
1644 for line in diff:
1645 diff = parseDiffTreeEntry(line)
1646 modifier = diff['status']
1647 path = diff['src']
1648 all_files.append(path)
1650 if modifier == "M":
1651 p4_edit(path)
1652 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1653 filesToChangeExecBit[path] = diff['dst_mode']
1654 editedFiles.add(path)
1655 elif modifier == "A":
1656 filesToAdd.add(path)
1657 filesToChangeExecBit[path] = diff['dst_mode']
1658 if path in filesToDelete:
1659 filesToDelete.remove(path)
1661 dst_mode = int(diff['dst_mode'], 8)
1662 if dst_mode == 0120000:
1663 symlinks.add(path)
1665 elif modifier == "D":
1666 filesToDelete.add(path)
1667 if path in filesToAdd:
1668 filesToAdd.remove(path)
1669 elif modifier == "C":
1670 src, dest = diff['src'], diff['dst']
1671 p4_integrate(src, dest)
1672 pureRenameCopy.add(dest)
1673 if diff['src_sha1'] != diff['dst_sha1']:
1674 p4_edit(dest)
1675 pureRenameCopy.discard(dest)
1676 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1677 p4_edit(dest)
1678 pureRenameCopy.discard(dest)
1679 filesToChangeExecBit[dest] = diff['dst_mode']
1680 if self.isWindows:
1681 # turn off read-only attribute
1682 os.chmod(dest, stat.S_IWRITE)
1683 os.unlink(dest)
1684 editedFiles.add(dest)
1685 elif modifier == "R":
1686 src, dest = diff['src'], diff['dst']
1687 if self.p4HasMoveCommand:
1688 p4_edit(src) # src must be open before move
1689 p4_move(src, dest) # opens for (move/delete, move/add)
1690 else:
1691 p4_integrate(src, dest)
1692 if diff['src_sha1'] != diff['dst_sha1']:
1693 p4_edit(dest)
1694 else:
1695 pureRenameCopy.add(dest)
1696 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1697 if not self.p4HasMoveCommand:
1698 p4_edit(dest) # with move: already open, writable
1699 filesToChangeExecBit[dest] = diff['dst_mode']
1700 if not self.p4HasMoveCommand:
1701 if self.isWindows:
1702 os.chmod(dest, stat.S_IWRITE)
1703 os.unlink(dest)
1704 filesToDelete.add(src)
1705 editedFiles.add(dest)
1706 elif modifier == "T":
1707 filesToChangeType.add(path)
1708 else:
1709 die("unknown modifier %s for %s" % (modifier, path))
1711 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1712 patchcmd = diffcmd + " | git apply "
1713 tryPatchCmd = patchcmd + "--check -"
1714 applyPatchCmd = patchcmd + "--check --apply -"
1715 patch_succeeded = True
1717 if os.system(tryPatchCmd) != 0:
1718 fixed_rcs_keywords = False
1719 patch_succeeded = False
1720 print "Unfortunately applying the change failed!"
1722 # Patch failed, maybe it's just RCS keyword woes. Look through
1723 # the patch to see if that's possible.
1724 if gitConfigBool("git-p4.attemptRCSCleanup"):
1725 file = None
1726 pattern = None
1727 kwfiles = {}
1728 for file in editedFiles | filesToDelete:
1729 # did this file's delta contain RCS keywords?
1730 pattern = p4_keywords_regexp_for_file(file)
1732 if pattern:
1733 # this file is a possibility...look for RCS keywords.
1734 regexp = re.compile(pattern, re.VERBOSE)
1735 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1736 if regexp.search(line):
1737 if verbose:
1738 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1739 kwfiles[file] = pattern
1740 break
1742 for file in kwfiles:
1743 if verbose:
1744 print "zapping %s with %s" % (line,pattern)
1745 # File is being deleted, so not open in p4. Must
1746 # disable the read-only bit on windows.
1747 if self.isWindows and file not in editedFiles:
1748 os.chmod(file, stat.S_IWRITE)
1749 self.patchRCSKeywords(file, kwfiles[file])
1750 fixed_rcs_keywords = True
1752 if fixed_rcs_keywords:
1753 print "Retrying the patch with RCS keywords cleaned up"
1754 if os.system(tryPatchCmd) == 0:
1755 patch_succeeded = True
1757 if not patch_succeeded:
1758 for f in editedFiles:
1759 p4_revert(f)
1760 return False
1763 # Apply the patch for real, and do add/delete/+x handling.
1765 system(applyPatchCmd)
1767 for f in filesToChangeType:
1768 p4_edit(f, "-t", "auto")
1769 for f in filesToAdd:
1770 p4_add(f)
1771 for f in filesToDelete:
1772 p4_revert(f)
1773 p4_delete(f)
1775 # Set/clear executable bits
1776 for f in filesToChangeExecBit.keys():
1777 mode = filesToChangeExecBit[f]
1778 setP4ExecBit(f, mode)
1780 if self.update_shelve:
1781 print("all_files = %s" % str(all_files))
1782 p4_reopen_in_change(self.update_shelve, all_files)
1785 # Build p4 change description, starting with the contents
1786 # of the git commit message.
1788 logMessage = extractLogMessageFromGitCommit(id)
1789 logMessage = logMessage.strip()
1790 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1792 template = self.prepareSubmitTemplate(self.update_shelve)
1793 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1795 if self.preserveUser:
1796 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1798 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1799 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1800 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1801 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1803 separatorLine = "######## everything below this line is just the diff #######\n"
1804 if not self.prepare_p4_only:
1805 submitTemplate += separatorLine
1806 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1808 (handle, fileName) = tempfile.mkstemp()
1809 tmpFile = os.fdopen(handle, "w+b")
1810 if self.isWindows:
1811 submitTemplate = submitTemplate.replace("\n", "\r\n")
1812 tmpFile.write(submitTemplate)
1813 tmpFile.close()
1815 if self.prepare_p4_only:
1817 # Leave the p4 tree prepared, and the submit template around
1818 # and let the user decide what to do next
1820 print
1821 print "P4 workspace prepared for submission."
1822 print "To submit or revert, go to client workspace"
1823 print " " + self.clientPath
1824 print
1825 print "To submit, use \"p4 submit\" to write a new description,"
1826 print "or \"p4 submit -i <%s\" to use the one prepared by" \
1827 " \"git p4\"." % fileName
1828 print "You can delete the file \"%s\" when finished." % fileName
1830 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1831 print "To preserve change ownership by user %s, you must\n" \
1832 "do \"p4 change -f <change>\" after submitting and\n" \
1833 "edit the User field."
1834 if pureRenameCopy:
1835 print "After submitting, renamed files must be re-synced."
1836 print "Invoke \"p4 sync -f\" on each of these files:"
1837 for f in pureRenameCopy:
1838 print " " + f
1840 print
1841 print "To revert the changes, use \"p4 revert ...\", and delete"
1842 print "the submit template file \"%s\"" % fileName
1843 if filesToAdd:
1844 print "Since the commit adds new files, they must be deleted:"
1845 for f in filesToAdd:
1846 print " " + f
1847 print
1848 return True
1851 # Let the user edit the change description, then submit it.
1853 submitted = False
1855 try:
1856 if self.edit_template(fileName):
1857 # read the edited message and submit
1858 tmpFile = open(fileName, "rb")
1859 message = tmpFile.read()
1860 tmpFile.close()
1861 if self.isWindows:
1862 message = message.replace("\r\n", "\n")
1863 submitTemplate = message[:message.index(separatorLine)]
1865 if self.update_shelve:
1866 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1867 elif self.shelve:
1868 p4_write_pipe(['shelve', '-i'], submitTemplate)
1869 else:
1870 p4_write_pipe(['submit', '-i'], submitTemplate)
1871 # The rename/copy happened by applying a patch that created a
1872 # new file. This leaves it writable, which confuses p4.
1873 for f in pureRenameCopy:
1874 p4_sync(f, "-f")
1876 if self.preserveUser:
1877 if p4User:
1878 # Get last changelist number. Cannot easily get it from
1879 # the submit command output as the output is
1880 # unmarshalled.
1881 changelist = self.lastP4Changelist()
1882 self.modifyChangelistUser(changelist, p4User)
1884 submitted = True
1886 finally:
1887 # skip this patch
1888 if not submitted or self.shelve:
1889 if self.shelve:
1890 print ("Reverting shelved files.")
1891 else:
1892 print ("Submission cancelled, undoing p4 changes.")
1893 for f in editedFiles | filesToDelete:
1894 p4_revert(f)
1895 for f in filesToAdd:
1896 p4_revert(f)
1897 os.remove(f)
1899 os.remove(fileName)
1900 return submitted
1902 # Export git tags as p4 labels. Create a p4 label and then tag
1903 # with that.
1904 def exportGitTags(self, gitTags):
1905 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1906 if len(validLabelRegexp) == 0:
1907 validLabelRegexp = defaultLabelRegexp
1908 m = re.compile(validLabelRegexp)
1910 for name in gitTags:
1912 if not m.match(name):
1913 if verbose:
1914 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1915 continue
1917 # Get the p4 commit this corresponds to
1918 logMessage = extractLogMessageFromGitCommit(name)
1919 values = extractSettingsGitLog(logMessage)
1921 if not values.has_key('change'):
1922 # a tag pointing to something not sent to p4; ignore
1923 if verbose:
1924 print "git tag %s does not give a p4 commit" % name
1925 continue
1926 else:
1927 changelist = values['change']
1929 # Get the tag details.
1930 inHeader = True
1931 isAnnotated = False
1932 body = []
1933 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1934 l = l.strip()
1935 if inHeader:
1936 if re.match(r'tag\s+', l):
1937 isAnnotated = True
1938 elif re.match(r'\s*$', l):
1939 inHeader = False
1940 continue
1941 else:
1942 body.append(l)
1944 if not isAnnotated:
1945 body = ["lightweight tag imported by git p4\n"]
1947 # Create the label - use the same view as the client spec we are using
1948 clientSpec = getClientSpec()
1950 labelTemplate = "Label: %s\n" % name
1951 labelTemplate += "Description:\n"
1952 for b in body:
1953 labelTemplate += "\t" + b + "\n"
1954 labelTemplate += "View:\n"
1955 for depot_side in clientSpec.mappings:
1956 labelTemplate += "\t%s\n" % depot_side
1958 if self.dry_run:
1959 print "Would create p4 label %s for tag" % name
1960 elif self.prepare_p4_only:
1961 print "Not creating p4 label %s for tag due to option" \
1962 " --prepare-p4-only" % name
1963 else:
1964 p4_write_pipe(["label", "-i"], labelTemplate)
1966 # Use the label
1967 p4_system(["tag", "-l", name] +
1968 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1970 if verbose:
1971 print "created p4 label for tag %s" % name
1973 def run(self, args):
1974 if len(args) == 0:
1975 self.master = currentGitBranch()
1976 elif len(args) == 1:
1977 self.master = args[0]
1978 if not branchExists(self.master):
1979 die("Branch %s does not exist" % self.master)
1980 else:
1981 return False
1983 if self.master:
1984 allowSubmit = gitConfig("git-p4.allowSubmit")
1985 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1986 die("%s is not in git-p4.allowSubmit" % self.master)
1988 [upstream, settings] = findUpstreamBranchPoint()
1989 self.depotPath = settings['depot-paths'][0]
1990 if len(self.origin) == 0:
1991 self.origin = upstream
1993 if self.update_shelve:
1994 self.shelve = True
1996 if self.preserveUser:
1997 if not self.canChangeChangelists():
1998 die("Cannot preserve user names without p4 super-user or admin permissions")
2000 # if not set from the command line, try the config file
2001 if self.conflict_behavior is None:
2002 val = gitConfig("git-p4.conflict")
2003 if val:
2004 if val not in self.conflict_behavior_choices:
2005 die("Invalid value '%s' for config git-p4.conflict" % val)
2006 else:
2007 val = "ask"
2008 self.conflict_behavior = val
2010 if self.verbose:
2011 print "Origin branch is " + self.origin
2013 if len(self.depotPath) == 0:
2014 print "Internal error: cannot locate perforce depot path from existing branches"
2015 sys.exit(128)
2017 self.useClientSpec = False
2018 if gitConfigBool("git-p4.useclientspec"):
2019 self.useClientSpec = True
2020 if self.useClientSpec:
2021 self.clientSpecDirs = getClientSpec()
2023 # Check for the existence of P4 branches
2024 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2026 if self.useClientSpec and not branchesDetected:
2027 # all files are relative to the client spec
2028 self.clientPath = getClientRoot()
2029 else:
2030 self.clientPath = p4Where(self.depotPath)
2032 if self.clientPath == "":
2033 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2035 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2036 self.oldWorkingDirectory = os.getcwd()
2038 # ensure the clientPath exists
2039 new_client_dir = False
2040 if not os.path.exists(self.clientPath):
2041 new_client_dir = True
2042 os.makedirs(self.clientPath)
2044 chdir(self.clientPath, is_client_path=True)
2045 if self.dry_run:
2046 print "Would synchronize p4 checkout in %s" % self.clientPath
2047 else:
2048 print "Synchronizing p4 checkout..."
2049 if new_client_dir:
2050 # old one was destroyed, and maybe nobody told p4
2051 p4_sync("...", "-f")
2052 else:
2053 p4_sync("...")
2054 self.check()
2056 commits = []
2057 if self.master:
2058 commitish = self.master
2059 else:
2060 commitish = 'HEAD'
2062 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2063 commits.append(line.strip())
2064 commits.reverse()
2066 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2067 self.checkAuthorship = False
2068 else:
2069 self.checkAuthorship = True
2071 if self.preserveUser:
2072 self.checkValidP4Users(commits)
2075 # Build up a set of options to be passed to diff when
2076 # submitting each commit to p4.
2078 if self.detectRenames:
2079 # command-line -M arg
2080 self.diffOpts = "-M"
2081 else:
2082 # If not explicitly set check the config variable
2083 detectRenames = gitConfig("git-p4.detectRenames")
2085 if detectRenames.lower() == "false" or detectRenames == "":
2086 self.diffOpts = ""
2087 elif detectRenames.lower() == "true":
2088 self.diffOpts = "-M"
2089 else:
2090 self.diffOpts = "-M%s" % detectRenames
2092 # no command-line arg for -C or --find-copies-harder, just
2093 # config variables
2094 detectCopies = gitConfig("git-p4.detectCopies")
2095 if detectCopies.lower() == "false" or detectCopies == "":
2096 pass
2097 elif detectCopies.lower() == "true":
2098 self.diffOpts += " -C"
2099 else:
2100 self.diffOpts += " -C%s" % detectCopies
2102 if gitConfigBool("git-p4.detectCopiesHarder"):
2103 self.diffOpts += " --find-copies-harder"
2106 # Apply the commits, one at a time. On failure, ask if should
2107 # continue to try the rest of the patches, or quit.
2109 if self.dry_run:
2110 print "Would apply"
2111 applied = []
2112 last = len(commits) - 1
2113 for i, commit in enumerate(commits):
2114 if self.dry_run:
2115 print " ", read_pipe(["git", "show", "-s",
2116 "--format=format:%h %s", commit])
2117 ok = True
2118 else:
2119 ok = self.applyCommit(commit)
2120 if ok:
2121 applied.append(commit)
2122 else:
2123 if self.prepare_p4_only and i < last:
2124 print "Processing only the first commit due to option" \
2125 " --prepare-p4-only"
2126 break
2127 if i < last:
2128 quit = False
2129 while True:
2130 # prompt for what to do, or use the option/variable
2131 if self.conflict_behavior == "ask":
2132 print "What do you want to do?"
2133 response = raw_input("[s]kip this commit but apply"
2134 " the rest, or [q]uit? ")
2135 if not response:
2136 continue
2137 elif self.conflict_behavior == "skip":
2138 response = "s"
2139 elif self.conflict_behavior == "quit":
2140 response = "q"
2141 else:
2142 die("Unknown conflict_behavior '%s'" %
2143 self.conflict_behavior)
2145 if response[0] == "s":
2146 print "Skipping this commit, but applying the rest"
2147 break
2148 if response[0] == "q":
2149 print "Quitting"
2150 quit = True
2151 break
2152 if quit:
2153 break
2155 chdir(self.oldWorkingDirectory)
2156 shelved_applied = "shelved" if self.shelve else "applied"
2157 if self.dry_run:
2158 pass
2159 elif self.prepare_p4_only:
2160 pass
2161 elif len(commits) == len(applied):
2162 print ("All commits {0}!".format(shelved_applied))
2164 sync = P4Sync()
2165 if self.branch:
2166 sync.branch = self.branch
2167 sync.run([])
2169 rebase = P4Rebase()
2170 rebase.rebase()
2172 else:
2173 if len(applied) == 0:
2174 print ("No commits {0}.".format(shelved_applied))
2175 else:
2176 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2177 for c in commits:
2178 if c in applied:
2179 star = "*"
2180 else:
2181 star = " "
2182 print star, read_pipe(["git", "show", "-s",
2183 "--format=format:%h %s", c])
2184 print "You will have to do 'git p4 sync' and rebase."
2186 if gitConfigBool("git-p4.exportLabels"):
2187 self.exportLabels = True
2189 if self.exportLabels:
2190 p4Labels = getP4Labels(self.depotPath)
2191 gitTags = getGitTags()
2193 missingGitTags = gitTags - p4Labels
2194 self.exportGitTags(missingGitTags)
2196 # exit with error unless everything applied perfectly
2197 if len(commits) != len(applied):
2198 sys.exit(1)
2200 return True
2202 class View(object):
2203 """Represent a p4 view ("p4 help views"), and map files in a
2204 repo according to the view."""
2206 def __init__(self, client_name):
2207 self.mappings = []
2208 self.client_prefix = "//%s/" % client_name
2209 # cache results of "p4 where" to lookup client file locations
2210 self.client_spec_path_cache = {}
2212 def append(self, view_line):
2213 """Parse a view line, splitting it into depot and client
2214 sides. Append to self.mappings, preserving order. This
2215 is only needed for tag creation."""
2217 # Split the view line into exactly two words. P4 enforces
2218 # structure on these lines that simplifies this quite a bit.
2220 # Either or both words may be double-quoted.
2221 # Single quotes do not matter.
2222 # Double-quote marks cannot occur inside the words.
2223 # A + or - prefix is also inside the quotes.
2224 # There are no quotes unless they contain a space.
2225 # The line is already white-space stripped.
2226 # The two words are separated by a single space.
2228 if view_line[0] == '"':
2229 # First word is double quoted. Find its end.
2230 close_quote_index = view_line.find('"', 1)
2231 if close_quote_index <= 0:
2232 die("No first-word closing quote found: %s" % view_line)
2233 depot_side = view_line[1:close_quote_index]
2234 # skip closing quote and space
2235 rhs_index = close_quote_index + 1 + 1
2236 else:
2237 space_index = view_line.find(" ")
2238 if space_index <= 0:
2239 die("No word-splitting space found: %s" % view_line)
2240 depot_side = view_line[0:space_index]
2241 rhs_index = space_index + 1
2243 # prefix + means overlay on previous mapping
2244 if depot_side.startswith("+"):
2245 depot_side = depot_side[1:]
2247 # prefix - means exclude this path, leave out of mappings
2248 exclude = False
2249 if depot_side.startswith("-"):
2250 exclude = True
2251 depot_side = depot_side[1:]
2253 if not exclude:
2254 self.mappings.append(depot_side)
2256 def convert_client_path(self, clientFile):
2257 # chop off //client/ part to make it relative
2258 if not clientFile.startswith(self.client_prefix):
2259 die("No prefix '%s' on clientFile '%s'" %
2260 (self.client_prefix, clientFile))
2261 return clientFile[len(self.client_prefix):]
2263 def update_client_spec_path_cache(self, files):
2264 """ Caching file paths by "p4 where" batch query """
2266 # List depot file paths exclude that already cached
2267 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2269 if len(fileArgs) == 0:
2270 return # All files in cache
2272 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2273 for res in where_result:
2274 if "code" in res and res["code"] == "error":
2275 # assume error is "... file(s) not in client view"
2276 continue
2277 if "clientFile" not in res:
2278 die("No clientFile in 'p4 where' output")
2279 if "unmap" in res:
2280 # it will list all of them, but only one not unmap-ped
2281 continue
2282 if gitConfigBool("core.ignorecase"):
2283 res['depotFile'] = res['depotFile'].lower()
2284 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2286 # not found files or unmap files set to ""
2287 for depotFile in fileArgs:
2288 if gitConfigBool("core.ignorecase"):
2289 depotFile = depotFile.lower()
2290 if depotFile not in self.client_spec_path_cache:
2291 self.client_spec_path_cache[depotFile] = ""
2293 def map_in_client(self, depot_path):
2294 """Return the relative location in the client where this
2295 depot file should live. Returns "" if the file should
2296 not be mapped in the client."""
2298 if gitConfigBool("core.ignorecase"):
2299 depot_path = depot_path.lower()
2301 if depot_path in self.client_spec_path_cache:
2302 return self.client_spec_path_cache[depot_path]
2304 die( "Error: %s is not found in client spec path" % depot_path )
2305 return ""
2307 class P4Sync(Command, P4UserMap):
2308 delete_actions = ( "delete", "move/delete", "purge" )
2310 def __init__(self):
2311 Command.__init__(self)
2312 P4UserMap.__init__(self)
2313 self.options = [
2314 optparse.make_option("--branch", dest="branch"),
2315 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2316 optparse.make_option("--changesfile", dest="changesFile"),
2317 optparse.make_option("--silent", dest="silent", action="store_true"),
2318 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2319 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2320 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2321 help="Import into refs/heads/ , not refs/remotes"),
2322 optparse.make_option("--max-changes", dest="maxChanges",
2323 help="Maximum number of changes to import"),
2324 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2325 help="Internal block size to use when iteratively calling p4 changes"),
2326 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2327 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2328 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2329 help="Only sync files that are included in the Perforce Client Spec"),
2330 optparse.make_option("-/", dest="cloneExclude",
2331 action="append", type="string",
2332 help="exclude depot path"),
2334 self.description = """Imports from Perforce into a git repository.\n
2335 example:
2336 //depot/my/project/ -- to import the current head
2337 //depot/my/project/@all -- to import everything
2338 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2340 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2342 self.usage += " //depot/path[@revRange]"
2343 self.silent = False
2344 self.createdBranches = set()
2345 self.committedChanges = set()
2346 self.branch = ""
2347 self.detectBranches = False
2348 self.detectLabels = False
2349 self.importLabels = False
2350 self.changesFile = ""
2351 self.syncWithOrigin = True
2352 self.importIntoRemotes = True
2353 self.maxChanges = ""
2354 self.changes_block_size = None
2355 self.keepRepoPath = False
2356 self.depotPaths = None
2357 self.p4BranchesInGit = []
2358 self.cloneExclude = []
2359 self.useClientSpec = False
2360 self.useClientSpec_from_options = False
2361 self.clientSpecDirs = None
2362 self.tempBranches = []
2363 self.tempBranchLocation = "refs/git-p4-tmp"
2364 self.largeFileSystem = None
2366 if gitConfig('git-p4.largeFileSystem'):
2367 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2368 self.largeFileSystem = largeFileSystemConstructor(
2369 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2372 if gitConfig("git-p4.syncFromOrigin") == "false":
2373 self.syncWithOrigin = False
2375 # This is required for the "append" cloneExclude action
2376 def ensure_value(self, attr, value):
2377 if not hasattr(self, attr) or getattr(self, attr) is None:
2378 setattr(self, attr, value)
2379 return getattr(self, attr)
2381 # Force a checkpoint in fast-import and wait for it to finish
2382 def checkpoint(self):
2383 self.gitStream.write("checkpoint\n\n")
2384 self.gitStream.write("progress checkpoint\n\n")
2385 out = self.gitOutput.readline()
2386 if self.verbose:
2387 print "checkpoint finished: " + out
2389 def extractFilesFromCommit(self, commit):
2390 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2391 for path in self.cloneExclude]
2392 files = []
2393 fnum = 0
2394 while commit.has_key("depotFile%s" % fnum):
2395 path = commit["depotFile%s" % fnum]
2397 if [p for p in self.cloneExclude
2398 if p4PathStartsWith(path, p)]:
2399 found = False
2400 else:
2401 found = [p for p in self.depotPaths
2402 if p4PathStartsWith(path, p)]
2403 if not found:
2404 fnum = fnum + 1
2405 continue
2407 file = {}
2408 file["path"] = path
2409 file["rev"] = commit["rev%s" % fnum]
2410 file["action"] = commit["action%s" % fnum]
2411 file["type"] = commit["type%s" % fnum]
2412 files.append(file)
2413 fnum = fnum + 1
2414 return files
2416 def extractJobsFromCommit(self, commit):
2417 jobs = []
2418 jnum = 0
2419 while commit.has_key("job%s" % jnum):
2420 job = commit["job%s" % jnum]
2421 jobs.append(job)
2422 jnum = jnum + 1
2423 return jobs
2425 def stripRepoPath(self, path, prefixes):
2426 """When streaming files, this is called to map a p4 depot path
2427 to where it should go in git. The prefixes are either
2428 self.depotPaths, or self.branchPrefixes in the case of
2429 branch detection."""
2431 if self.useClientSpec:
2432 # branch detection moves files up a level (the branch name)
2433 # from what client spec interpretation gives
2434 path = self.clientSpecDirs.map_in_client(path)
2435 if self.detectBranches:
2436 for b in self.knownBranches:
2437 if path.startswith(b + "/"):
2438 path = path[len(b)+1:]
2440 elif self.keepRepoPath:
2441 # Preserve everything in relative path name except leading
2442 # //depot/; just look at first prefix as they all should
2443 # be in the same depot.
2444 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2445 if p4PathStartsWith(path, depot):
2446 path = path[len(depot):]
2448 else:
2449 for p in prefixes:
2450 if p4PathStartsWith(path, p):
2451 path = path[len(p):]
2452 break
2454 path = wildcard_decode(path)
2455 return path
2457 def splitFilesIntoBranches(self, commit):
2458 """Look at each depotFile in the commit to figure out to what
2459 branch it belongs."""
2461 if self.clientSpecDirs:
2462 files = self.extractFilesFromCommit(commit)
2463 self.clientSpecDirs.update_client_spec_path_cache(files)
2465 branches = {}
2466 fnum = 0
2467 while commit.has_key("depotFile%s" % fnum):
2468 path = commit["depotFile%s" % fnum]
2469 found = [p for p in self.depotPaths
2470 if p4PathStartsWith(path, p)]
2471 if not found:
2472 fnum = fnum + 1
2473 continue
2475 file = {}
2476 file["path"] = path
2477 file["rev"] = commit["rev%s" % fnum]
2478 file["action"] = commit["action%s" % fnum]
2479 file["type"] = commit["type%s" % fnum]
2480 fnum = fnum + 1
2482 # start with the full relative path where this file would
2483 # go in a p4 client
2484 if self.useClientSpec:
2485 relPath = self.clientSpecDirs.map_in_client(path)
2486 else:
2487 relPath = self.stripRepoPath(path, self.depotPaths)
2489 for branch in self.knownBranches.keys():
2490 # add a trailing slash so that a commit into qt/4.2foo
2491 # doesn't end up in qt/4.2, e.g.
2492 if relPath.startswith(branch + "/"):
2493 if branch not in branches:
2494 branches[branch] = []
2495 branches[branch].append(file)
2496 break
2498 return branches
2500 def writeToGitStream(self, gitMode, relPath, contents):
2501 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2502 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2503 for d in contents:
2504 self.gitStream.write(d)
2505 self.gitStream.write('\n')
2507 def encodeWithUTF8(self, path):
2508 try:
2509 path.decode('ascii')
2510 except:
2511 encoding = 'utf8'
2512 if gitConfig('git-p4.pathEncoding'):
2513 encoding = gitConfig('git-p4.pathEncoding')
2514 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2515 if self.verbose:
2516 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2517 return path
2519 # output one file from the P4 stream
2520 # - helper for streamP4Files
2522 def streamOneP4File(self, file, contents):
2523 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2524 relPath = self.encodeWithUTF8(relPath)
2525 if verbose:
2526 size = int(self.stream_file['fileSize'])
2527 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2528 sys.stdout.flush()
2530 (type_base, type_mods) = split_p4_type(file["type"])
2532 git_mode = "100644"
2533 if "x" in type_mods:
2534 git_mode = "100755"
2535 if type_base == "symlink":
2536 git_mode = "120000"
2537 # p4 print on a symlink sometimes contains "target\n";
2538 # if it does, remove the newline
2539 data = ''.join(contents)
2540 if not data:
2541 # Some version of p4 allowed creating a symlink that pointed
2542 # to nothing. This causes p4 errors when checking out such
2543 # a change, and errors here too. Work around it by ignoring
2544 # the bad symlink; hopefully a future change fixes it.
2545 print "\nIgnoring empty symlink in %s" % file['depotFile']
2546 return
2547 elif data[-1] == '\n':
2548 contents = [data[:-1]]
2549 else:
2550 contents = [data]
2552 if type_base == "utf16":
2553 # p4 delivers different text in the python output to -G
2554 # than it does when using "print -o", or normal p4 client
2555 # operations. utf16 is converted to ascii or utf8, perhaps.
2556 # But ascii text saved as -t utf16 is completely mangled.
2557 # Invoke print -o to get the real contents.
2559 # On windows, the newlines will always be mangled by print, so put
2560 # them back too. This is not needed to the cygwin windows version,
2561 # just the native "NT" type.
2563 try:
2564 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2565 except Exception as e:
2566 if 'Translation of file content failed' in str(e):
2567 type_base = 'binary'
2568 else:
2569 raise e
2570 else:
2571 if p4_version_string().find('/NT') >= 0:
2572 text = text.replace('\r\n', '\n')
2573 contents = [ text ]
2575 if type_base == "apple":
2576 # Apple filetype files will be streamed as a concatenation of
2577 # its appledouble header and the contents. This is useless
2578 # on both macs and non-macs. If using "print -q -o xx", it
2579 # will create "xx" with the data, and "%xx" with the header.
2580 # This is also not very useful.
2582 # Ideally, someday, this script can learn how to generate
2583 # appledouble files directly and import those to git, but
2584 # non-mac machines can never find a use for apple filetype.
2585 print "\nIgnoring apple filetype file %s" % file['depotFile']
2586 return
2588 # Note that we do not try to de-mangle keywords on utf16 files,
2589 # even though in theory somebody may want that.
2590 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2591 if pattern:
2592 regexp = re.compile(pattern, re.VERBOSE)
2593 text = ''.join(contents)
2594 text = regexp.sub(r'$\1$', text)
2595 contents = [ text ]
2597 if self.largeFileSystem:
2598 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2600 self.writeToGitStream(git_mode, relPath, contents)
2602 def streamOneP4Deletion(self, file):
2603 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2604 relPath = self.encodeWithUTF8(relPath)
2605 if verbose:
2606 sys.stdout.write("delete %s\n" % relPath)
2607 sys.stdout.flush()
2608 self.gitStream.write("D %s\n" % relPath)
2610 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2611 self.largeFileSystem.removeLargeFile(relPath)
2613 # handle another chunk of streaming data
2614 def streamP4FilesCb(self, marshalled):
2616 # catch p4 errors and complain
2617 err = None
2618 if "code" in marshalled:
2619 if marshalled["code"] == "error":
2620 if "data" in marshalled:
2621 err = marshalled["data"].rstrip()
2623 if not err and 'fileSize' in self.stream_file:
2624 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2625 if required_bytes > 0:
2626 err = 'Not enough space left on %s! Free at least %i MB.' % (
2627 os.getcwd(), required_bytes/1024/1024
2630 if err:
2631 f = None
2632 if self.stream_have_file_info:
2633 if "depotFile" in self.stream_file:
2634 f = self.stream_file["depotFile"]
2635 # force a failure in fast-import, else an empty
2636 # commit will be made
2637 self.gitStream.write("\n")
2638 self.gitStream.write("die-now\n")
2639 self.gitStream.close()
2640 # ignore errors, but make sure it exits first
2641 self.importProcess.wait()
2642 if f:
2643 die("Error from p4 print for %s: %s" % (f, err))
2644 else:
2645 die("Error from p4 print: %s" % err)
2647 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2648 # start of a new file - output the old one first
2649 self.streamOneP4File(self.stream_file, self.stream_contents)
2650 self.stream_file = {}
2651 self.stream_contents = []
2652 self.stream_have_file_info = False
2654 # pick up the new file information... for the
2655 # 'data' field we need to append to our array
2656 for k in marshalled.keys():
2657 if k == 'data':
2658 if 'streamContentSize' not in self.stream_file:
2659 self.stream_file['streamContentSize'] = 0
2660 self.stream_file['streamContentSize'] += len(marshalled['data'])
2661 self.stream_contents.append(marshalled['data'])
2662 else:
2663 self.stream_file[k] = marshalled[k]
2665 if (verbose and
2666 'streamContentSize' in self.stream_file and
2667 'fileSize' in self.stream_file and
2668 'depotFile' in self.stream_file):
2669 size = int(self.stream_file["fileSize"])
2670 if size > 0:
2671 progress = 100*self.stream_file['streamContentSize']/size
2672 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2673 sys.stdout.flush()
2675 self.stream_have_file_info = True
2677 # Stream directly from "p4 files" into "git fast-import"
2678 def streamP4Files(self, files):
2679 filesForCommit = []
2680 filesToRead = []
2681 filesToDelete = []
2683 for f in files:
2684 filesForCommit.append(f)
2685 if f['action'] in self.delete_actions:
2686 filesToDelete.append(f)
2687 else:
2688 filesToRead.append(f)
2690 # deleted files...
2691 for f in filesToDelete:
2692 self.streamOneP4Deletion(f)
2694 if len(filesToRead) > 0:
2695 self.stream_file = {}
2696 self.stream_contents = []
2697 self.stream_have_file_info = False
2699 # curry self argument
2700 def streamP4FilesCbSelf(entry):
2701 self.streamP4FilesCb(entry)
2703 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2705 p4CmdList(["-x", "-", "print"],
2706 stdin=fileArgs,
2707 cb=streamP4FilesCbSelf)
2709 # do the last chunk
2710 if self.stream_file.has_key('depotFile'):
2711 self.streamOneP4File(self.stream_file, self.stream_contents)
2713 def make_email(self, userid):
2714 if userid in self.users:
2715 return self.users[userid]
2716 else:
2717 return "%s <a@b>" % userid
2719 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2720 """ Stream a p4 tag.
2721 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2724 if verbose:
2725 print "writing tag %s for commit %s" % (labelName, commit)
2726 gitStream.write("tag %s\n" % labelName)
2727 gitStream.write("from %s\n" % commit)
2729 if labelDetails.has_key('Owner'):
2730 owner = labelDetails["Owner"]
2731 else:
2732 owner = None
2734 # Try to use the owner of the p4 label, or failing that,
2735 # the current p4 user id.
2736 if owner:
2737 email = self.make_email(owner)
2738 else:
2739 email = self.make_email(self.p4UserId())
2740 tagger = "%s %s %s" % (email, epoch, self.tz)
2742 gitStream.write("tagger %s\n" % tagger)
2744 print "labelDetails=",labelDetails
2745 if labelDetails.has_key('Description'):
2746 description = labelDetails['Description']
2747 else:
2748 description = 'Label from git p4'
2750 gitStream.write("data %d\n" % len(description))
2751 gitStream.write(description)
2752 gitStream.write("\n")
2754 def inClientSpec(self, path):
2755 if not self.clientSpecDirs:
2756 return True
2757 inClientSpec = self.clientSpecDirs.map_in_client(path)
2758 if not inClientSpec and self.verbose:
2759 print('Ignoring file outside of client spec: {0}'.format(path))
2760 return inClientSpec
2762 def hasBranchPrefix(self, path):
2763 if not self.branchPrefixes:
2764 return True
2765 hasPrefix = [p for p in self.branchPrefixes
2766 if p4PathStartsWith(path, p)]
2767 if not hasPrefix and self.verbose:
2768 print('Ignoring file outside of prefix: {0}'.format(path))
2769 return hasPrefix
2771 def commit(self, details, files, branch, parent = ""):
2772 epoch = details["time"]
2773 author = details["user"]
2774 jobs = self.extractJobsFromCommit(details)
2776 if self.verbose:
2777 print('commit into {0}'.format(branch))
2779 if self.clientSpecDirs:
2780 self.clientSpecDirs.update_client_spec_path_cache(files)
2782 files = [f for f in files
2783 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2785 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2786 print('Ignoring revision {0} as it would produce an empty commit.'
2787 .format(details['change']))
2788 return
2790 self.gitStream.write("commit %s\n" % branch)
2791 self.gitStream.write("mark :%s\n" % details["change"])
2792 self.committedChanges.add(int(details["change"]))
2793 committer = ""
2794 if author not in self.users:
2795 self.getUserMapFromPerforceServer()
2796 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2798 self.gitStream.write("committer %s\n" % committer)
2800 self.gitStream.write("data <<EOT\n")
2801 self.gitStream.write(details["desc"])
2802 if len(jobs) > 0:
2803 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2804 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2805 (','.join(self.branchPrefixes), details["change"]))
2806 if len(details['options']) > 0:
2807 self.gitStream.write(": options = %s" % details['options'])
2808 self.gitStream.write("]\nEOT\n\n")
2810 if len(parent) > 0:
2811 if self.verbose:
2812 print "parent %s" % parent
2813 self.gitStream.write("from %s\n" % parent)
2815 self.streamP4Files(files)
2816 self.gitStream.write("\n")
2818 change = int(details["change"])
2820 if self.labels.has_key(change):
2821 label = self.labels[change]
2822 labelDetails = label[0]
2823 labelRevisions = label[1]
2824 if self.verbose:
2825 print "Change %s is labelled %s" % (change, labelDetails)
2827 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2828 for p in self.branchPrefixes])
2830 if len(files) == len(labelRevisions):
2832 cleanedFiles = {}
2833 for info in files:
2834 if info["action"] in self.delete_actions:
2835 continue
2836 cleanedFiles[info["depotFile"]] = info["rev"]
2838 if cleanedFiles == labelRevisions:
2839 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2841 else:
2842 if not self.silent:
2843 print ("Tag %s does not match with change %s: files do not match."
2844 % (labelDetails["label"], change))
2846 else:
2847 if not self.silent:
2848 print ("Tag %s does not match with change %s: file count is different."
2849 % (labelDetails["label"], change))
2851 # Build a dictionary of changelists and labels, for "detect-labels" option.
2852 def getLabels(self):
2853 self.labels = {}
2855 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2856 if len(l) > 0 and not self.silent:
2857 print "Finding files belonging to labels in %s" % `self.depotPaths`
2859 for output in l:
2860 label = output["label"]
2861 revisions = {}
2862 newestChange = 0
2863 if self.verbose:
2864 print "Querying files for label %s" % label
2865 for file in p4CmdList(["files"] +
2866 ["%s...@%s" % (p, label)
2867 for p in self.depotPaths]):
2868 revisions[file["depotFile"]] = file["rev"]
2869 change = int(file["change"])
2870 if change > newestChange:
2871 newestChange = change
2873 self.labels[newestChange] = [output, revisions]
2875 if self.verbose:
2876 print "Label changes: %s" % self.labels.keys()
2878 # Import p4 labels as git tags. A direct mapping does not
2879 # exist, so assume that if all the files are at the same revision
2880 # then we can use that, or it's something more complicated we should
2881 # just ignore.
2882 def importP4Labels(self, stream, p4Labels):
2883 if verbose:
2884 print "import p4 labels: " + ' '.join(p4Labels)
2886 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2887 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2888 if len(validLabelRegexp) == 0:
2889 validLabelRegexp = defaultLabelRegexp
2890 m = re.compile(validLabelRegexp)
2892 for name in p4Labels:
2893 commitFound = False
2895 if not m.match(name):
2896 if verbose:
2897 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2898 continue
2900 if name in ignoredP4Labels:
2901 continue
2903 labelDetails = p4CmdList(['label', "-o", name])[0]
2905 # get the most recent changelist for each file in this label
2906 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2907 for p in self.depotPaths])
2909 if change.has_key('change'):
2910 # find the corresponding git commit; take the oldest commit
2911 changelist = int(change['change'])
2912 if changelist in self.committedChanges:
2913 gitCommit = ":%d" % changelist # use a fast-import mark
2914 commitFound = True
2915 else:
2916 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2917 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2918 if len(gitCommit) == 0:
2919 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2920 else:
2921 commitFound = True
2922 gitCommit = gitCommit.strip()
2924 if commitFound:
2925 # Convert from p4 time format
2926 try:
2927 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2928 except ValueError:
2929 print "Could not convert label time %s" % labelDetails['Update']
2930 tmwhen = 1
2932 when = int(time.mktime(tmwhen))
2933 self.streamTag(stream, name, labelDetails, gitCommit, when)
2934 if verbose:
2935 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2936 else:
2937 if verbose:
2938 print "Label %s has no changelists - possibly deleted?" % name
2940 if not commitFound:
2941 # We can't import this label; don't try again as it will get very
2942 # expensive repeatedly fetching all the files for labels that will
2943 # never be imported. If the label is moved in the future, the
2944 # ignore will need to be removed manually.
2945 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2947 def guessProjectName(self):
2948 for p in self.depotPaths:
2949 if p.endswith("/"):
2950 p = p[:-1]
2951 p = p[p.strip().rfind("/") + 1:]
2952 if not p.endswith("/"):
2953 p += "/"
2954 return p
2956 def getBranchMapping(self):
2957 lostAndFoundBranches = set()
2959 user = gitConfig("git-p4.branchUser")
2960 if len(user) > 0:
2961 command = "branches -u %s" % user
2962 else:
2963 command = "branches"
2965 for info in p4CmdList(command):
2966 details = p4Cmd(["branch", "-o", info["branch"]])
2967 viewIdx = 0
2968 while details.has_key("View%s" % viewIdx):
2969 paths = details["View%s" % viewIdx].split(" ")
2970 viewIdx = viewIdx + 1
2971 # require standard //depot/foo/... //depot/bar/... mapping
2972 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2973 continue
2974 source = paths[0]
2975 destination = paths[1]
2976 ## HACK
2977 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2978 source = source[len(self.depotPaths[0]):-4]
2979 destination = destination[len(self.depotPaths[0]):-4]
2981 if destination in self.knownBranches:
2982 if not self.silent:
2983 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2984 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2985 continue
2987 self.knownBranches[destination] = source
2989 lostAndFoundBranches.discard(destination)
2991 if source not in self.knownBranches:
2992 lostAndFoundBranches.add(source)
2994 # Perforce does not strictly require branches to be defined, so we also
2995 # check git config for a branch list.
2997 # Example of branch definition in git config file:
2998 # [git-p4]
2999 # branchList=main:branchA
3000 # branchList=main:branchB
3001 # branchList=branchA:branchC
3002 configBranches = gitConfigList("git-p4.branchList")
3003 for branch in configBranches:
3004 if branch:
3005 (source, destination) = branch.split(":")
3006 self.knownBranches[destination] = source
3008 lostAndFoundBranches.discard(destination)
3010 if source not in self.knownBranches:
3011 lostAndFoundBranches.add(source)
3014 for branch in lostAndFoundBranches:
3015 self.knownBranches[branch] = branch
3017 def getBranchMappingFromGitBranches(self):
3018 branches = p4BranchesInGit(self.importIntoRemotes)
3019 for branch in branches.keys():
3020 if branch == "master":
3021 branch = "main"
3022 else:
3023 branch = branch[len(self.projectName):]
3024 self.knownBranches[branch] = branch
3026 def updateOptionDict(self, d):
3027 option_keys = {}
3028 if self.keepRepoPath:
3029 option_keys['keepRepoPath'] = 1
3031 d["options"] = ' '.join(sorted(option_keys.keys()))
3033 def readOptions(self, d):
3034 self.keepRepoPath = (d.has_key('options')
3035 and ('keepRepoPath' in d['options']))
3037 def gitRefForBranch(self, branch):
3038 if branch == "main":
3039 return self.refPrefix + "master"
3041 if len(branch) <= 0:
3042 return branch
3044 return self.refPrefix + self.projectName + branch
3046 def gitCommitByP4Change(self, ref, change):
3047 if self.verbose:
3048 print "looking in ref " + ref + " for change %s using bisect..." % change
3050 earliestCommit = ""
3051 latestCommit = parseRevision(ref)
3053 while True:
3054 if self.verbose:
3055 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3056 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3057 if len(next) == 0:
3058 if self.verbose:
3059 print "argh"
3060 return ""
3061 log = extractLogMessageFromGitCommit(next)
3062 settings = extractSettingsGitLog(log)
3063 currentChange = int(settings['change'])
3064 if self.verbose:
3065 print "current change %s" % currentChange
3067 if currentChange == change:
3068 if self.verbose:
3069 print "found %s" % next
3070 return next
3072 if currentChange < change:
3073 earliestCommit = "^%s" % next
3074 else:
3075 latestCommit = "%s" % next
3077 return ""
3079 def importNewBranch(self, branch, maxChange):
3080 # make fast-import flush all changes to disk and update the refs using the checkpoint
3081 # command so that we can try to find the branch parent in the git history
3082 self.gitStream.write("checkpoint\n\n");
3083 self.gitStream.flush();
3084 branchPrefix = self.depotPaths[0] + branch + "/"
3085 range = "@1,%s" % maxChange
3086 #print "prefix" + branchPrefix
3087 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3088 if len(changes) <= 0:
3089 return False
3090 firstChange = changes[0]
3091 #print "first change in branch: %s" % firstChange
3092 sourceBranch = self.knownBranches[branch]
3093 sourceDepotPath = self.depotPaths[0] + sourceBranch
3094 sourceRef = self.gitRefForBranch(sourceBranch)
3095 #print "source " + sourceBranch
3097 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3098 #print "branch parent: %s" % branchParentChange
3099 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3100 if len(gitParent) > 0:
3101 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3102 #print "parent git commit: %s" % gitParent
3104 self.importChanges(changes)
3105 return True
3107 def searchParent(self, parent, branch, target):
3108 parentFound = False
3109 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3110 "--no-merges", parent]):
3111 blob = blob.strip()
3112 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3113 parentFound = True
3114 if self.verbose:
3115 print "Found parent of %s in commit %s" % (branch, blob)
3116 break
3117 if parentFound:
3118 return blob
3119 else:
3120 return None
3122 def importChanges(self, changes):
3123 cnt = 1
3124 for change in changes:
3125 description = p4_describe(change)
3126 self.updateOptionDict(description)
3128 if not self.silent:
3129 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3130 sys.stdout.flush()
3131 cnt = cnt + 1
3133 try:
3134 if self.detectBranches:
3135 branches = self.splitFilesIntoBranches(description)
3136 for branch in branches.keys():
3137 ## HACK --hwn
3138 branchPrefix = self.depotPaths[0] + branch + "/"
3139 self.branchPrefixes = [ branchPrefix ]
3141 parent = ""
3143 filesForCommit = branches[branch]
3145 if self.verbose:
3146 print "branch is %s" % branch
3148 self.updatedBranches.add(branch)
3150 if branch not in self.createdBranches:
3151 self.createdBranches.add(branch)
3152 parent = self.knownBranches[branch]
3153 if parent == branch:
3154 parent = ""
3155 else:
3156 fullBranch = self.projectName + branch
3157 if fullBranch not in self.p4BranchesInGit:
3158 if not self.silent:
3159 print("\n Importing new branch %s" % fullBranch);
3160 if self.importNewBranch(branch, change - 1):
3161 parent = ""
3162 self.p4BranchesInGit.append(fullBranch)
3163 if not self.silent:
3164 print("\n Resuming with change %s" % change);
3166 if self.verbose:
3167 print "parent determined through known branches: %s" % parent
3169 branch = self.gitRefForBranch(branch)
3170 parent = self.gitRefForBranch(parent)
3172 if self.verbose:
3173 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3175 if len(parent) == 0 and branch in self.initialParents:
3176 parent = self.initialParents[branch]
3177 del self.initialParents[branch]
3179 blob = None
3180 if len(parent) > 0:
3181 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3182 if self.verbose:
3183 print "Creating temporary branch: " + tempBranch
3184 self.commit(description, filesForCommit, tempBranch)
3185 self.tempBranches.append(tempBranch)
3186 self.checkpoint()
3187 blob = self.searchParent(parent, branch, tempBranch)
3188 if blob:
3189 self.commit(description, filesForCommit, branch, blob)
3190 else:
3191 if self.verbose:
3192 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3193 self.commit(description, filesForCommit, branch, parent)
3194 else:
3195 files = self.extractFilesFromCommit(description)
3196 self.commit(description, files, self.branch,
3197 self.initialParent)
3198 # only needed once, to connect to the previous commit
3199 self.initialParent = ""
3200 except IOError:
3201 print self.gitError.read()
3202 sys.exit(1)
3204 def importHeadRevision(self, revision):
3205 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3207 details = {}
3208 details["user"] = "git perforce import user"
3209 details["desc"] = ("Initial import of %s from the state at revision %s\n"
3210 % (' '.join(self.depotPaths), revision))
3211 details["change"] = revision
3212 newestRevision = 0
3214 fileCnt = 0
3215 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3217 for info in p4CmdList(["files"] + fileArgs):
3219 if 'code' in info and info['code'] == 'error':
3220 sys.stderr.write("p4 returned an error: %s\n"
3221 % info['data'])
3222 if info['data'].find("must refer to client") >= 0:
3223 sys.stderr.write("This particular p4 error is misleading.\n")
3224 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3225 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
3226 sys.exit(1)
3227 if 'p4ExitCode' in info:
3228 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3229 sys.exit(1)
3232 change = int(info["change"])
3233 if change > newestRevision:
3234 newestRevision = change
3236 if info["action"] in self.delete_actions:
3237 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3238 #fileCnt = fileCnt + 1
3239 continue
3241 for prop in ["depotFile", "rev", "action", "type" ]:
3242 details["%s%s" % (prop, fileCnt)] = info[prop]
3244 fileCnt = fileCnt + 1
3246 details["change"] = newestRevision
3248 # Use time from top-most change so that all git p4 clones of
3249 # the same p4 repo have the same commit SHA1s.
3250 res = p4_describe(newestRevision)
3251 details["time"] = res["time"]
3253 self.updateOptionDict(details)
3254 try:
3255 self.commit(details, self.extractFilesFromCommit(details), self.branch)
3256 except IOError:
3257 print "IO error with git fast-import. Is your git version recent enough?"
3258 print self.gitError.read()
3261 def run(self, args):
3262 self.depotPaths = []
3263 self.changeRange = ""
3264 self.previousDepotPaths = []
3265 self.hasOrigin = False
3267 # map from branch depot path to parent branch
3268 self.knownBranches = {}
3269 self.initialParents = {}
3271 if self.importIntoRemotes:
3272 self.refPrefix = "refs/remotes/p4/"
3273 else:
3274 self.refPrefix = "refs/heads/p4/"
3276 if self.syncWithOrigin:
3277 self.hasOrigin = originP4BranchesExist()
3278 if self.hasOrigin:
3279 if not self.silent:
3280 print 'Syncing with origin first, using "git fetch origin"'
3281 system("git fetch origin")
3283 branch_arg_given = bool(self.branch)
3284 if len(self.branch) == 0:
3285 self.branch = self.refPrefix + "master"
3286 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3287 system("git update-ref %s refs/heads/p4" % self.branch)
3288 system("git branch -D p4")
3290 # accept either the command-line option, or the configuration variable
3291 if self.useClientSpec:
3292 # will use this after clone to set the variable
3293 self.useClientSpec_from_options = True
3294 else:
3295 if gitConfigBool("git-p4.useclientspec"):
3296 self.useClientSpec = True
3297 if self.useClientSpec:
3298 self.clientSpecDirs = getClientSpec()
3300 # TODO: should always look at previous commits,
3301 # merge with previous imports, if possible.
3302 if args == []:
3303 if self.hasOrigin:
3304 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3306 # branches holds mapping from branch name to sha1
3307 branches = p4BranchesInGit(self.importIntoRemotes)
3309 # restrict to just this one, disabling detect-branches
3310 if branch_arg_given:
3311 short = self.branch.split("/")[-1]
3312 if short in branches:
3313 self.p4BranchesInGit = [ short ]
3314 else:
3315 self.p4BranchesInGit = branches.keys()
3317 if len(self.p4BranchesInGit) > 1:
3318 if not self.silent:
3319 print "Importing from/into multiple branches"
3320 self.detectBranches = True
3321 for branch in branches.keys():
3322 self.initialParents[self.refPrefix + branch] = \
3323 branches[branch]
3325 if self.verbose:
3326 print "branches: %s" % self.p4BranchesInGit
3328 p4Change = 0
3329 for branch in self.p4BranchesInGit:
3330 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
3332 settings = extractSettingsGitLog(logMsg)
3334 self.readOptions(settings)
3335 if (settings.has_key('depot-paths')
3336 and settings.has_key ('change')):
3337 change = int(settings['change']) + 1
3338 p4Change = max(p4Change, change)
3340 depotPaths = sorted(settings['depot-paths'])
3341 if self.previousDepotPaths == []:
3342 self.previousDepotPaths = depotPaths
3343 else:
3344 paths = []
3345 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3346 prev_list = prev.split("/")
3347 cur_list = cur.split("/")
3348 for i in range(0, min(len(cur_list), len(prev_list))):
3349 if cur_list[i] <> prev_list[i]:
3350 i = i - 1
3351 break
3353 paths.append ("/".join(cur_list[:i + 1]))
3355 self.previousDepotPaths = paths
3357 if p4Change > 0:
3358 self.depotPaths = sorted(self.previousDepotPaths)
3359 self.changeRange = "@%s,#head" % p4Change
3360 if not self.silent and not self.detectBranches:
3361 print "Performing incremental import into %s git branch" % self.branch
3363 # accept multiple ref name abbreviations:
3364 # refs/foo/bar/branch -> use it exactly
3365 # p4/branch -> prepend refs/remotes/ or refs/heads/
3366 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3367 if not self.branch.startswith("refs/"):
3368 if self.importIntoRemotes:
3369 prepend = "refs/remotes/"
3370 else:
3371 prepend = "refs/heads/"
3372 if not self.branch.startswith("p4/"):
3373 prepend += "p4/"
3374 self.branch = prepend + self.branch
3376 if len(args) == 0 and self.depotPaths:
3377 if not self.silent:
3378 print "Depot paths: %s" % ' '.join(self.depotPaths)
3379 else:
3380 if self.depotPaths and self.depotPaths != args:
3381 print ("previous import used depot path %s and now %s was specified. "
3382 "This doesn't work!" % (' '.join (self.depotPaths),
3383 ' '.join (args)))
3384 sys.exit(1)
3386 self.depotPaths = sorted(args)
3388 revision = ""
3389 self.users = {}
3391 # Make sure no revision specifiers are used when --changesfile
3392 # is specified.
3393 bad_changesfile = False
3394 if len(self.changesFile) > 0:
3395 for p in self.depotPaths:
3396 if p.find("@") >= 0 or p.find("#") >= 0:
3397 bad_changesfile = True
3398 break
3399 if bad_changesfile:
3400 die("Option --changesfile is incompatible with revision specifiers")
3402 newPaths = []
3403 for p in self.depotPaths:
3404 if p.find("@") != -1:
3405 atIdx = p.index("@")
3406 self.changeRange = p[atIdx:]
3407 if self.changeRange == "@all":
3408 self.changeRange = ""
3409 elif ',' not in self.changeRange:
3410 revision = self.changeRange
3411 self.changeRange = ""
3412 p = p[:atIdx]
3413 elif p.find("#") != -1:
3414 hashIdx = p.index("#")
3415 revision = p[hashIdx:]
3416 p = p[:hashIdx]
3417 elif self.previousDepotPaths == []:
3418 # pay attention to changesfile, if given, else import
3419 # the entire p4 tree at the head revision
3420 if len(self.changesFile) == 0:
3421 revision = "#head"
3423 p = re.sub ("\.\.\.$", "", p)
3424 if not p.endswith("/"):
3425 p += "/"
3427 newPaths.append(p)
3429 self.depotPaths = newPaths
3431 # --detect-branches may change this for each branch
3432 self.branchPrefixes = self.depotPaths
3434 self.loadUserMapFromCache()
3435 self.labels = {}
3436 if self.detectLabels:
3437 self.getLabels();
3439 if self.detectBranches:
3440 ## FIXME - what's a P4 projectName ?
3441 self.projectName = self.guessProjectName()
3443 if self.hasOrigin:
3444 self.getBranchMappingFromGitBranches()
3445 else:
3446 self.getBranchMapping()
3447 if self.verbose:
3448 print "p4-git branches: %s" % self.p4BranchesInGit
3449 print "initial parents: %s" % self.initialParents
3450 for b in self.p4BranchesInGit:
3451 if b != "master":
3453 ## FIXME
3454 b = b[len(self.projectName):]
3455 self.createdBranches.add(b)
3457 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3459 self.importProcess = subprocess.Popen(["git", "fast-import"],
3460 stdin=subprocess.PIPE,
3461 stdout=subprocess.PIPE,
3462 stderr=subprocess.PIPE);
3463 self.gitOutput = self.importProcess.stdout
3464 self.gitStream = self.importProcess.stdin
3465 self.gitError = self.importProcess.stderr
3467 if revision:
3468 self.importHeadRevision(revision)
3469 else:
3470 changes = []
3472 if len(self.changesFile) > 0:
3473 output = open(self.changesFile).readlines()
3474 changeSet = set()
3475 for line in output:
3476 changeSet.add(int(line))
3478 for change in changeSet:
3479 changes.append(change)
3481 changes.sort()
3482 else:
3483 # catch "git p4 sync" with no new branches, in a repo that
3484 # does not have any existing p4 branches
3485 if len(args) == 0:
3486 if not self.p4BranchesInGit:
3487 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3489 # The default branch is master, unless --branch is used to
3490 # specify something else. Make sure it exists, or complain
3491 # nicely about how to use --branch.
3492 if not self.detectBranches:
3493 if not branch_exists(self.branch):
3494 if branch_arg_given:
3495 die("Error: branch %s does not exist." % self.branch)
3496 else:
3497 die("Error: no branch %s; perhaps specify one with --branch." %
3498 self.branch)
3500 if self.verbose:
3501 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3502 self.changeRange)
3503 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3505 if len(self.maxChanges) > 0:
3506 changes = changes[:min(int(self.maxChanges), len(changes))]
3508 if len(changes) == 0:
3509 if not self.silent:
3510 print "No changes to import!"
3511 else:
3512 if not self.silent and not self.detectBranches:
3513 print "Import destination: %s" % self.branch
3515 self.updatedBranches = set()
3517 if not self.detectBranches:
3518 if args:
3519 # start a new branch
3520 self.initialParent = ""
3521 else:
3522 # build on a previous revision
3523 self.initialParent = parseRevision(self.branch)
3525 self.importChanges(changes)
3527 if not self.silent:
3528 print ""
3529 if len(self.updatedBranches) > 0:
3530 sys.stdout.write("Updated branches: ")
3531 for b in self.updatedBranches:
3532 sys.stdout.write("%s " % b)
3533 sys.stdout.write("\n")
3535 if gitConfigBool("git-p4.importLabels"):
3536 self.importLabels = True
3538 if self.importLabels:
3539 p4Labels = getP4Labels(self.depotPaths)
3540 gitTags = getGitTags()
3542 missingP4Labels = p4Labels - gitTags
3543 self.importP4Labels(self.gitStream, missingP4Labels)
3545 self.gitStream.close()
3546 if self.importProcess.wait() != 0:
3547 die("fast-import failed: %s" % self.gitError.read())
3548 self.gitOutput.close()
3549 self.gitError.close()
3551 # Cleanup temporary branches created during import
3552 if self.tempBranches != []:
3553 for branch in self.tempBranches:
3554 read_pipe("git update-ref -d %s" % branch)
3555 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3557 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3558 # a convenient shortcut refname "p4".
3559 if self.importIntoRemotes:
3560 head_ref = self.refPrefix + "HEAD"
3561 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3562 system(["git", "symbolic-ref", head_ref, self.branch])
3564 return True
3566 class P4Rebase(Command):
3567 def __init__(self):
3568 Command.__init__(self)
3569 self.options = [
3570 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3572 self.importLabels = False
3573 self.description = ("Fetches the latest revision from perforce and "
3574 + "rebases the current work (branch) against it")
3576 def run(self, args):
3577 sync = P4Sync()
3578 sync.importLabels = self.importLabels
3579 sync.run([])
3581 return self.rebase()
3583 def rebase(self):
3584 if os.system("git update-index --refresh") != 0:
3585 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3586 if len(read_pipe("git diff-index HEAD --")) > 0:
3587 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3589 [upstream, settings] = findUpstreamBranchPoint()
3590 if len(upstream) == 0:
3591 die("Cannot find upstream branchpoint for rebase")
3593 # the branchpoint may be p4/foo~3, so strip off the parent
3594 upstream = re.sub("~[0-9]+$", "", upstream)
3596 print "Rebasing the current branch onto %s" % upstream
3597 oldHead = read_pipe("git rev-parse HEAD").strip()
3598 system("git rebase %s" % upstream)
3599 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3600 return True
3602 class P4Clone(P4Sync):
3603 def __init__(self):
3604 P4Sync.__init__(self)
3605 self.description = "Creates a new git repository and imports from Perforce into it"
3606 self.usage = "usage: %prog [options] //depot/path[@revRange]"
3607 self.options += [
3608 optparse.make_option("--destination", dest="cloneDestination",
3609 action='store', default=None,
3610 help="where to leave result of the clone"),
3611 optparse.make_option("--bare", dest="cloneBare",
3612 action="store_true", default=False),
3614 self.cloneDestination = None
3615 self.needsGit = False
3616 self.cloneBare = False
3618 def defaultDestination(self, args):
3619 ## TODO: use common prefix of args?
3620 depotPath = args[0]
3621 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3622 depotDir = re.sub("(#[^#]*)$", "", depotDir)
3623 depotDir = re.sub(r"\.\.\.$", "", depotDir)
3624 depotDir = re.sub(r"/$", "", depotDir)
3625 return os.path.split(depotDir)[1]
3627 def run(self, args):
3628 if len(args) < 1:
3629 return False
3631 if self.keepRepoPath and not self.cloneDestination:
3632 sys.stderr.write("Must specify destination for --keep-path\n")
3633 sys.exit(1)
3635 depotPaths = args
3637 if not self.cloneDestination and len(depotPaths) > 1:
3638 self.cloneDestination = depotPaths[-1]
3639 depotPaths = depotPaths[:-1]
3641 self.cloneExclude = ["/"+p for p in self.cloneExclude]
3642 for p in depotPaths:
3643 if not p.startswith("//"):
3644 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3645 return False
3647 if not self.cloneDestination:
3648 self.cloneDestination = self.defaultDestination(args)
3650 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3652 if not os.path.exists(self.cloneDestination):
3653 os.makedirs(self.cloneDestination)
3654 chdir(self.cloneDestination)
3656 init_cmd = [ "git", "init" ]
3657 if self.cloneBare:
3658 init_cmd.append("--bare")
3659 retcode = subprocess.call(init_cmd)
3660 if retcode:
3661 raise CalledProcessError(retcode, init_cmd)
3663 if not P4Sync.run(self, depotPaths):
3664 return False
3666 # create a master branch and check out a work tree
3667 if gitBranchExists(self.branch):
3668 system([ "git", "branch", "master", self.branch ])
3669 if not self.cloneBare:
3670 system([ "git", "checkout", "-f" ])
3671 else:
3672 print 'Not checking out any branch, use ' \
3673 '"git checkout -q -b master <branch>"'
3675 # auto-set this variable if invoked with --use-client-spec
3676 if self.useClientSpec_from_options:
3677 system("git config --bool git-p4.useclientspec true")
3679 return True
3681 class P4Branches(Command):
3682 def __init__(self):
3683 Command.__init__(self)
3684 self.options = [ ]
3685 self.description = ("Shows the git branches that hold imports and their "
3686 + "corresponding perforce depot paths")
3687 self.verbose = False
3689 def run(self, args):
3690 if originP4BranchesExist():
3691 createOrUpdateBranchesFromOrigin()
3693 cmdline = "git rev-parse --symbolic "
3694 cmdline += " --remotes"
3696 for line in read_pipe_lines(cmdline):
3697 line = line.strip()
3699 if not line.startswith('p4/') or line == "p4/HEAD":
3700 continue
3701 branch = line
3703 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3704 settings = extractSettingsGitLog(log)
3706 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3707 return True
3709 class HelpFormatter(optparse.IndentedHelpFormatter):
3710 def __init__(self):
3711 optparse.IndentedHelpFormatter.__init__(self)
3713 def format_description(self, description):
3714 if description:
3715 return description + "\n"
3716 else:
3717 return ""
3719 def printUsage(commands):
3720 print "usage: %s <command> [options]" % sys.argv[0]
3721 print ""
3722 print "valid commands: %s" % ", ".join(commands)
3723 print ""
3724 print "Try %s <command> --help for command specific help." % sys.argv[0]
3725 print ""
3727 commands = {
3728 "debug" : P4Debug,
3729 "submit" : P4Submit,
3730 "commit" : P4Submit,
3731 "sync" : P4Sync,
3732 "rebase" : P4Rebase,
3733 "clone" : P4Clone,
3734 "rollback" : P4RollBack,
3735 "branches" : P4Branches
3739 def main():
3740 if len(sys.argv[1:]) == 0:
3741 printUsage(commands.keys())
3742 sys.exit(2)
3744 cmdName = sys.argv[1]
3745 try:
3746 klass = commands[cmdName]