[git] Remove the '_' prefix from all new functions
[jhbuild.git] / jhbuild / versioncontrol / git.py
blob9b264d7485985b58add4c23ead26f051ca971c67
1 # jhbuild - a build script for GNOME 1.x and 2.x
2 # Copyright (C) 2001-2006 James Henstridge
3 # Copyright (C) 2007-2008 Marc-Andre Lureau
5 # git.py: some code to handle various GIT operations
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 __all__ = []
22 __metaclass__ = type
24 import os
25 import stat
26 import urlparse
27 import subprocess
28 import re
29 import urllib
30 import sys
31 import logging
33 from jhbuild.errors import FatalError, CommandError
34 from jhbuild.utils.cmds import get_output, check_version
35 from jhbuild.versioncontrol import Repository, Branch, register_repo_type
36 import jhbuild.versioncontrol.svn
37 from jhbuild.commands.sanitycheck import inpath
38 from jhbuild.utils.sxml import sxml
40 # Make sure that the urlparse module considers git:// and git+ssh://
41 # schemes to be netloc aware and set to allow relative URIs.
42 if 'git' not in urlparse.uses_netloc:
43 urlparse.uses_netloc.append('git')
44 if 'git' not in urlparse.uses_relative:
45 urlparse.uses_relative.append('git')
46 if 'git+ssh' not in urlparse.uses_netloc:
47 urlparse.uses_netloc.append('git+ssh')
48 if 'git+ssh' not in urlparse.uses_relative:
49 urlparse.uses_relative.append('git+ssh')
50 if 'ssh' not in urlparse.uses_relative:
51 urlparse.uses_relative.append('ssh')
53 def get_git_extra_env():
54 # we run git without the JHBuild LD_LIBRARY_PATH and PATH, as it can
55 # lead to errors if it picks up jhbuilded libraries, such as nss
56 return { 'LD_LIBRARY_PATH': os.environ.get('UNMANGLED_LD_LIBRARY_PATH'),
57 'PATH': os.environ.get('UNMANGLED_PATH')}
60 class GitUnknownBranchNameError(Exception):
61 pass
64 class GitRepository(Repository):
65 """A class representing a GIT repository.
67 Note that this is just the parent directory for a bunch of git
68 branches, making it easy to switch to a mirror URI.
69 """
71 init_xml_attrs = ['href']
73 def __init__(self, config, name, href):
74 Repository.__init__(self, config, name)
75 # allow user to adjust location of branch.
76 self.href = config.repos.get(name, href)
78 branch_xml_attrs = ['module', 'subdir', 'checkoutdir', 'revision', 'tag']
80 def branch(self, name, module = None, subdir="", checkoutdir = None,
81 revision = None, tag = None):
82 if module is None:
83 module = name
85 mirror_module = None
86 if self.config.dvcs_mirror_dir:
87 mirror_module = os.path.join(self.config.dvcs_mirror_dir, module)
89 # allow remapping of branch for module
90 if name in self.config.branches:
91 try:
92 new_module, revision = self.config.branches.get(name)
93 except (ValueError, TypeError):
94 logging.warning(_('ignored bad branch redefinition for module:') + ' ' + name)
95 else:
96 if new_module:
97 module = new_module
98 if not (urlparse.urlparse(module)[0] or module[0] == '/'):
99 if self.href.endswith('/'):
100 base_href = self.href
101 else:
102 base_href = self.href + '/'
103 module = base_href + module
105 if mirror_module:
106 return GitBranch(self, mirror_module, subdir, checkoutdir,
107 revision, tag, unmirrored_module=module)
108 else:
109 return GitBranch(self, module, subdir, checkoutdir, revision, tag)
111 def to_sxml(self):
112 return [sxml.repository(type='git', name=self.name, href=self.href)]
115 class GitBranch(Branch):
116 """A class representing a GIT branch."""
118 def __init__(self, repository, module, subdir, checkoutdir=None,
119 branch=None, tag=None, unmirrored_module=None):
120 Branch.__init__(self, repository, module, checkoutdir)
121 self.subdir = subdir
122 self.branch = branch
123 self.tag = tag
124 self.unmirrored_module = unmirrored_module
126 def get_module_basename(self):
127 name = os.path.basename(self.module)
128 if name.endswith('.git'):
129 name = name[:-4]
130 return name
132 def srcdir(self):
133 path_elements = [self.checkoutroot]
134 if self.checkoutdir:
135 path_elements.append(self.checkoutdir)
136 else:
137 path_elements.append(os.path.basename(self.module))
138 if self.subdir:
139 path_elements.append(self.subdir)
140 return os.path.join(*path_elements)
141 srcdir = property(srcdir)
143 def branchname(self):
144 return self.branch
145 branchname = property(branchname)
147 def execute_git_predicate(self, predicate):
148 """A git command wrapper for the cases, where only the boolean outcome
149 is of interest.
151 try:
152 get_output(predicate, cwd=self.get_checkoutdir(),
153 extra_env=get_git_extra_env())
154 except CommandError:
155 return False
156 return True
158 def is_local_branch(self, branch):
159 return self.execute_git_predicate( ['git', 'show-ref', '--quiet',
160 '--verify', 'refs/heads/' + branch])
162 def is_inside_work_tree(self):
163 return self.execute_git_predicate(
164 ['git', 'rev-parse', '--is-inside-work-tree'])
166 def is_tracking_a_remote_branch(self, local_branch):
167 if not local_branch:
168 return False
169 current_branch_remote_config = 'branch.%s.remote' % local_branch
170 return self.execute_git_predicate(
171 ['git', 'config', '--get', current_branch_remote_config])
173 def is_dirty(self, ignore_submodules=True):
174 submodule_options = []
175 if ignore_submodules:
176 if not self.check_version_git('1.5.6'):
177 raise CommandError(_('Need at least git-1.5.6 from June/08 '
178 'to operate'))
179 submodule_options = ['--ignore-submodules']
180 return not self.execute_git_predicate(
181 ['git', 'diff', '--exit-code', '--quiet'] + submodule_options
182 + ['HEAD'])
184 def check_version_git(self, version_spec):
185 return check_version(['git', '--version'], r'git version ([\d.]+)',
186 version_spec, extra_env=get_git_extra_env())
188 def get_current_branch(self):
189 """Returns either a branchname or None if head is detached"""
190 if not self.is_inside_work_tree():
191 raise CommandError(_('Unexpected: Checkoutdir is not a git '
192 'repository:' + self.get_checkoutdir()))
193 try:
194 return os.path.basename(
195 get_output(['git', 'symbolic-ref', '-q', 'HEAD'],
196 cwd=self.get_checkoutdir(),
197 extra_env=get_git_extra_env()).strip())
198 except CommandError:
199 return None
201 def find_remote_branch_online_if_necessary(self, buildscript,
202 remote_name, branch_name):
203 """Try to find the given branch first, locally, then remotely, and state
204 the availability in the return value."""
205 wanted_ref = remote_name + '/' + branch_name
206 if self.execute_git_predicate( ['git', 'show-ref', wanted_ref]):
207 return True
208 buildscript.execute(['git', 'fetch'], cwd=self.get_checkoutdir(),
209 extra_env=get_git_extra_env())
210 return self.execute_git_predicate( ['git', 'show-ref', wanted_ref])
212 def get_branch_switch_destination(self):
213 current_branch = self.get_current_branch()
214 wanted_branch = self.branch or 'master'
216 # Always switch away from a detached head.
217 if not current_branch:
218 return wanted_branch
220 assert(current_branch and wanted_branch)
221 # If the current branch is not tracking a remote branch it is assumed to
222 # be a local work branch, and it won't be considered for a change.
223 if current_branch != wanted_branch \
224 and self.is_tracking_a_remote_branch(current_branch):
225 return wanted_branch
227 return None
229 def switch_branch_if_necessary(self, buildscript):
231 The switch depends on the requested tag, the requested branch, and the
232 state and type of the current branch.
234 An imminent branch switch generates an error if there are uncommited
235 changes.
237 wanted_branch = self.get_branch_switch_destination()
238 switch_command = []
239 if self.tag:
240 switch_command= ['git', 'checkout', self.tag]
241 elif wanted_branch:
242 if self.is_local_branch(wanted_branch):
243 switch_command = ['git', 'checkout', wanted_branch]
244 else:
245 if not self.find_remote_branch_online_if_necessary(
246 buildscript, 'origin', wanted_branch):
247 raise CommandError(_('The requested branch "%s" is '
248 'not available. Neither locally, nor remotely '
249 'in the origin remote.' % wanted_branch))
250 switch_command = ['git', 'checkout', '--track', '-b',
251 wanted_branch, 'origin/' + wanted_branch]
253 if switch_command:
254 if self.is_dirty():
255 raise CommandError(_('Refusing to switch a dirty tree.'))
256 buildscript.execute(switch_command, cwd=self.get_checkoutdir(),
257 extra_env=get_git_extra_env())
259 def pull_current_branch(self, buildscript):
260 """Pull the current branch if it is tracking a remote branch."""
261 if not self.is_tracking_a_remote_branch(self.get_current_branch()):
262 return
264 git_extra_args = {'cwd': self.get_checkoutdir(),
265 'extra_env': get_git_extra_env()}
267 stashed = False
268 if self.is_dirty(ignore_submodules=True):
269 stashed = True
270 buildscript.execute(['git', 'stash', 'save', 'jhbuild-stash'],
271 **git_extra_args)
273 buildscript.execute(['git', 'pull', '--rebase'], **git_extra_args)
275 if stashed:
276 # git stash pop was introduced in 1.5.5,
277 if self.check_version_git('1.5.5'):
278 buildscript.execute(['git', 'stash', 'pop'], **git_extra_args)
279 else:
280 buildscript.execute(['git', 'stash', 'apply', 'jhbuild-stash'],
281 **git_extra_args)
283 def rewind_to_sticky_date(self, buildscript):
284 if self.config.quiet_mode:
285 quiet = ['-q']
286 else:
287 quiet = []
288 commit = self._get_commit_from_date()
289 branch = 'jhbuild-date-branch'
290 branch_cmd = ['git', 'checkout'] + quiet + [branch]
291 git_extra_args = {'cwd': self.get_checkoutdir(),
292 'extra_env': get_git_extra_env()}
293 try:
294 buildscript.execute(branch_cmd, **git_extra_args)
295 except CommandError:
296 branch_cmd = ['git', 'checkout'] + quiet + ['-b', branch]
297 buildscript.execute(branch_cmd, **git_extra_args)
298 buildscript.execute(['git', 'reset', '--hard', commit], **git_extra_args)
300 def get_remote_branches_list(self):
301 return [x.strip() for x in get_output(['git', 'branch', '-r'],
302 cwd=self.get_checkoutdir(),
303 extra_env=get_git_extra_env()).splitlines()]
305 def exists(self):
306 try:
307 refs = get_output(['git', 'ls-remote', self.module],
308 extra_env=get_git_extra_env())
309 except:
310 return False
312 #FIXME: Parse output from ls-remote to work out if tag/branch is present
314 return True
316 def _get_commit_from_date(self):
317 cmd = ['git', 'log', '--max-count=1',
318 '--until=%s' % self.config.sticky_date]
319 cmd_desc = ' '.join(cmd)
320 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
321 cwd=self.get_checkoutdir(),
322 env=get_git_extra_env())
323 stdout = proc.communicate()[0]
324 if not stdout.strip():
325 raise CommandError(_('Command %s returned no output') % cmd_desc)
326 for line in stdout.splitlines():
327 if line.startswith('commit '):
328 commit = line.split(None, 1)[1].strip()
329 return commit
330 raise CommandError(_('Command %s did not include commit line: %r')
331 % (cmd_desc, stdout))
333 def _export(self, buildscript):
334 # FIXME: should implement this properly
335 self._checkout(buildscript)
337 def _update_submodules(self, buildscript):
338 if os.path.exists(os.path.join(self.get_checkoutdir(), '.gitmodules')):
339 cmd = ['git', 'submodule', 'init']
340 buildscript.execute(cmd, cwd=self.get_checkoutdir(),
341 extra_env=get_git_extra_env())
342 cmd = ['git', 'submodule', 'update']
343 buildscript.execute(cmd, cwd=self.get_checkoutdir(),
344 extra_env=get_git_extra_env())
346 def update_dvcs_mirror(self, buildscript):
347 if not self.config.dvcs_mirror_dir:
348 return
349 if self.config.nonetwork:
350 return
352 mirror_dir = os.path.join(self.config.dvcs_mirror_dir,
353 self.get_module_basename() + '.git')
355 if os.path.exists(mirror_dir):
356 buildscript.execute(['git', 'fetch'], cwd=mirror_dir,
357 extra_env=get_git_extra_env())
358 else:
359 buildscript.execute(
360 ['git', 'clone', '--mirror', self.unmirrored_module],
361 cwd=self.config.dvcs_mirror_dir,
362 extra_env=get_git_extra_env())
364 def _checkout(self, buildscript, copydir=None):
366 if self.config.quiet_mode:
367 quiet = ['-q']
368 else:
369 quiet = []
371 self.update_dvcs_mirror(buildscript)
373 cmd = ['git', 'clone'] + quiet + [self.module]
374 if self.checkoutdir:
375 cmd.append(self.checkoutdir)
377 if copydir:
378 buildscript.execute(cmd, cwd=copydir, extra_env=get_git_extra_env())
379 else:
380 buildscript.execute(cmd, cwd=self.config.checkoutroot,
381 extra_env=get_git_extra_env())
383 self._update(buildscript, copydir=copydir, update_mirror=False)
386 def _update(self, buildscript, copydir=None, update_mirror=True):
387 cwd = self.get_checkoutdir()
388 git_extra_args = {'cwd': cwd, 'extra_env': get_git_extra_env()}
390 if not os.path.exists(os.path.join(cwd, '.git')):
391 if os.path.exists(os.path.join(cwd, '.svn')):
392 raise CommandError(_('Failed to update module as it switched to git (you should check for changes then remove the directory).'))
393 raise CommandError(_('Failed to update module (missing .git) (you should check for changes then remove the directory).'))
395 if update_mirror:
396 self.update_dvcs_mirror(buildscript)
398 self.switch_branch_if_necessary(buildscript)
400 self.pull_current_branch(buildscript)
402 if self.config.sticky_date:
403 self.rewind_to_sticky_date(buildscript)
405 self._update_submodules(buildscript)
407 def may_checkout(self, buildscript):
408 if buildscript.config.nonetwork and not buildscript.config.dvcs_mirror_dir:
409 return False
410 return True
412 def checkout(self, buildscript):
413 if not inpath('git', os.environ['PATH'].split(os.pathsep)):
414 raise CommandError(_('%s not found') % 'git')
415 Branch.checkout(self, buildscript)
417 def tree_id(self):
418 if not os.path.exists(self.get_checkoutdir()):
419 return None
420 try:
421 output = get_output(['git', 'rev-parse', 'HEAD'],
422 cwd = self.get_checkoutdir(), get_stderr=False,
423 extra_env=get_git_extra_env())
424 except CommandError:
425 return None
426 except GitUnknownBranchNameError:
427 return None
428 return output.strip()
430 def to_sxml(self):
431 attrs = {}
432 if self.branch:
433 attrs['branch'] = self.branch
434 return [sxml.branch(repo=self.repository.name,
435 module=self.module,
436 tag=self.tree_id(),
437 **attrs)]
440 class GitSvnBranch(GitBranch):
441 def __init__(self, repository, module, checkoutdir, revision=None):
442 GitBranch.__init__(self, repository, module, "", checkoutdir, branch="git-svn")
443 self.revision = revision
445 def may_checkout(self, buildscript):
446 return Branch.may_checkout(self, buildscript)
448 def _get_externals(self, buildscript, branch="git-svn"):
449 cwd = self.get_checkoutdir()
450 try:
451 externals = {}
452 match = None
453 external_expr = re.compile(r"\+dir_prop: (.*?) svn:externals (.*)$")
454 rev_expr = re.compile(r"^r(\d+)$")
455 # the unhandled.log file has the revision numbers
456 # encoded as r#num we should only parse as far as self.revision
457 for line in open(os.path.join(cwd, '.git', 'svn', branch, 'unhandled.log')):
458 m = external_expr.search(line)
459 if m:
460 match = m
461 rev_match = rev_expr.search(line)
462 if self.revision and rev_match:
463 if rev_match.group(1) > self.revision:
464 break
465 except IOError:
466 # we couldn't find an unhandled.log to parse so try
467 # git svn show-externals - note this is broken in git < 1.5.6
468 try:
469 output = get_output(['git', 'svn', 'show-externals'], cwd=cwd,
470 extra_env=get_git_extra_env())
471 # we search for comment lines to strip them out
472 comment_line = re.compile(r"^#.*")
473 ext = ''
474 for line in output.splitlines():
475 if not comment_line.search(line):
476 ext += ' ' + line
478 match = re.compile("^(\.) (.+)").search(". " + ext)
479 except OSError:
480 raise FatalError(_("External handling failed\n If you are running git version < 1.5.6 it is recommended you update.\n"))
482 # only parse the final match
483 if match:
484 branch = match.group(1)
485 external = urllib.unquote(match.group(2).replace("%0A", " ").strip("%20 ")).split()
486 revision_expr = re.compile(r"-r(\d*)")
487 i = 0
488 while i < len(external):
489 # see if we have a revision number
490 match = revision_expr.search(external[i+1])
491 if match:
492 externals[external[i]] = (external[i+2], match.group(1))
493 i = i+3
494 else:
495 externals[external[i]] = (external[i+1], None)
496 i = i+2
498 for extdir in externals.iterkeys():
499 uri = externals[extdir][0]
500 revision = externals[extdir][1]
501 extdir = cwd+os.sep+extdir
502 # FIXME: the "right way" is to use submodules
503 extbranch = GitSvnBranch(self.repository, uri, extdir, revision)
505 try:
506 os.stat(extdir)[stat.ST_MODE]
507 extbranch._update(buildscript)
508 except OSError:
509 extbranch._checkout(buildscript)
511 def _checkout(self, buildscript, copydir=None):
512 if self.config.sticky_date:
513 raise FatalError(_('date based checkout not yet supported\n'))
515 cmd = ['git', 'svn', 'clone', self.module]
516 if self.checkoutdir:
517 cmd.append(self.checkoutdir)
519 # FIXME (add self.revision support)
520 try:
521 last_revision = jhbuild.versioncontrol.svn.get_info (self.module)['last changed rev']
522 if not self.revision:
523 cmd.extend(['-r', last_revision])
524 except KeyError:
525 raise FatalError(_('Cannot get last revision from %s. Check the module location.') % self.module)
527 if copydir:
528 buildscript.execute(cmd, cwd=copydir,
529 extra_env=get_git_extra_env())
530 else:
531 buildscript.execute(cmd, cwd=self.config.checkoutroot,
532 extra_env=get_git_extra_env())
534 try:
535 # is known to fail on some versions
536 cmd = ['git', 'svn', 'show-ignore']
537 s = get_output(cmd, cwd = self.get_checkoutdir(copydir),
538 extra_env=get_git_extra_env())
539 fd = file(os.path.join(
540 self.get_checkoutdir(copydir), '.git/info/exclude'), 'a')
541 fd.write(s)
542 fc.close()
543 buildscript.execute(cmd, cwd=self.get_checkoutdir(copydir),
544 extra_env=get_git_extra_env())
545 except:
546 pass
548 # FIXME, git-svn should support externals
549 self._get_externals(buildscript, self.branch)
551 def _update(self, buildscript, copydir=None):
552 if self.config.sticky_date:
553 raise FatalError(_('date based checkout not yet supported\n'))
555 if self.config.quiet_mode:
556 quiet = ['-q']
557 else:
558 quiet = []
560 cwd = self.get_checkoutdir()
561 git_extra_args = {'cwd': cwd, 'extra_env': get_git_extra_env()}
563 last_revision = get_output(['git', 'svn', 'find-rev', 'HEAD'],
564 **git_extra_args)
566 stashed = False
567 if get_output(['git', 'diff'], **git_extra_args):
568 # stash uncommitted changes on the current branch
569 stashed = True
570 buildscript.execute(['git', 'stash', 'save', 'jhbuild-stash'],
571 **git_extra_args)
573 buildscript.execute(['git', 'checkout'] + quiet + ['master'],
574 **git_extra_args)
575 buildscript.execute(['git', 'svn', 'rebase'], **git_extra_args)
577 if stashed:
578 buildscript.execute(['git', 'stash', 'pop'], **git_extra_args)
580 current_revision = get_output(['git', 'svn', 'find-rev', 'HEAD'],
581 **git_extra_args)
583 if last_revision != current_revision:
584 try:
585 # is known to fail on some versions
586 cmd = "git svn show-ignore >> .git/info/exclude"
587 buildscript.execute(cmd, **git_extra_args)
588 except:
589 pass
591 # FIXME, git-svn should support externals
592 self._get_externals(buildscript, self.branch)
594 class GitCvsBranch(GitBranch):
595 def __init__(self, repository, module, checkoutdir, revision=None):
596 GitBranch.__init__(self, repository, module, "", checkoutdir)
597 self.revision = revision
599 def may_checkout(self, buildscript):
600 return Branch.may_checkout(self, buildscript)
602 def branchname(self):
603 for b in ['remotes/' + str(self.branch), self.branch, 'trunk', 'master']:
604 if self.branch_exist(b):
605 return b
606 raise
607 branchname = property(branchname)
609 def _checkout(self, buildscript, copydir=None):
611 cmd = ['git', 'cvsimport', '-r', 'cvs', '-p', 'b,HEAD',
612 '-k', '-m', '-a', '-v', '-d', self.repository.cvsroot, '-C']
614 if self.checkoutdir:
615 cmd.append(self.checkoutdir)
616 else:
617 cmd.append(self.module)
619 cmd.append(self.module)
621 if copydir:
622 buildscript.execute(cmd, cwd=copydir, extra_env=get_git_extra_env())
623 else:
624 buildscript.execute(cmd, cwd=self.config.checkoutroot,
625 extra_env=get_git_extra_env())
627 def _update(self, buildscript, copydir=None):
628 if self.config.sticky_date:
629 raise FatalError(_('date based checkout not yet supported\n'))
631 cwd = self.get_checkoutdir()
632 git_extra_args = {'cwd': cwd, 'extra_env': get_git_extra_env()}
634 stashed = False
635 # stash uncommitted changes on the current branch
636 if get_output(['git', 'diff'], **git_extra_args):
637 # stash uncommitted changes on the current branch
638 stashed = True
639 buildscript.execute(['git', 'stash', 'save', 'jhbuild-stash'],
640 **git_extra_args)
642 self._checkout(buildscript, copydir=copydir)
644 if stashed:
645 buildscript.execute(['git', 'stash', 'pop'], **git_extra_args)
647 register_repo_type('git', GitRepository)