1 # jhbuild - a tool to ease building collections of source packages
2 # Copyright (C) 2001-2006 James Henstridge
3 # Copyright (C) 2007-2008 Marc-Andre Lureau
5 # git.py: some code to handle various GIT operations
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with this program; if not, write to the Free Software
19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
33 from jhbuild
.errors
import FatalError
, CommandError
34 from jhbuild
.utils
.cmds
import get_output
, check_version
35 from jhbuild
.versioncontrol
import Repository
, Branch
, register_repo_type
36 import jhbuild
.versioncontrol
.svn
37 from jhbuild
.commands
.sanitycheck
import inpath
38 from jhbuild
.utils
.sxml
import sxml
40 # Make sure that the urlparse module considers git:// and git+ssh://
41 # schemes to be netloc aware and set to allow relative URIs.
42 if 'git' not in urlparse
.uses_netloc
:
43 urlparse
.uses_netloc
.append('git')
44 if 'git' not in urlparse
.uses_relative
:
45 urlparse
.uses_relative
.append('git')
46 if 'git+ssh' not in urlparse
.uses_netloc
:
47 urlparse
.uses_netloc
.append('git+ssh')
48 if 'git+ssh' not in urlparse
.uses_relative
:
49 urlparse
.uses_relative
.append('git+ssh')
50 if 'ssh' not in urlparse
.uses_relative
:
51 urlparse
.uses_relative
.append('ssh')
53 def get_git_extra_env():
54 # we run git without the JHBuild LD_LIBRARY_PATH and PATH, as it can
55 # lead to errors if it picks up jhbuilded libraries, such as nss
56 return { 'LD_LIBRARY_PATH': os
.environ
.get('UNMANGLED_LD_LIBRARY_PATH'),
57 'PATH': os
.environ
.get('UNMANGLED_PATH')}
59 def get_git_mirror_directory(mirror_root
, checkoutdir
, module
):
60 """Calculate the mirror directory from the arguments and return it."""
61 mirror_dir
= os
.path
.join(mirror_root
, checkoutdir
or
62 os
.path
.basename(module
))
63 if mirror_dir
.endswith('.git'):
66 return mirror_dir
+ '.git'
68 class GitUnknownBranchNameError(Exception):
72 class GitRepository(Repository
):
73 """A class representing a GIT repository.
75 Note that this is just the parent directory for a bunch of git
76 branches, making it easy to switch to a mirror URI.
79 init_xml_attrs
= ['href']
81 def __init__(self
, config
, name
, href
):
82 Repository
.__init
__(self
, config
, name
)
83 # allow user to adjust location of branch.
84 self
.href
= config
.repos
.get(name
, href
)
86 branch_xml_attrs
= ['module', 'subdir', 'checkoutdir', 'revision', 'tag']
88 def branch(self
, name
, module
= None, subdir
="", checkoutdir
= None,
89 revision
= None, tag
= None):
94 if self
.config
.dvcs_mirror_dir
:
95 mirror_module
= get_git_mirror_directory(
96 self
.config
.dvcs_mirror_dir
, checkoutdir
, module
)
98 # allow remapping of branch for module, it supports two modes of
100 if name
in self
.config
.branches
:
101 branch_mapping
= self
.config
.branches
.get(name
)
102 if type(branch_mapping
) is str:
103 # passing a single string will override the branch name
104 revision
= branch_mapping
106 # otherwise it is assumed it is a pair, redefining both
107 # git URI and the branch to use
109 new_module
, revision
= self
.config
.branches
.get(name
)
110 except (ValueError, TypeError):
111 logging
.warning(_('ignored bad branch redefinition for module:') + ' ' + name
)
115 if not (urlparse
.urlparse(module
)[0] or module
[0] == '/'):
116 if self
.href
.endswith('/'):
117 base_href
= self
.href
119 base_href
= self
.href
+ '/'
120 module
= base_href
+ module
123 return GitBranch(self
, mirror_module
, subdir
, checkoutdir
,
124 revision
, tag
, unmirrored_module
=module
)
126 return GitBranch(self
, module
, subdir
, checkoutdir
, revision
, tag
)
129 return [sxml
.repository(type='git', name
=self
.name
, href
=self
.href
)]
131 def get_sysdeps(self
):
135 class GitBranch(Branch
):
136 """A class representing a GIT branch."""
138 dirty_branch_suffix
= '-dirty'
140 def __init__(self
, repository
, module
, subdir
, checkoutdir
=None,
141 branch
=None, tag
=None, unmirrored_module
=None):
142 Branch
.__init
__(self
, repository
, module
, checkoutdir
)
146 self
.unmirrored_module
= unmirrored_module
148 def get_module_basename(self
):
149 # prevent basename() from returning empty strings on trailing '/'
150 name
= self
.module
.rstrip(os
.sep
)
151 name
= os
.path
.basename(name
)
152 if name
.endswith('.git'):
157 path_elements
= [self
.checkoutroot
]
159 path_elements
.append(self
.checkoutdir
)
161 path_elements
.append(self
.get_module_basename())
163 path_elements
.append(self
.subdir
)
164 return os
.path
.join(*path_elements
)
165 srcdir
= property(srcdir
)
167 def branchname(self
):
169 branchname
= property(branchname
)
171 def execute_git_predicate(self
, predicate
):
172 """A git command wrapper for the cases, where only the boolean outcome
176 get_output(predicate
, cwd
=self
.get_checkoutdir(),
177 extra_env
=get_git_extra_env())
182 def is_local_branch(self
, branch
):
183 is_local_head
= self
.execute_git_predicate( ['git', 'show-ref', '--quiet',
184 '--verify', 'refs/heads/' + branch
])
187 return self
.execute_git_predicate(['git', 'rev-parse', branch
])
189 def is_inside_work_tree(self
):
190 return self
.execute_git_predicate(
191 ['git', 'rev-parse', '--is-inside-work-tree'])
193 def is_tracking_a_remote_branch(self
, local_branch
):
196 current_branch_remote_config
= 'branch.%s.remote' % local_branch
197 return self
.execute_git_predicate(
198 ['git', 'config', '--get', current_branch_remote_config
])
200 def is_dirty(self
, ignore_submodules
=True):
201 submodule_options
= []
202 if ignore_submodules
:
203 if not self
.check_version_git('1.5.6'):
204 raise CommandError(_('Need at least git-1.5.6 from June/08 '
206 submodule_options
= ['--ignore-submodules']
207 return not self
.execute_git_predicate(
208 ['git', 'diff', '--exit-code', '--quiet'] + submodule_options
211 def check_version_git(self
, version_spec
):
212 return check_version(['git', '--version'], r
'git version ([\d.]+)',
213 version_spec
, extra_env
=get_git_extra_env())
215 def get_current_branch(self
):
216 """Returns either a branchname or None if head is detached"""
217 if not self
.is_inside_work_tree():
218 raise CommandError(_('Unexpected: Checkoutdir is not a git '
219 'repository:' + self
.get_checkoutdir()))
221 full_branch
= get_output(['git', 'symbolic-ref', '-q', 'HEAD'],
222 cwd
=self
.get_checkoutdir(),
223 extra_env
=get_git_extra_env()).strip()
224 # strip refs/heads/ to get the branch name only
225 return full_branch
.replace('refs/heads/', '')
229 def find_remote_branch_online_if_necessary(self
, buildscript
,
230 remote_name
, branch_name
):
231 """Try to find the given branch first, locally, then remotely, and state
232 the availability in the return value."""
233 wanted_ref
= remote_name
+ '/' + branch_name
234 if self
.execute_git_predicate( ['git', 'show-ref', wanted_ref
]):
236 buildscript
.execute(['git', 'fetch'], cwd
=self
.get_checkoutdir(),
237 extra_env
=get_git_extra_env())
238 return self
.execute_git_predicate( ['git', 'show-ref', wanted_ref
])
240 def get_branch_switch_destination(self
):
241 current_branch
= self
.get_current_branch()
242 wanted_branch
= self
.branch
or 'master'
244 # Always switch away from a detached head.
245 if not current_branch
:
248 assert(current_branch
and wanted_branch
)
249 # If the current branch is not tracking a remote branch it is assumed to
250 # be a local work branch, and it won't be considered for a change.
251 if current_branch
!= wanted_branch \
252 and self
.is_tracking_a_remote_branch(current_branch
):
257 def switch_branch_if_necessary(self
, buildscript
):
259 The switch depends on the requested tag, the requested branch, and the
260 state and type of the current branch.
262 An imminent branch switch generates an error if there are uncommited
265 wanted_branch
= self
.get_branch_switch_destination()
268 switch_command
= ['git', 'checkout', self
.tag
]
270 if self
.is_local_branch(wanted_branch
):
271 switch_command
= ['git', 'checkout', wanted_branch
]
273 if not self
.find_remote_branch_online_if_necessary(
274 buildscript
, 'origin', wanted_branch
):
275 raise CommandError(_('The requested branch "%s" is '
276 'not available. Neither locally, nor remotely '
277 'in the origin remote.' % wanted_branch
))
278 switch_command
= ['git', 'checkout', '--track', '-b',
279 wanted_branch
, 'origin/' + wanted_branch
]
283 raise CommandError(_('Refusing to switch a dirty tree.'))
284 buildscript
.execute(switch_command
, cwd
=self
.get_checkoutdir(),
285 extra_env
=get_git_extra_env())
287 def rebase_current_branch(self
, buildscript
):
288 """Pull the current branch if it is tracking a remote branch."""
289 branch
= self
.get_current_branch();
290 if not self
.is_tracking_a_remote_branch(branch
):
293 git_extra_args
= {'cwd': self
.get_checkoutdir(),
294 'extra_env': get_git_extra_env()}
297 if self
.is_dirty(ignore_submodules
=True):
299 buildscript
.execute(['git', 'stash', 'save', 'jhbuild-stash'],
302 buildscript
.execute(['git', 'rebase', 'origin/' + branch
],
306 # git stash pop was introduced in 1.5.5,
307 if self
.check_version_git('1.5.5'):
308 buildscript
.execute(['git', 'stash', 'pop'], **git_extra_args
)
310 buildscript
.execute(['git', 'stash', 'apply', 'jhbuild-stash'],
313 def move_to_sticky_date(self
, buildscript
):
314 if self
.config
.quiet_mode
:
318 commit
= self
._get
_commit
_from
_date
()
319 branch
= 'jhbuild-date-branch'
320 branch_cmd
= ['git', 'checkout'] + quiet
+ [branch
]
321 git_extra_args
= {'cwd': self
.get_checkoutdir(),
322 'extra_env': get_git_extra_env()}
323 if self
.config
.sticky_date
== 'none':
324 current_branch
= self
.get_current_branch()
325 if current_branch
and current_branch
== branch
:
326 buildscript
.execute(['git', 'checkout'] + quiet
+ ['master'],
330 buildscript
.execute(branch_cmd
, **git_extra_args
)
332 branch_cmd
= ['git', 'checkout'] + quiet
+ ['-b', branch
]
333 buildscript
.execute(branch_cmd
, **git_extra_args
)
334 buildscript
.execute(['git', 'reset', '--hard', commit
], **git_extra_args
)
336 def get_remote_branches_list(self
):
337 return [x
.strip() for x
in get_output(['git', 'branch', '-r'],
338 cwd
=self
.get_checkoutdir(),
339 extra_env
=get_git_extra_env()).splitlines()]
343 refs
= get_output(['git', 'ls-remote', self
.module
],
344 extra_env
=get_git_extra_env())
348 #FIXME: Parse output from ls-remote to work out if tag/branch is present
352 def _get_commit_from_date(self
):
353 cmd
= ['git', 'log', '--max-count=1', '--first-parent',
354 '--until=%s' % self
.config
.sticky_date
, 'master']
355 cmd_desc
= ' '.join(cmd
)
356 proc
= subprocess
.Popen(cmd
, stdout
=subprocess
.PIPE
,
357 cwd
=self
.get_checkoutdir(),
358 env
=get_git_extra_env())
359 stdout
= proc
.communicate()[0]
360 if not stdout
.strip():
361 raise CommandError(_('Command %s returned no output') % cmd_desc
)
362 for line
in stdout
.splitlines():
363 if line
.startswith('commit '):
364 commit
= line
.split(None, 1)[1].strip()
366 raise CommandError(_('Command %s did not include commit line: %r')
367 % (cmd_desc
, stdout
))
369 def _export(self
, buildscript
):
370 # FIXME: should implement this properly
371 self
._checkout
(buildscript
)
373 def _update_submodules(self
, buildscript
):
374 if os
.path
.exists(os
.path
.join(self
.get_checkoutdir(), '.gitmodules')):
375 cmd
= ['git', 'submodule', 'init']
376 buildscript
.execute(cmd
, cwd
=self
.get_checkoutdir(),
377 extra_env
=get_git_extra_env())
378 cmd
= ['git', 'submodule', 'update']
379 buildscript
.execute(cmd
, cwd
=self
.get_checkoutdir(),
380 extra_env
=get_git_extra_env())
382 def update_dvcs_mirror(self
, buildscript
):
383 if not self
.config
.dvcs_mirror_dir
:
385 if self
.config
.nonetwork
:
388 # Calculate a new in case a configuration reload changed the mirror root.
389 mirror_dir
= get_git_mirror_directory(self
.config
.dvcs_mirror_dir
,
390 self
.checkoutdir
, self
.unmirrored_module
)
392 if os
.path
.exists(mirror_dir
):
393 buildscript
.execute(['git', 'remote', 'set-url', 'origin',
394 self
.unmirrored_module
], cwd
=mirror_dir
,
395 extra_env
=get_git_extra_env())
396 buildscript
.execute(['git', 'fetch'], cwd
=mirror_dir
,
397 extra_env
=get_git_extra_env())
400 ['git', 'clone', '--mirror', self
.unmirrored_module
,
401 mirror_dir
], extra_env
=get_git_extra_env())
403 def _checkout(self
, buildscript
, copydir
=None):
406 if self
.config
.quiet_mode
:
407 extra_opts
.append('-q')
409 if self
.config
.shallow_clone
:
410 extra_opts
.append('--depth=1')
412 self
.update_dvcs_mirror(buildscript
)
414 cmd
= ['git', 'clone'] + extra_opts
+ [self
.module
]
416 cmd
.append(self
.checkoutdir
)
418 if self
.branch
is not None:
419 cmd
.extend(['-b', self
.branch
])
422 buildscript
.execute(cmd
, cwd
=copydir
, extra_env
=get_git_extra_env())
424 buildscript
.execute(cmd
, cwd
=self
.config
.checkoutroot
,
425 extra_env
=get_git_extra_env())
427 self
._update
(buildscript
, copydir
=copydir
, update_mirror
=False)
430 def _update(self
, buildscript
, copydir
=None, update_mirror
=True):
431 cwd
= self
.get_checkoutdir()
432 git_extra_args
= {'cwd': cwd
, 'extra_env': get_git_extra_env()}
434 if not os
.path
.exists(os
.path
.join(cwd
, '.git')):
435 if os
.path
.exists(os
.path
.join(cwd
, '.svn')):
436 raise CommandError(_('Failed to update module as it switched to git (you should check for changes then remove the directory).'))
437 raise CommandError(_('Failed to update module (missing .git) (you should check for changes then remove the directory).'))
440 self
.update_dvcs_mirror(buildscript
)
442 buildscript
.execute(['git', 'remote', 'set-url', 'origin',
443 self
.module
], **git_extra_args
)
445 buildscript
.execute(['git', 'remote', 'update', 'origin'],
448 if self
.config
.sticky_date
:
449 self
.move_to_sticky_date(buildscript
)
451 self
.switch_branch_if_necessary(buildscript
)
453 self
.rebase_current_branch(buildscript
)
455 self
._update
_submodules
(buildscript
)
457 def may_checkout(self
, buildscript
):
458 if buildscript
.config
.nonetwork
and not buildscript
.config
.dvcs_mirror_dir
:
462 def checkout(self
, buildscript
):
463 if not inpath('git', os
.environ
['PATH'].split(os
.pathsep
)):
464 raise CommandError(_('%s not found') % 'git')
465 Branch
.checkout(self
, buildscript
)
467 def delete_unknown_files(self
, buildscript
):
468 git_extra_args
= {'cwd': self
.get_checkoutdir(), 'extra_env': get_git_extra_env()}
469 buildscript
.execute(['git', 'clean', '-d', '-f', '-x'], **git_extra_args
)
472 if not os
.path
.exists(self
.get_checkoutdir()):
475 output
= get_output(['git', 'rev-parse', 'HEAD'],
476 cwd
= self
.get_checkoutdir(), get_stderr
=False,
477 extra_env
=get_git_extra_env())
480 except GitUnknownBranchNameError
:
484 id_suffix
= self
.dirty_branch_suffix
485 return output
.strip() + id_suffix
490 attrs
['revision'] = self
.branch
492 attrs
['checkoutdir'] = self
.checkoutdir
494 attrs
['subdir'] = self
.subdir
495 return [sxml
.branch(repo
=self
.repository
.name
,
501 class GitSvnBranch(GitBranch
):
502 def __init__(self
, repository
, module
, checkoutdir
, revision
=None):
503 GitBranch
.__init
__(self
, repository
, module
, "", checkoutdir
, branch
="git-svn")
504 self
.revision
= revision
506 def may_checkout(self
, buildscript
):
507 return Branch
.may_checkout(self
, buildscript
)
509 def _get_externals(self
, buildscript
, branch
="git-svn"):
510 cwd
= self
.get_checkoutdir()
514 external_expr
= re
.compile(r
"\+dir_prop: (.*?) svn:externals (.*)$")
515 rev_expr
= re
.compile(r
"^r(\d+)$")
516 # the unhandled.log file has the revision numbers
517 # encoded as r#num we should only parse as far as self.revision
518 for line
in open(os
.path
.join(cwd
, '.git', 'svn', branch
, 'unhandled.log')):
519 m
= external_expr
.search(line
)
522 rev_match
= rev_expr
.search(line
)
523 if self
.revision
and rev_match
:
524 if rev_match
.group(1) > self
.revision
:
527 # we couldn't find an unhandled.log to parse so try
528 # git svn show-externals - note this is broken in git < 1.5.6
530 output
= get_output(['git', 'svn', 'show-externals'], cwd
=cwd
,
531 extra_env
=get_git_extra_env())
532 # we search for comment lines to strip them out
533 comment_line
= re
.compile(r
"^#.*")
535 for line
in output
.splitlines():
536 if not comment_line
.search(line
):
539 match
= re
.compile("^(\.) (.+)").search(". " + ext
)
541 raise FatalError(_("External handling failed\n If you are running git version < 1.5.6 it is recommended you update.\n"))
543 # only parse the final match
545 branch
= match
.group(1)
546 external
= urllib
.unquote(match
.group(2).replace("%0A", " ").strip("%20 ")).split()
547 revision_expr
= re
.compile(r
"-r(\d*)")
549 while i
< len(external
):
550 # see if we have a revision number
551 match
= revision_expr
.search(external
[i
+1])
553 externals
[external
[i
]] = (external
[i
+2], match
.group(1))
556 externals
[external
[i
]] = (external
[i
+1], None)
559 for extdir
in externals
.iterkeys():
560 uri
= externals
[extdir
][0]
561 revision
= externals
[extdir
][1]
562 extdir
= cwd
+os
.sep
+extdir
563 # FIXME: the "right way" is to use submodules
564 extbranch
= GitSvnBranch(self
.repository
, uri
, extdir
, revision
)
567 os
.stat(extdir
)[stat
.ST_MODE
]
568 extbranch
._update
(buildscript
)
570 extbranch
._checkout
(buildscript
)
572 def _checkout(self
, buildscript
, copydir
=None):
573 if self
.config
.sticky_date
:
574 raise FatalError(_('date based checkout not yet supported\n'))
576 cmd
= ['git', 'svn', 'clone', self
.module
]
578 cmd
.append(self
.checkoutdir
)
580 # FIXME (add self.revision support)
582 last_revision
= jhbuild
.versioncontrol
.svn
.get_info (self
.module
)['last changed rev']
583 if not self
.revision
:
584 cmd
.extend(['-r', last_revision
])
586 raise FatalError(_('Cannot get last revision from %s. Check the module location.') % self
.module
)
589 buildscript
.execute(cmd
, cwd
=copydir
,
590 extra_env
=get_git_extra_env())
592 buildscript
.execute(cmd
, cwd
=self
.config
.checkoutroot
,
593 extra_env
=get_git_extra_env())
596 # is known to fail on some versions
597 cmd
= ['git', 'svn', 'show-ignore']
598 s
= get_output(cmd
, cwd
= self
.get_checkoutdir(copydir
),
599 extra_env
=get_git_extra_env())
600 fd
= file(os
.path
.join(
601 self
.get_checkoutdir(copydir
), '.git/info/exclude'), 'a')
604 buildscript
.execute(cmd
, cwd
=self
.get_checkoutdir(copydir
),
605 extra_env
=get_git_extra_env())
609 # FIXME, git-svn should support externals
610 self
._get
_externals
(buildscript
, self
.branch
)
612 def _update(self
, buildscript
, copydir
=None):
613 if self
.config
.sticky_date
:
614 raise FatalError(_('date based checkout not yet supported\n'))
616 if self
.config
.quiet_mode
:
621 cwd
= self
.get_checkoutdir()
622 git_extra_args
= {'cwd': cwd
, 'extra_env': get_git_extra_env()}
624 last_revision
= get_output(['git', 'svn', 'find-rev', 'HEAD'],
628 if get_output(['git', 'diff'], **git_extra_args
):
629 # stash uncommitted changes on the current branch
631 buildscript
.execute(['git', 'stash', 'save', 'jhbuild-stash'],
634 buildscript
.execute(['git', 'checkout'] + quiet
+ ['master'],
636 buildscript
.execute(['git', 'svn', 'rebase'], **git_extra_args
)
639 buildscript
.execute(['git', 'stash', 'pop'], **git_extra_args
)
641 current_revision
= get_output(['git', 'svn', 'find-rev', 'HEAD'],
644 if last_revision
!= current_revision
:
646 # is known to fail on some versions
647 cmd
= "git svn show-ignore >> .git/info/exclude"
648 buildscript
.execute(cmd
, **git_extra_args
)
652 # FIXME, git-svn should support externals
653 self
._get
_externals
(buildscript
, self
.branch
)
655 class GitCvsBranch(GitBranch
):
656 def __init__(self
, repository
, module
, checkoutdir
, revision
=None):
657 GitBranch
.__init
__(self
, repository
, module
, "", checkoutdir
)
658 self
.revision
= revision
660 def may_checkout(self
, buildscript
):
661 return Branch
.may_checkout(self
, buildscript
)
663 def branchname(self
):
664 for b
in ['remotes/' + str(self
.branch
), self
.branch
, 'trunk', 'master']:
665 if self
.branch_exist(b
):
668 branchname
= property(branchname
)
670 def _checkout(self
, buildscript
, copydir
=None):
672 cmd
= ['git', 'cvsimport', '-r', 'cvs', '-p', 'b,HEAD',
673 '-k', '-m', '-a', '-v', '-d', self
.repository
.cvsroot
, '-C']
676 cmd
.append(self
.checkoutdir
)
678 cmd
.append(self
.module
)
680 cmd
.append(self
.module
)
683 buildscript
.execute(cmd
, cwd
=copydir
, extra_env
=get_git_extra_env())
685 buildscript
.execute(cmd
, cwd
=self
.config
.checkoutroot
,
686 extra_env
=get_git_extra_env())
688 def _update(self
, buildscript
, copydir
=None):
689 if self
.config
.sticky_date
:
690 raise FatalError(_('date based checkout not yet supported\n'))
692 cwd
= self
.get_checkoutdir()
693 git_extra_args
= {'cwd': cwd
, 'extra_env': get_git_extra_env()}
696 # stash uncommitted changes on the current branch
697 if get_output(['git', 'diff'], **git_extra_args
):
698 # stash uncommitted changes on the current branch
700 buildscript
.execute(['git', 'stash', 'save', 'jhbuild-stash'],
703 self
._checkout
(buildscript
, copydir
=copydir
)
706 buildscript
.execute(['git', 'stash', 'pop'], **git_extra_args
)
708 register_repo_type('git', GitRepository
)