contrib/remote-helpers/git-remote-bzr

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2012 Felipe Contreras
   4 #
   5
   6 #
   7 # Just copy to your ~/bin, or anywhere in your $PATH.
   8 # Then you can clone with:
   9 # % git clone bzr::/path/to/bzr/repo/or/url
  10 #
  11 # For example:
  12 # % git clone bzr::$HOME/myrepo
  13 # or
  14 # % git clone bzr::lp:myrepo
  15 #
  16
  17 import sys
  18
  19 import bzrlib
  20 if hasattr(bzrlib, "initialize"):
  21     bzrlib.initialize()
  22
  23 import bzrlib.plugin
  24 bzrlib.plugin.load_plugins()
  25
  26 import bzrlib.generate_ids
  27 import bzrlib.transport
  28 import bzrlib.errors
  29 import bzrlib.ui
  30
  31 import sys
  32 import os
  33 import json
  34 import re
  35 import StringIO
  36 import atexit, shutil, hashlib, urlparse, subprocess
  37
  38 NAME_RE = re.compile('^([^<>]+)')
  39 AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
  40 RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
  41
  42 def die(msg, *args):
  43     sys.stderr.write('ERROR: %s\n' % (msg % args))
  44     sys.exit(1)
  45
  46 def warn(msg, *args):
  47     sys.stderr.write('WARNING: %s\n' % (msg % args))
  48
  49 def gittz(tz):
  50     return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
  51
  52 class Marks:
  53
  54     def __init__(self, path):
  55         self.path = path
  56         self.tips = {}
  57         self.marks = {}
  58         self.rev_marks = {}
  59         self.last_mark = 0
  60         self.load()
  61
  62     def load(self):
  63         if not os.path.exists(self.path):
  64             return
  65
  66         tmp = json.load(open(self.path))
  67         self.tips = tmp['tips']
  68         self.marks = tmp['marks']
  69         self.last_mark = tmp['last-mark']
  70
  71         for rev, mark in self.marks.iteritems():
  72             self.rev_marks[mark] = rev
  73
  74     def dict(self):
  75         return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  76
  77     def store(self):
  78         json.dump(self.dict(), open(self.path, 'w'))
  79
  80     def __str__(self):
  81         return str(self.dict())
  82
  83     def from_rev(self, rev):
  84         return self.marks[rev]
  85
  86     def to_rev(self, mark):
  87         return self.rev_marks[mark]
  88
  89     def next_mark(self):
  90         self.last_mark += 1
  91         return self.last_mark
  92
  93     def get_mark(self, rev):
  94         self.last_mark += 1
  95         self.marks[rev] = self.last_mark
  96         return self.last_mark
  97
  98     def is_marked(self, rev):
  99         return rev in self.marks
 100
 101     def new_mark(self, rev, mark):
 102         self.marks[rev] = mark
 103         self.rev_marks[mark] = rev
 104         self.last_mark = mark
 105
 106     def get_tip(self, branch):
 107         return self.tips.get(branch, None)
 108
 109     def set_tip(self, branch, tip):
 110         self.tips[branch] = tip
 111
 112 class Parser:
 113
 114     def __init__(self, repo):
 115         self.repo = repo
 116         self.line = self.get_line()
 117
 118     def get_line(self):
 119         return sys.stdin.readline().strip()
 120
 121     def __getitem__(self, i):
 122         return self.line.split()[i]
 123
 124     def check(self, word):
 125         return self.line.startswith(word)
 126
 127     def each_block(self, separator):
 128         while self.line != separator:
 129             yield self.line
 130             self.line = self.get_line()
 131
 132     def __iter__(self):
 133         return self.each_block('')
 134
 135     def next(self):
 136         self.line = self.get_line()
 137         if self.line == 'done':
 138             self.line = None
 139
 140     def get_mark(self):
 141         i = self.line.index(':') + 1
 142         return int(self.line[i:])
 143
 144     def get_data(self):
 145         if not self.check('data'):
 146             return None
 147         i = self.line.index(' ') + 1
 148         size = int(self.line[i:])
 149         return sys.stdin.read(size)
 150
 151     def get_author(self):
 152         m = RAW_AUTHOR_RE.match(self.line)
 153         if not m:
 154             return None
 155         _, name, email, date, tz = m.groups()
 156         committer = '%s <%s>' % (name, email)
 157         tz = int(tz)
 158         tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 159         return (committer, int(date), tz)
 160
 161 def rev_to_mark(rev):
 162     global marks
 163     return marks.from_rev(rev)
 164
 165 def mark_to_rev(mark):
 166     global marks
 167     return marks.to_rev(mark)
 168
 169 def fixup_user(user):
 170     name = mail = None
 171     user = user.replace('"', '')
 172     m = AUTHOR_RE.match(user)
 173     if m:
 174         name = m.group(1)
 175         mail = m.group(2).strip()
 176     else:
 177         m = NAME_RE.match(user)
 178         if m:
 179             name = m.group(1).strip()
 180
 181     return '%s <%s>' % (name, mail)
 182
 183 def get_filechanges(cur, prev):
 184     modified = {}
 185     removed = {}
 186
 187     changes = cur.changes_from(prev)
 188
 189     def u(s):
 190         return s.encode('utf-8')
 191
 192     for path, fid, kind in changes.added:
 193         modified[u(path)] = fid
 194     for path, fid, kind in changes.removed:
 195         removed[u(path)] = None
 196     for path, fid, kind, mod, _ in changes.modified:
 197         modified[u(path)] = fid
 198     for oldpath, newpath, fid, kind, mod, _ in changes.renamed:
 199         removed[u(oldpath)] = None
 200         if kind == 'directory':
 201             lst = cur.list_files(from_dir=newpath, recursive=True)
 202             for path, file_class, kind, fid, entry in lst:
 203                 if kind != 'directory':
 204                     modified[u(newpath + '/' + path)] = fid
 205         else:
 206             modified[u(newpath)] = fid
 207
 208     return modified, removed
 209
 210 def export_files(tree, files):
 211     global marks, filenodes
 212
 213     final = []
 214     for path, fid in files.iteritems():
 215         kind = tree.kind(fid)
 216
 217         h = tree.get_file_sha1(fid)
 218
 219         if kind == 'symlink':
 220             d = tree.get_symlink_target(fid)
 221             mode = '120000'
 222         elif kind == 'file':
 223
 224             if tree.is_executable(fid):
 225                 mode = '100755'
 226             else:
 227                 mode = '100644'
 228
 229             # is the blob already exported?
 230             if h in filenodes:
 231                 mark = filenodes[h]
 232                 final.append((mode, mark, path))
 233                 continue
 234
 235             d = tree.get_file_text(fid)
 236         elif kind == 'directory':
 237             continue
 238         else:
 239             die("Unhandled kind '%s' for path '%s'" % (kind, path))
 240
 241         mark = marks.next_mark()
 242         filenodes[h] = mark
 243
 244         print "blob"
 245         print "mark :%u" % mark
 246         print "data %d" % len(d)
 247         print d
 248
 249         final.append((mode, mark, path))
 250
 251     return final
 252
 253 def export_branch(branch, name):
 254     global prefix
 255
 256     ref = '%s/heads/%s' % (prefix, name)
 257     tip = marks.get_tip(name)
 258
 259     repo = branch.repository
 260     repo.lock_read()
 261     revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
 262     count = 0
 263
 264     revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
 265
 266     for revid in revs:
 267
 268         rev = repo.get_revision(revid)
 269
 270         parents = rev.parent_ids
 271         time = rev.timestamp
 272         tz = rev.timezone
 273         committer = rev.committer.encode('utf-8')
 274         committer = "%s %u %s" % (fixup_user(committer), time, gittz(tz))
 275         authors = rev.get_apparent_authors()
 276         if authors:
 277             author = authors[0].encode('utf-8')
 278             author = "%s %u %s" % (fixup_user(author), time, gittz(tz))
 279         else:
 280             author = committer
 281         msg = rev.message.encode('utf-8')
 282
 283         msg += '\n'
 284
 285         if len(parents) == 0:
 286             parent = bzrlib.revision.NULL_REVISION
 287         else:
 288             parent = parents[0]
 289
 290         cur_tree = repo.revision_tree(revid)
 291         prev = repo.revision_tree(parent)
 292         modified, removed = get_filechanges(cur_tree, prev)
 293
 294         modified_final = export_files(cur_tree, modified)
 295
 296         if len(parents) == 0:
 297             print 'reset %s' % ref
 298
 299         print "commit %s" % ref
 300         print "mark :%d" % (marks.get_mark(revid))
 301         print "author %s" % (author)
 302         print "committer %s" % (committer)
 303         print "data %d" % (len(msg))
 304         print msg
 305
 306         for i, p in enumerate(parents):
 307             try:
 308                 m = rev_to_mark(p)
 309             except KeyError:
 310                 # ghost?
 311                 continue
 312             if i == 0:
 313                 print "from :%s" % m
 314             else:
 315                 print "merge :%s" % m
 316
 317         for f in removed:
 318             print "D %s" % (f,)
 319         for f in modified_final:
 320             print "M %s :%u %s" % f
 321         print
 322
 323         count += 1
 324         if (count % 100 == 0):
 325             print "progress revision %s (%d/%d)" % (revid, count, len(revs))
 326             print "#############################################################"
 327
 328     repo.unlock()
 329
 330     revid = branch.last_revision()
 331
 332     # make sure the ref is updated
 333     print "reset %s" % ref
 334     print "from :%u" % rev_to_mark(revid)
 335     print
 336
 337     marks.set_tip(name, revid)
 338
 339 def export_tag(repo, name):
 340     global tags, prefix
 341
 342     ref = '%s/tags/%s' % (prefix, name)
 343     print "reset %s" % ref
 344     print "from :%u" % rev_to_mark(tags[name])
 345     print
 346
 347 def do_import(parser):
 348     global dirname
 349
 350     branch = parser.repo
 351     path = os.path.join(dirname, 'marks-git')
 352
 353     print "feature done"
 354     if os.path.exists(path):
 355         print "feature import-marks=%s" % path
 356     print "feature export-marks=%s" % path
 357     sys.stdout.flush()
 358
 359     while parser.check('import'):
 360         ref = parser[1]
 361         if ref.startswith('refs/heads/'):
 362             name = ref[len('refs/heads/'):]
 363             export_branch(branch, name)
 364         if ref.startswith('refs/tags/'):
 365             name = ref[len('refs/tags/'):]
 366             export_tag(branch, name)
 367         parser.next()
 368
 369     print 'done'
 370
 371     sys.stdout.flush()
 372
 373 def parse_blob(parser):
 374     global blob_marks
 375
 376     parser.next()
 377     mark = parser.get_mark()
 378     parser.next()
 379     data = parser.get_data()
 380     blob_marks[mark] = data
 381     parser.next()
 382
 383 class CustomTree():
 384
 385     def __init__(self, repo, revid, parents, files):
 386         global files_cache
 387
 388         self.updates = {}
 389
 390         def copy_tree(revid):
 391             files = files_cache[revid] = {}
 392             tree = repo.repository.revision_tree(revid)
 393             repo.lock_read()
 394             try:
 395                 for path, entry in tree.iter_entries_by_dir():
 396                     files[path] = entry.file_id
 397             finally:
 398                 repo.unlock()
 399             return files
 400
 401         if len(parents) == 0:
 402             self.base_id = bzrlib.revision.NULL_REVISION
 403             self.base_files = {}
 404         else:
 405             self.base_id = parents[0]
 406             self.base_files = files_cache.get(self.base_id, None)
 407             if not self.base_files:
 408                 self.base_files = copy_tree(self.base_id)
 409
 410         self.files = files_cache[revid] = self.base_files.copy()
 411
 412         for path, f in files.iteritems():
 413             fid = self.files.get(path, None)
 414             if not fid:
 415                 fid = bzrlib.generate_ids.gen_file_id(path)
 416             f['path'] = path
 417             self.updates[fid] = f
 418
 419     def last_revision(self):
 420         return self.base_id
 421
 422     def iter_changes(self):
 423         changes = []
 424
 425         def get_parent(dirname, basename):
 426             parent_fid = self.base_files.get(dirname, None)
 427             if parent_fid:
 428                 return parent_fid
 429             parent_fid = self.files.get(dirname, None)
 430             if parent_fid:
 431                 return parent_fid
 432             if basename == '':
 433                 return None
 434             fid = bzrlib.generate_ids.gen_file_id(path)
 435             add_entry(fid, dirname, 'directory')
 436             return fid
 437
 438         def add_entry(fid, path, kind, mode = None):
 439             dirname, basename = os.path.split(path)
 440             parent_fid = get_parent(dirname, basename)
 441
 442             executable = False
 443             if mode == '100755':
 444                 executable = True
 445             elif mode == '120000':
 446                 kind = 'symlink'
 447
 448             change = (fid,
 449                     (None, path),
 450                     True,
 451                     (False, True),
 452                     (None, parent_fid),
 453                     (None, basename),
 454                     (None, kind),
 455                     (None, executable))
 456             self.files[path] = change[0]
 457             changes.append(change)
 458
 459         def update_entry(fid, path, kind, mode = None):
 460             dirname, basename = os.path.split(path)
 461             parent_fid = get_parent(dirname, basename)
 462
 463             executable = False
 464             if mode == '100755':
 465                 executable = True
 466             elif mode == '120000':
 467                 kind = 'symlink'
 468
 469             change = (fid,
 470                     (path, path),
 471                     True,
 472                     (True, True),
 473                     (None, parent_fid),
 474                     (None, basename),
 475                     (None, kind),
 476                     (None, executable))
 477             self.files[path] = change[0]
 478             changes.append(change)
 479
 480         def remove_entry(fid, path, kind):
 481             dirname, basename = os.path.split(path)
 482             parent_fid = get_parent(dirname, basename)
 483             change = (fid,
 484                     (path, None),
 485                     True,
 486                     (True, False),
 487                     (parent_fid, None),
 488                     (None, None),
 489                     (None, None),
 490                     (None, None))
 491             del self.files[path]
 492             changes.append(change)
 493
 494         for fid, f in self.updates.iteritems():
 495             path = f['path']
 496
 497             if 'deleted' in f:
 498                 remove_entry(fid, path, 'file')
 499                 continue
 500
 501             if path in self.base_files:
 502                 update_entry(fid, path, 'file', f['mode'])
 503             else:
 504                 add_entry(fid, path, 'file', f['mode'])
 505
 506         return changes
 507
 508     def get_file_with_stat(self, file_id, path=None):
 509         mark = self.updates[file_id]['mark']
 510         return (StringIO.StringIO(blob_marks[mark]), None)
 511
 512     def get_symlink_target(self, file_id):
 513         mark = self.updates[file_id]['mark']
 514         return blob_marks[mark]
 515
 516 def c_style_unescape(string):
 517     if string[0] == string[-1] == '"':
 518         return string.decode('string-escape')[1:-1]
 519     return string
 520
 521 def parse_commit(parser):
 522     global marks, blob_marks, parsed_refs
 523     global mode
 524
 525     parents = []
 526
 527     ref = parser[1]
 528     parser.next()
 529
 530     if ref != 'refs/heads/master':
 531         die("bzr doesn't support multiple branches; use 'master'")
 532
 533     commit_mark = parser.get_mark()
 534     parser.next()
 535     author = parser.get_author()
 536     parser.next()
 537     committer = parser.get_author()
 538     parser.next()
 539     data = parser.get_data()
 540     parser.next()
 541     if parser.check('from'):
 542         parents.append(parser.get_mark())
 543         parser.next()
 544     while parser.check('merge'):
 545         parents.append(parser.get_mark())
 546         parser.next()
 547
 548     # fast-export adds an extra newline
 549     if data[-1] == '\n':
 550         data = data[:-1]
 551
 552     files = {}
 553
 554     for line in parser:
 555         if parser.check('M'):
 556             t, m, mark_ref, path = line.split(' ', 3)
 557             mark = int(mark_ref[1:])
 558             f = { 'mode' : m, 'mark' : mark }
 559         elif parser.check('D'):
 560             t, path = line.split(' ')
 561             f = { 'deleted' : True }
 562         else:
 563             die('Unknown file command: %s' % line)
 564         path = c_style_unescape(path).decode('utf-8')
 565         files[path] = f
 566
 567     repo = parser.repo
 568
 569     committer, date, tz = committer
 570     parents = [str(mark_to_rev(p)) for p in parents]
 571     revid = bzrlib.generate_ids.gen_revision_id(committer, date)
 572     props = {}
 573     props['branch-nick'] = repo.nick
 574
 575     mtree = CustomTree(repo, revid, parents, files)
 576     changes = mtree.iter_changes()
 577
 578     repo.lock_write()
 579     try:
 580         builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
 581         try:
 582             list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
 583             builder.finish_inventory()
 584             builder.commit(data.decode('utf-8', 'replace'))
 585         except Exception, e:
 586             builder.abort()
 587             raise
 588     finally:
 589         repo.unlock()
 590
 591     parsed_refs[ref] = revid
 592     marks.new_mark(revid, commit_mark)
 593
 594 def parse_reset(parser):
 595     global parsed_refs
 596
 597     ref = parser[1]
 598     parser.next()
 599
 600     if ref != 'refs/heads/master':
 601         die("bzr doesn't support multiple branches; use 'master'")
 602
 603     # ugh
 604     if parser.check('commit'):
 605         parse_commit(parser)
 606         return
 607     if not parser.check('from'):
 608         return
 609     from_mark = parser.get_mark()
 610     parser.next()
 611
 612     parsed_refs[ref] = mark_to_rev(from_mark)
 613
 614 def do_export(parser):
 615     global parsed_refs, dirname, peer
 616
 617     parser.next()
 618
 619     for line in parser.each_block('done'):
 620         if parser.check('blob'):
 621             parse_blob(parser)
 622         elif parser.check('commit'):
 623             parse_commit(parser)
 624         elif parser.check('reset'):
 625             parse_reset(parser)
 626         elif parser.check('tag'):
 627             pass
 628         elif parser.check('feature'):
 629             pass
 630         else:
 631             die('unhandled export command: %s' % line)
 632
 633     repo = parser.repo
 634
 635     for ref, revid in parsed_refs.iteritems():
 636         if ref == 'refs/heads/master':
 637             repo.generate_revision_history(revid, marks.get_tip('master'))
 638             if peer:
 639                 try:
 640                     repo.push(peer, stop_revision=revid)
 641                 except bzrlib.errors.DivergedBranches:
 642                     print "error %s non-fast forward" % ref
 643                     continue
 644             else:
 645                 wt = repo.bzrdir.open_workingtree()
 646                 wt.update()
 647         print "ok %s" % ref
 648
 649     print
 650
 651 def do_capabilities(parser):
 652     global dirname
 653
 654     print "import"
 655     print "export"
 656     print "refspec refs/heads/*:%s/heads/*" % prefix
 657     print "refspec refs/tags/*:%s/tags/*" % prefix
 658
 659     path = os.path.join(dirname, 'marks-git')
 660
 661     if os.path.exists(path):
 662         print "*import-marks %s" % path
 663     print "*export-marks %s" % path
 664
 665     print
 666
 667 def ref_is_valid(name):
 668     return not True in [c in name for c in '~^: \\']
 669
 670 def do_list(parser):
 671     global tags
 672     print "? refs/heads/%s" % 'master'
 673
 674     branch = parser.repo
 675     branch.lock_read()
 676     for tag, revid in branch.tags.get_tag_dict().items():
 677         try:
 678             branch.revision_id_to_dotted_revno(revid)
 679         except bzrlib.errors.NoSuchRevision:
 680             continue
 681         if not ref_is_valid(tag):
 682             continue
 683         print "? refs/tags/%s" % tag
 684         tags[tag] = revid
 685     branch.unlock()
 686     print "@refs/heads/%s HEAD" % 'master'
 687     print
 688
 689 def get_repo(url, alias):
 690     global dirname, peer
 691
 692     origin = bzrlib.bzrdir.BzrDir.open(url)
 693     branch = origin.open_branch()
 694
 695     if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
 696         clone_path = os.path.join(dirname, 'clone')
 697         remote_branch = branch
 698         if os.path.exists(clone_path):
 699             # pull
 700             d = bzrlib.bzrdir.BzrDir.open(clone_path)
 701             branch = d.open_branch()
 702             result = branch.pull(remote_branch, [], None, False)
 703         else:
 704             # clone
 705             d = origin.sprout(clone_path, None,
 706                     hardlink=True, create_tree_if_local=False,
 707                     source_branch=remote_branch)
 708             branch = d.open_branch()
 709             branch.bind(remote_branch)
 710
 711         peer = remote_branch
 712     else:
 713         peer = None
 714
 715     return branch
 716
 717 def fix_path(alias, orig_url):
 718     url = urlparse.urlparse(orig_url, 'file')
 719     if url.scheme != 'file' or os.path.isabs(url.path):
 720         return
 721     abs_url = urlparse.urljoin("%s/" % os.getcwd(), orig_url)
 722     cmd = ['git', 'config', 'remote.%s.url' % alias, "bzr::%s" % abs_url]
 723     subprocess.call(cmd)
 724
 725 def main(args):
 726     global marks, prefix, dirname
 727     global tags, filenodes
 728     global blob_marks
 729     global parsed_refs
 730     global files_cache
 731     global is_tmp
 732
 733     alias = args[1]
 734     url = args[2]
 735
 736     tags = {}
 737     filenodes = {}
 738     blob_marks = {}
 739     parsed_refs = {}
 740     files_cache = {}
 741     marks = None
 742
 743     if alias[5:] == url:
 744         is_tmp = True
 745         alias = hashlib.sha1(alias).hexdigest()
 746     else:
 747         is_tmp = False
 748
 749     prefix = 'refs/bzr/%s' % alias
 750     gitdir = os.environ['GIT_DIR']
 751     dirname = os.path.join(gitdir, 'bzr', alias)
 752
 753     if not is_tmp:
 754         fix_path(alias, url)
 755
 756     if not os.path.exists(dirname):
 757         os.makedirs(dirname)
 758
 759     bzrlib.ui.ui_factory.be_quiet(True)
 760
 761     repo = get_repo(url, alias)
 762
 763     marks_path = os.path.join(dirname, 'marks-int')
 764     marks = Marks(marks_path)
 765
 766     parser = Parser(repo)
 767     for line in parser:
 768         if parser.check('capabilities'):
 769             do_capabilities(parser)
 770         elif parser.check('list'):
 771             do_list(parser)
 772         elif parser.check('import'):
 773             do_import(parser)
 774         elif parser.check('export'):
 775             do_export(parser)
 776         else:
 777             die('unhandled command: %s' % line)
 778         sys.stdout.flush()
 779
 780 def bye():
 781     if not marks:
 782         return
 783     if not is_tmp:
 784         marks.store()
 785     else:
 786         shutil.rmtree(dirname)
 787
 788 atexit.register(bye)
 789 sys.exit(main(sys.argv))