contrib/remote-helpers/git-remote-bzr

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2012 Felipe Contreras
   4 #
   5
   6 #
   7 # Just copy to your ~/bin, or anywhere in your $PATH.
   8 # Then you can clone with:
   9 # % git clone bzr::/path/to/bzr/repo/or/url
  10 #
  11 # For example:
  12 # % git clone bzr::$HOME/myrepo
  13 # or
  14 # % git clone bzr::lp:myrepo
  15 #
  16
  17 import sys
  18
  19 import bzrlib
  20 if hasattr(bzrlib, "initialize"):
  21     bzrlib.initialize()
  22
  23 import bzrlib.plugin
  24 bzrlib.plugin.load_plugins()
  25
  26 import bzrlib.generate_ids
  27 import bzrlib.transport
  28 import bzrlib.errors
  29
  30 import sys
  31 import os
  32 import json
  33 import re
  34 import StringIO
  35
  36 NAME_RE = re.compile('^([^<>]+)')
  37 AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
  38 RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
  39
  40 def die(msg, *args):
  41     sys.stderr.write('ERROR: %s\n' % (msg % args))
  42     sys.exit(1)
  43
  44 def warn(msg, *args):
  45     sys.stderr.write('WARNING: %s\n' % (msg % args))
  46
  47 def gittz(tz):
  48     return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
  49
  50 class Marks:
  51
  52     def __init__(self, path):
  53         self.path = path
  54         self.tips = {}
  55         self.marks = {}
  56         self.rev_marks = {}
  57         self.last_mark = 0
  58         self.load()
  59
  60     def load(self):
  61         if not os.path.exists(self.path):
  62             return
  63
  64         tmp = json.load(open(self.path))
  65         self.tips = tmp['tips']
  66         self.marks = tmp['marks']
  67         self.last_mark = tmp['last-mark']
  68
  69         for rev, mark in self.marks.iteritems():
  70             self.rev_marks[mark] = rev
  71
  72     def dict(self):
  73         return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  74
  75     def store(self):
  76         json.dump(self.dict(), open(self.path, 'w'))
  77
  78     def __str__(self):
  79         return str(self.dict())
  80
  81     def from_rev(self, rev):
  82         return self.marks[rev]
  83
  84     def to_rev(self, mark):
  85         return self.rev_marks[mark]
  86
  87     def next_mark(self):
  88         self.last_mark += 1
  89         return self.last_mark
  90
  91     def get_mark(self, rev):
  92         self.last_mark += 1
  93         self.marks[rev] = self.last_mark
  94         return self.last_mark
  95
  96     def is_marked(self, rev):
  97         return self.marks.has_key(rev)
  98
  99     def new_mark(self, rev, mark):
 100         self.marks[rev] = mark
 101         self.rev_marks[mark] = rev
 102         self.last_mark = mark
 103
 104     def get_tip(self, branch):
 105         return self.tips.get(branch, None)
 106
 107     def set_tip(self, branch, tip):
 108         self.tips[branch] = tip
 109
 110 class Parser:
 111
 112     def __init__(self, repo):
 113         self.repo = repo
 114         self.line = self.get_line()
 115
 116     def get_line(self):
 117         return sys.stdin.readline().strip()
 118
 119     def __getitem__(self, i):
 120         return self.line.split()[i]
 121
 122     def check(self, word):
 123         return self.line.startswith(word)
 124
 125     def each_block(self, separator):
 126         while self.line != separator:
 127             yield self.line
 128             self.line = self.get_line()
 129
 130     def __iter__(self):
 131         return self.each_block('')
 132
 133     def next(self):
 134         self.line = self.get_line()
 135         if self.line == 'done':
 136             self.line = None
 137
 138     def get_mark(self):
 139         i = self.line.index(':') + 1
 140         return int(self.line[i:])
 141
 142     def get_data(self):
 143         if not self.check('data'):
 144             return None
 145         i = self.line.index(' ') + 1
 146         size = int(self.line[i:])
 147         return sys.stdin.read(size)
 148
 149     def get_author(self):
 150         m = RAW_AUTHOR_RE.match(self.line)
 151         if not m:
 152             return None
 153         _, name, email, date, tz = m.groups()
 154         committer = '%s <%s>' % (name, email)
 155         tz = int(tz)
 156         tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 157         return (committer, int(date), tz)
 158
 159 def rev_to_mark(rev):
 160     global marks
 161     return marks.from_rev(rev)
 162
 163 def mark_to_rev(mark):
 164     global marks
 165     return marks.to_rev(mark)
 166
 167 def fixup_user(user):
 168     name = mail = None
 169     user = user.replace('"', '')
 170     m = AUTHOR_RE.match(user)
 171     if m:
 172         name = m.group(1)
 173         mail = m.group(2).strip()
 174     else:
 175         m = NAME_RE.match(user)
 176         if m:
 177             name = m.group(1).strip()
 178
 179     return '%s <%s>' % (name, mail)
 180
 181 def get_filechanges(cur, prev):
 182     modified = {}
 183     removed = {}
 184
 185     changes = cur.changes_from(prev)
 186
 187     def u(s):
 188         return s.encode('utf-8')
 189
 190     for path, fid, kind in changes.added:
 191         modified[u(path)] = fid
 192     for path, fid, kind in changes.removed:
 193         removed[u(path)] = None
 194     for path, fid, kind, mod, _ in changes.modified:
 195         modified[u(path)] = fid
 196     for oldpath, newpath, fid, kind, mod, _ in changes.renamed:
 197         removed[u(oldpath)] = None
 198         if kind == 'directory':
 199             lst = cur.list_files(from_dir=newpath, recursive=True)
 200             for path, file_class, kind, fid, entry in lst:
 201                 if kind != 'directory':
 202                     modified[u(newpath + '/' + path)] = fid
 203         else:
 204             modified[u(newpath)] = fid
 205
 206     return modified, removed
 207
 208 def export_files(tree, files):
 209     global marks, filenodes
 210
 211     final = []
 212     for path, fid in files.iteritems():
 213         kind = tree.kind(fid)
 214
 215         h = tree.get_file_sha1(fid)
 216
 217         if kind == 'symlink':
 218             d = tree.get_symlink_target(fid)
 219             mode = '120000'
 220         elif kind == 'file':
 221
 222             if tree.is_executable(fid):
 223                 mode = '100755'
 224             else:
 225                 mode = '100644'
 226
 227             # is the blog already exported?
 228             if h in filenodes:
 229                 mark = filenodes[h]
 230                 final.append((mode, mark, path))
 231                 continue
 232
 233             d = tree.get_file_text(fid)
 234         elif kind == 'directory':
 235             continue
 236         else:
 237             die("Unhandled kind '%s' for path '%s'" % (kind, path))
 238
 239         mark = marks.next_mark()
 240         filenodes[h] = mark
 241
 242         print "blob"
 243         print "mark :%u" % mark
 244         print "data %d" % len(d)
 245         print d
 246
 247         final.append((mode, mark, path))
 248
 249     return final
 250
 251 def export_branch(branch, name):
 252     global prefix, dirname
 253
 254     ref = '%s/heads/%s' % (prefix, name)
 255     tip = marks.get_tip(name)
 256
 257     repo = branch.repository
 258     repo.lock_read()
 259     revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
 260     count = 0
 261
 262     revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
 263
 264     for revid in revs:
 265
 266         rev = repo.get_revision(revid)
 267
 268         parents = rev.parent_ids
 269         time = rev.timestamp
 270         tz = rev.timezone
 271         committer = rev.committer.encode('utf-8')
 272         committer = "%s %u %s" % (fixup_user(committer), time, gittz(tz))
 273         authors = rev.get_apparent_authors()
 274         if authors:
 275             author = authors[0].encode('utf-8')
 276             author = "%s %u %s" % (fixup_user(author), time, gittz(tz))
 277         else:
 278             author = committer
 279         msg = rev.message.encode('utf-8')
 280
 281         msg += '\n'
 282
 283         if len(parents) == 0:
 284             parent = bzrlib.revision.NULL_REVISION
 285         else:
 286             parent = parents[0]
 287
 288         cur_tree = repo.revision_tree(revid)
 289         prev = repo.revision_tree(parent)
 290         modified, removed = get_filechanges(cur_tree, prev)
 291
 292         modified_final = export_files(cur_tree, modified)
 293
 294         if len(parents) == 0:
 295             print 'reset %s' % ref
 296
 297         print "commit %s" % ref
 298         print "mark :%d" % (marks.get_mark(revid))
 299         print "author %s" % (author)
 300         print "committer %s" % (committer)
 301         print "data %d" % (len(msg))
 302         print msg
 303
 304         for i, p in enumerate(parents):
 305             try:
 306                 m = rev_to_mark(p)
 307             except KeyError:
 308                 # ghost?
 309                 continue
 310             if i == 0:
 311                 print "from :%s" % m
 312             else:
 313                 print "merge :%s" % m
 314
 315         for f in removed:
 316             print "D %s" % (f,)
 317         for f in modified_final:
 318             print "M %s :%u %s" % f
 319         print
 320
 321         count += 1
 322         if (count % 100 == 0):
 323             print "progress revision %s (%d/%d)" % (revid, count, len(revs))
 324             print "#############################################################"
 325
 326     repo.unlock()
 327
 328     revid = branch.last_revision()
 329
 330     # make sure the ref is updated
 331     print "reset %s" % ref
 332     print "from :%u" % rev_to_mark(revid)
 333     print
 334
 335     marks.set_tip(name, revid)
 336
 337 def export_tag(repo, name):
 338     global tags
 339     print "reset refs/tags/%s" % name
 340     print "from :%u" % rev_to_mark(tags[name])
 341     print
 342
 343 def do_import(parser):
 344     global dirname
 345
 346     branch = parser.repo
 347     path = os.path.join(dirname, 'marks-git')
 348
 349     print "feature done"
 350     if os.path.exists(path):
 351         print "feature import-marks=%s" % path
 352     print "feature export-marks=%s" % path
 353     sys.stdout.flush()
 354
 355     while parser.check('import'):
 356         ref = parser[1]
 357         if ref.startswith('refs/heads/'):
 358             name = ref[len('refs/heads/'):]
 359             export_branch(branch, name)
 360         if ref.startswith('refs/tags/'):
 361             name = ref[len('refs/tags/'):]
 362             export_tag(branch, name)
 363         parser.next()
 364
 365     print 'done'
 366
 367     sys.stdout.flush()
 368
 369 def parse_blob(parser):
 370     global blob_marks
 371
 372     parser.next()
 373     mark = parser.get_mark()
 374     parser.next()
 375     data = parser.get_data()
 376     blob_marks[mark] = data
 377     parser.next()
 378
 379 class CustomTree():
 380
 381     def __init__(self, repo, revid, parents, files):
 382         global files_cache
 383
 384         self.repo = repo
 385         self.revid = revid
 386         self.parents = parents
 387         self.updates = {}
 388
 389         def copy_tree(revid):
 390             files = files_cache[revid] = {}
 391             tree = repo.repository.revision_tree(revid)
 392             repo.lock_read()
 393             try:
 394                 for path, entry in tree.iter_entries_by_dir():
 395                     files[path] = entry.file_id
 396             finally:
 397                 repo.unlock()
 398             return files
 399
 400         if len(parents) == 0:
 401             self.base_id = bzrlib.revision.NULL_REVISION
 402             self.base_files = {}
 403         else:
 404             self.base_id = parents[0]
 405             self.base_files = files_cache.get(self.base_id, None)
 406             if not self.base_files:
 407                 self.base_files = copy_tree(self.base_id)
 408
 409         self.files = files_cache[revid] = self.base_files.copy()
 410
 411         for path, f in files.iteritems():
 412             fid = self.files.get(path, None)
 413             if not fid:
 414                 fid = bzrlib.generate_ids.gen_file_id(path)
 415             f['path'] = path
 416             self.updates[fid] = f
 417
 418     def last_revision(self):
 419         return self.base_id
 420
 421     def iter_changes(self):
 422         changes = []
 423
 424         def get_parent(dirname, basename):
 425             parent_fid = self.base_files.get(dirname, None)
 426             if parent_fid:
 427                 return parent_fid
 428             parent_fid = self.files.get(dirname, None)
 429             if parent_fid:
 430                 return parent_fid
 431             if basename == '':
 432                 return None
 433             fid = bzrlib.generate_ids.gen_file_id(path)
 434             d = add_entry(fid, dirname, 'directory')
 435             return fid
 436
 437         def add_entry(fid, path, kind, mode = None):
 438             dirname, basename = os.path.split(path)
 439             parent_fid = get_parent(dirname, basename)
 440
 441             executable = False
 442             if mode == '100755':
 443                 executable = True
 444             elif mode == '120000':
 445                 kind = 'symlink'
 446
 447             change = (fid,
 448                     (None, path),
 449                     True,
 450                     (False, True),
 451                     (None, parent_fid),
 452                     (None, basename),
 453                     (None, kind),
 454                     (None, executable))
 455             self.files[path] = change[0]
 456             changes.append(change)
 457             return change
 458
 459         def update_entry(fid, path, kind, mode = None):
 460             dirname, basename = os.path.split(path)
 461             parent_fid = get_parent(dirname, basename)
 462
 463             executable = False
 464             if mode == '100755':
 465                 executable = True
 466             elif mode == '120000':
 467                 kind = 'symlink'
 468
 469             change = (fid,
 470                     (path, path),
 471                     True,
 472                     (True, True),
 473                     (None, parent_fid),
 474                     (None, basename),
 475                     (None, kind),
 476                     (None, executable))
 477             self.files[path] = change[0]
 478             changes.append(change)
 479             return change
 480
 481         def remove_entry(fid, path, kind):
 482             dirname, basename = os.path.split(path)
 483             parent_fid = get_parent(dirname, basename)
 484             change = (fid,
 485                     (path, None),
 486                     True,
 487                     (True, False),
 488                     (parent_fid, None),
 489                     (None, None),
 490                     (None, None),
 491                     (None, None))
 492             del self.files[path]
 493             changes.append(change)
 494             return change
 495
 496         for fid, f in self.updates.iteritems():
 497             path = f['path']
 498
 499             if 'deleted' in f:
 500                 remove_entry(fid, path, 'file')
 501                 continue
 502
 503             if path in self.base_files:
 504                 update_entry(fid, path, 'file', f['mode'])
 505             else:
 506                 add_entry(fid, path, 'file', f['mode'])
 507
 508         return changes
 509
 510     def get_file_with_stat(self, file_id, path=None):
 511         return (StringIO.StringIO(self.updates[file_id]['data']), None)
 512
 513     def get_symlink_target(self, file_id):
 514         return self.updates[file_id]['data']
 515
 516 def c_style_unescape(string):
 517     if string[0] == string[-1] == '"':
 518         return string.decode('string-escape')[1:-1]
 519     return string
 520
 521 def parse_commit(parser):
 522     global marks, blob_marks, bmarks, parsed_refs
 523     global mode
 524
 525     parents = []
 526
 527     ref = parser[1]
 528     parser.next()
 529
 530     if ref != 'refs/heads/master':
 531         die("bzr doesn't support multiple branches; use 'master'")
 532
 533     commit_mark = parser.get_mark()
 534     parser.next()
 535     author = parser.get_author()
 536     parser.next()
 537     committer = parser.get_author()
 538     parser.next()
 539     data = parser.get_data()
 540     parser.next()
 541     if parser.check('from'):
 542         parents.append(parser.get_mark())
 543         parser.next()
 544     while parser.check('merge'):
 545         parents.append(parser.get_mark())
 546         parser.next()
 547
 548     files = {}
 549
 550     for line in parser:
 551         if parser.check('M'):
 552             t, m, mark_ref, path = line.split(' ', 3)
 553             mark = int(mark_ref[1:])
 554             f = { 'mode' : m, 'data' : blob_marks[mark] }
 555         elif parser.check('D'):
 556             t, path = line.split(' ')
 557             f = { 'deleted' : True }
 558         else:
 559             die('Unknown file command: %s' % line)
 560         path = c_style_unescape(path).decode('utf-8')
 561         files[path] = f
 562
 563     repo = parser.repo
 564
 565     committer, date, tz = committer
 566     parents = [str(mark_to_rev(p)) for p in parents]
 567     revid = bzrlib.generate_ids.gen_revision_id(committer, date)
 568     props = {}
 569     props['branch-nick'] = repo.nick
 570
 571     mtree = CustomTree(repo, revid, parents, files)
 572     changes = mtree.iter_changes()
 573
 574     repo.lock_write()
 575     try:
 576         builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
 577         try:
 578             list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
 579             builder.finish_inventory()
 580             builder.commit(data.decode('utf-8', 'replace'))
 581         except Exception, e:
 582             builder.abort()
 583             raise
 584     finally:
 585         repo.unlock()
 586
 587     parsed_refs[ref] = revid
 588     marks.new_mark(revid, commit_mark)
 589
 590 def parse_reset(parser):
 591     global parsed_refs
 592
 593     ref = parser[1]
 594     parser.next()
 595
 596     if ref != 'refs/heads/master':
 597         die("bzr doesn't support multiple branches; use 'master'")
 598
 599     # ugh
 600     if parser.check('commit'):
 601         parse_commit(parser)
 602         return
 603     if not parser.check('from'):
 604         return
 605     from_mark = parser.get_mark()
 606     parser.next()
 607
 608     parsed_refs[ref] = mark_to_rev(from_mark)
 609
 610 def do_export(parser):
 611     global parsed_refs, dirname, peer
 612
 613     parser.next()
 614
 615     for line in parser.each_block('done'):
 616         if parser.check('blob'):
 617             parse_blob(parser)
 618         elif parser.check('commit'):
 619             parse_commit(parser)
 620         elif parser.check('reset'):
 621             parse_reset(parser)
 622         elif parser.check('tag'):
 623             pass
 624         elif parser.check('feature'):
 625             pass
 626         else:
 627             die('unhandled export command: %s' % line)
 628
 629     repo = parser.repo
 630
 631     for ref, revid in parsed_refs.iteritems():
 632         if ref == 'refs/heads/master':
 633             repo.generate_revision_history(revid, marks.get_tip('master'))
 634             revno, revid = repo.last_revision_info()
 635             if peer:
 636                 if hasattr(peer, "import_last_revision_info_and_tags"):
 637                     peer.import_last_revision_info_and_tags(repo, revno, revid)
 638                 else:
 639                     peer.import_last_revision_info(repo.repository, revno, revid)
 640             else:
 641                 wt = repo.bzrdir.open_workingtree()
 642                 wt.update()
 643         print "ok %s" % ref
 644     print
 645
 646 def do_capabilities(parser):
 647     global dirname
 648
 649     print "import"
 650     print "export"
 651     print "refspec refs/heads/*:%s/heads/*" % prefix
 652
 653     path = os.path.join(dirname, 'marks-git')
 654
 655     if os.path.exists(path):
 656         print "*import-marks %s" % path
 657     print "*export-marks %s" % path
 658
 659     print
 660
 661 def ref_is_valid(name):
 662     return not True in [c in name for c in '~^: \\']
 663
 664 def do_list(parser):
 665     global tags
 666     print "? refs/heads/%s" % 'master'
 667
 668     branch = parser.repo
 669     branch.lock_read()
 670     for tag, revid in branch.tags.get_tag_dict().items():
 671         try:
 672             branch.revision_id_to_dotted_revno(revid)
 673         except bzrlib.errors.NoSuchRevision:
 674             continue
 675         if not ref_is_valid(tag):
 676             continue
 677         print "? refs/tags/%s" % tag
 678         tags[tag] = revid
 679     branch.unlock()
 680     print "@refs/heads/%s HEAD" % 'master'
 681     print
 682
 683 def get_repo(url, alias):
 684     global dirname, peer
 685
 686     origin = bzrlib.bzrdir.BzrDir.open(url)
 687     branch = origin.open_branch()
 688
 689     if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
 690         clone_path = os.path.join(dirname, 'clone')
 691         remote_branch = branch
 692         if os.path.exists(clone_path):
 693             # pull
 694             d = bzrlib.bzrdir.BzrDir.open(clone_path)
 695             branch = d.open_branch()
 696             result = branch.pull(remote_branch, [], None, False)
 697         else:
 698             # clone
 699             d = origin.sprout(clone_path, None,
 700                     hardlink=True, create_tree_if_local=False,
 701                     source_branch=remote_branch)
 702             branch = d.open_branch()
 703             branch.bind(remote_branch)
 704
 705         peer = remote_branch
 706     else:
 707         peer = None
 708
 709     return branch
 710
 711 def main(args):
 712     global marks, prefix, dirname
 713     global tags, filenodes
 714     global blob_marks
 715     global parsed_refs
 716     global files_cache
 717
 718     alias = args[1]
 719     url = args[2]
 720
 721     prefix = 'refs/bzr/%s' % alias
 722     tags = {}
 723     filenodes = {}
 724     blob_marks = {}
 725     parsed_refs = {}
 726     files_cache = {}
 727
 728     gitdir = os.environ['GIT_DIR']
 729     dirname = os.path.join(gitdir, 'bzr', alias)
 730
 731     if not os.path.exists(dirname):
 732         os.makedirs(dirname)
 733
 734     repo = get_repo(url, alias)
 735
 736     marks_path = os.path.join(dirname, 'marks-int')
 737     marks = Marks(marks_path)
 738
 739     parser = Parser(repo)
 740     for line in parser:
 741         if parser.check('capabilities'):
 742             do_capabilities(parser)
 743         elif parser.check('list'):
 744             do_list(parser)
 745         elif parser.check('import'):
 746             do_import(parser)
 747         elif parser.check('export'):
 748             do_export(parser)
 749         else:
 750             die('unhandled command: %s' % line)
 751         sys.stdout.flush()
 752
 753     marks.store()
 754
 755 sys.exit(main(sys.argv))