contrib/remote-helpers/git-remote-bzr

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2012 Felipe Contreras
   4 #
   5
   6 #
   7 # Just copy to your ~/bin, or anywhere in your $PATH.
   8 # Then you can clone with:
   9 # % git clone bzr::/path/to/bzr/repo/or/url
  10 #
  11 # For example:
  12 # % git clone bzr::$HOME/myrepo
  13 # or
  14 # % git clone bzr::lp:myrepo
  15 #
  16
  17 import sys
  18
  19 import bzrlib
  20 if hasattr(bzrlib, "initialize"):
  21     bzrlib.initialize()
  22
  23 import bzrlib.plugin
  24 bzrlib.plugin.load_plugins()
  25
  26 import bzrlib.generate_ids
  27 import bzrlib.transport
  28 import bzrlib.errors
  29
  30 import sys
  31 import os
  32 import json
  33 import re
  34 import StringIO
  35 import atexit
  36
  37 NAME_RE = re.compile('^([^<>]+)')
  38 AUTHOR_RE = re.compile('^([^<>]+?)? ?<([^<>]*)>$')
  39 RAW_AUTHOR_RE = re.compile('^(\w+) (.+)? <(.*)> (\d+) ([+-]\d+)')
  40
  41 def die(msg, *args):
  42     sys.stderr.write('ERROR: %s\n' % (msg % args))
  43     sys.exit(1)
  44
  45 def warn(msg, *args):
  46     sys.stderr.write('WARNING: %s\n' % (msg % args))
  47
  48 def gittz(tz):
  49     return '%+03d%02d' % (tz / 3600, tz % 3600 / 60)
  50
  51 class Marks:
  52
  53     def __init__(self, path):
  54         self.path = path
  55         self.tips = {}
  56         self.marks = {}
  57         self.rev_marks = {}
  58         self.last_mark = 0
  59         self.load()
  60
  61     def load(self):
  62         if not os.path.exists(self.path):
  63             return
  64
  65         tmp = json.load(open(self.path))
  66         self.tips = tmp['tips']
  67         self.marks = tmp['marks']
  68         self.last_mark = tmp['last-mark']
  69
  70         for rev, mark in self.marks.iteritems():
  71             self.rev_marks[mark] = rev
  72
  73     def dict(self):
  74         return { 'tips': self.tips, 'marks': self.marks, 'last-mark' : self.last_mark }
  75
  76     def store(self):
  77         json.dump(self.dict(), open(self.path, 'w'))
  78
  79     def __str__(self):
  80         return str(self.dict())
  81
  82     def from_rev(self, rev):
  83         return self.marks[rev]
  84
  85     def to_rev(self, mark):
  86         return self.rev_marks[mark]
  87
  88     def next_mark(self):
  89         self.last_mark += 1
  90         return self.last_mark
  91
  92     def get_mark(self, rev):
  93         self.last_mark += 1
  94         self.marks[rev] = self.last_mark
  95         return self.last_mark
  96
  97     def is_marked(self, rev):
  98         return rev in self.marks
  99
 100     def new_mark(self, rev, mark):
 101         self.marks[rev] = mark
 102         self.rev_marks[mark] = rev
 103         self.last_mark = mark
 104
 105     def get_tip(self, branch):
 106         return self.tips.get(branch, None)
 107
 108     def set_tip(self, branch, tip):
 109         self.tips[branch] = tip
 110
 111 class Parser:
 112
 113     def __init__(self, repo):
 114         self.repo = repo
 115         self.line = self.get_line()
 116
 117     def get_line(self):
 118         return sys.stdin.readline().strip()
 119
 120     def __getitem__(self, i):
 121         return self.line.split()[i]
 122
 123     def check(self, word):
 124         return self.line.startswith(word)
 125
 126     def each_block(self, separator):
 127         while self.line != separator:
 128             yield self.line
 129             self.line = self.get_line()
 130
 131     def __iter__(self):
 132         return self.each_block('')
 133
 134     def next(self):
 135         self.line = self.get_line()
 136         if self.line == 'done':
 137             self.line = None
 138
 139     def get_mark(self):
 140         i = self.line.index(':') + 1
 141         return int(self.line[i:])
 142
 143     def get_data(self):
 144         if not self.check('data'):
 145             return None
 146         i = self.line.index(' ') + 1
 147         size = int(self.line[i:])
 148         return sys.stdin.read(size)
 149
 150     def get_author(self):
 151         m = RAW_AUTHOR_RE.match(self.line)
 152         if not m:
 153             return None
 154         _, name, email, date, tz = m.groups()
 155         committer = '%s <%s>' % (name, email)
 156         tz = int(tz)
 157         tz = ((tz / 100) * 3600) + ((tz % 100) * 60)
 158         return (committer, int(date), tz)
 159
 160 def rev_to_mark(rev):
 161     global marks
 162     return marks.from_rev(rev)
 163
 164 def mark_to_rev(mark):
 165     global marks
 166     return marks.to_rev(mark)
 167
 168 def fixup_user(user):
 169     name = mail = None
 170     user = user.replace('"', '')
 171     m = AUTHOR_RE.match(user)
 172     if m:
 173         name = m.group(1)
 174         mail = m.group(2).strip()
 175     else:
 176         m = NAME_RE.match(user)
 177         if m:
 178             name = m.group(1).strip()
 179
 180     return '%s <%s>' % (name, mail)
 181
 182 def get_filechanges(cur, prev):
 183     modified = {}
 184     removed = {}
 185
 186     changes = cur.changes_from(prev)
 187
 188     def u(s):
 189         return s.encode('utf-8')
 190
 191     for path, fid, kind in changes.added:
 192         modified[u(path)] = fid
 193     for path, fid, kind in changes.removed:
 194         removed[u(path)] = None
 195     for path, fid, kind, mod, _ in changes.modified:
 196         modified[u(path)] = fid
 197     for oldpath, newpath, fid, kind, mod, _ in changes.renamed:
 198         removed[u(oldpath)] = None
 199         if kind == 'directory':
 200             lst = cur.list_files(from_dir=newpath, recursive=True)
 201             for path, file_class, kind, fid, entry in lst:
 202                 if kind != 'directory':
 203                     modified[u(newpath + '/' + path)] = fid
 204         else:
 205             modified[u(newpath)] = fid
 206
 207     return modified, removed
 208
 209 def export_files(tree, files):
 210     global marks, filenodes
 211
 212     final = []
 213     for path, fid in files.iteritems():
 214         kind = tree.kind(fid)
 215
 216         h = tree.get_file_sha1(fid)
 217
 218         if kind == 'symlink':
 219             d = tree.get_symlink_target(fid)
 220             mode = '120000'
 221         elif kind == 'file':
 222
 223             if tree.is_executable(fid):
 224                 mode = '100755'
 225             else:
 226                 mode = '100644'
 227
 228             # is the blob already exported?
 229             if h in filenodes:
 230                 mark = filenodes[h]
 231                 final.append((mode, mark, path))
 232                 continue
 233
 234             d = tree.get_file_text(fid)
 235         elif kind == 'directory':
 236             continue
 237         else:
 238             die("Unhandled kind '%s' for path '%s'" % (kind, path))
 239
 240         mark = marks.next_mark()
 241         filenodes[h] = mark
 242
 243         print "blob"
 244         print "mark :%u" % mark
 245         print "data %d" % len(d)
 246         print d
 247
 248         final.append((mode, mark, path))
 249
 250     return final
 251
 252 def export_branch(branch, name):
 253     global prefix
 254
 255     ref = '%s/heads/%s' % (prefix, name)
 256     tip = marks.get_tip(name)
 257
 258     repo = branch.repository
 259     repo.lock_read()
 260     revs = branch.iter_merge_sorted_revisions(None, tip, 'exclude', 'forward')
 261     count = 0
 262
 263     revs = [revid for revid, _, _, _ in revs if not marks.is_marked(revid)]
 264
 265     for revid in revs:
 266
 267         rev = repo.get_revision(revid)
 268
 269         parents = rev.parent_ids
 270         time = rev.timestamp
 271         tz = rev.timezone
 272         committer = rev.committer.encode('utf-8')
 273         committer = "%s %u %s" % (fixup_user(committer), time, gittz(tz))
 274         authors = rev.get_apparent_authors()
 275         if authors:
 276             author = authors[0].encode('utf-8')
 277             author = "%s %u %s" % (fixup_user(author), time, gittz(tz))
 278         else:
 279             author = committer
 280         msg = rev.message.encode('utf-8')
 281
 282         msg += '\n'
 283
 284         if len(parents) == 0:
 285             parent = bzrlib.revision.NULL_REVISION
 286         else:
 287             parent = parents[0]
 288
 289         cur_tree = repo.revision_tree(revid)
 290         prev = repo.revision_tree(parent)
 291         modified, removed = get_filechanges(cur_tree, prev)
 292
 293         modified_final = export_files(cur_tree, modified)
 294
 295         if len(parents) == 0:
 296             print 'reset %s' % ref
 297
 298         print "commit %s" % ref
 299         print "mark :%d" % (marks.get_mark(revid))
 300         print "author %s" % (author)
 301         print "committer %s" % (committer)
 302         print "data %d" % (len(msg))
 303         print msg
 304
 305         for i, p in enumerate(parents):
 306             try:
 307                 m = rev_to_mark(p)
 308             except KeyError:
 309                 # ghost?
 310                 continue
 311             if i == 0:
 312                 print "from :%s" % m
 313             else:
 314                 print "merge :%s" % m
 315
 316         for f in removed:
 317             print "D %s" % (f,)
 318         for f in modified_final:
 319             print "M %s :%u %s" % f
 320         print
 321
 322         count += 1
 323         if (count % 100 == 0):
 324             print "progress revision %s (%d/%d)" % (revid, count, len(revs))
 325             print "#############################################################"
 326
 327     repo.unlock()
 328
 329     revid = branch.last_revision()
 330
 331     # make sure the ref is updated
 332     print "reset %s" % ref
 333     print "from :%u" % rev_to_mark(revid)
 334     print
 335
 336     marks.set_tip(name, revid)
 337
 338 def export_tag(repo, name):
 339     global tags, prefix
 340
 341     ref = '%s/tags/%s' % (prefix, name)
 342     print "reset %s" % ref
 343     print "from :%u" % rev_to_mark(tags[name])
 344     print
 345
 346 def do_import(parser):
 347     global dirname
 348
 349     branch = parser.repo
 350     path = os.path.join(dirname, 'marks-git')
 351
 352     print "feature done"
 353     if os.path.exists(path):
 354         print "feature import-marks=%s" % path
 355     print "feature export-marks=%s" % path
 356     sys.stdout.flush()
 357
 358     while parser.check('import'):
 359         ref = parser[1]
 360         if ref.startswith('refs/heads/'):
 361             name = ref[len('refs/heads/'):]
 362             export_branch(branch, name)
 363         if ref.startswith('refs/tags/'):
 364             name = ref[len('refs/tags/'):]
 365             export_tag(branch, name)
 366         parser.next()
 367
 368     print 'done'
 369
 370     sys.stdout.flush()
 371
 372 def parse_blob(parser):
 373     global blob_marks
 374
 375     parser.next()
 376     mark = parser.get_mark()
 377     parser.next()
 378     data = parser.get_data()
 379     blob_marks[mark] = data
 380     parser.next()
 381
 382 class CustomTree():
 383
 384     def __init__(self, repo, revid, parents, files):
 385         global files_cache
 386
 387         self.repo = repo
 388         self.revid = revid
 389         self.parents = parents
 390         self.updates = {}
 391
 392         def copy_tree(revid):
 393             files = files_cache[revid] = {}
 394             tree = repo.repository.revision_tree(revid)
 395             repo.lock_read()
 396             try:
 397                 for path, entry in tree.iter_entries_by_dir():
 398                     files[path] = entry.file_id
 399             finally:
 400                 repo.unlock()
 401             return files
 402
 403         if len(parents) == 0:
 404             self.base_id = bzrlib.revision.NULL_REVISION
 405             self.base_files = {}
 406         else:
 407             self.base_id = parents[0]
 408             self.base_files = files_cache.get(self.base_id, None)
 409             if not self.base_files:
 410                 self.base_files = copy_tree(self.base_id)
 411
 412         self.files = files_cache[revid] = self.base_files.copy()
 413
 414         for path, f in files.iteritems():
 415             fid = self.files.get(path, None)
 416             if not fid:
 417                 fid = bzrlib.generate_ids.gen_file_id(path)
 418             f['path'] = path
 419             self.updates[fid] = f
 420
 421     def last_revision(self):
 422         return self.base_id
 423
 424     def iter_changes(self):
 425         changes = []
 426
 427         def get_parent(dirname, basename):
 428             parent_fid = self.base_files.get(dirname, None)
 429             if parent_fid:
 430                 return parent_fid
 431             parent_fid = self.files.get(dirname, None)
 432             if parent_fid:
 433                 return parent_fid
 434             if basename == '':
 435                 return None
 436             fid = bzrlib.generate_ids.gen_file_id(path)
 437             d = add_entry(fid, dirname, 'directory')
 438             return fid
 439
 440         def add_entry(fid, path, kind, mode = None):
 441             dirname, basename = os.path.split(path)
 442             parent_fid = get_parent(dirname, basename)
 443
 444             executable = False
 445             if mode == '100755':
 446                 executable = True
 447             elif mode == '120000':
 448                 kind = 'symlink'
 449
 450             change = (fid,
 451                     (None, path),
 452                     True,
 453                     (False, True),
 454                     (None, parent_fid),
 455                     (None, basename),
 456                     (None, kind),
 457                     (None, executable))
 458             self.files[path] = change[0]
 459             changes.append(change)
 460             return change
 461
 462         def update_entry(fid, path, kind, mode = None):
 463             dirname, basename = os.path.split(path)
 464             parent_fid = get_parent(dirname, basename)
 465
 466             executable = False
 467             if mode == '100755':
 468                 executable = True
 469             elif mode == '120000':
 470                 kind = 'symlink'
 471
 472             change = (fid,
 473                     (path, path),
 474                     True,
 475                     (True, True),
 476                     (None, parent_fid),
 477                     (None, basename),
 478                     (None, kind),
 479                     (None, executable))
 480             self.files[path] = change[0]
 481             changes.append(change)
 482             return change
 483
 484         def remove_entry(fid, path, kind):
 485             dirname, basename = os.path.split(path)
 486             parent_fid = get_parent(dirname, basename)
 487             change = (fid,
 488                     (path, None),
 489                     True,
 490                     (True, False),
 491                     (parent_fid, None),
 492                     (None, None),
 493                     (None, None),
 494                     (None, None))
 495             del self.files[path]
 496             changes.append(change)
 497             return change
 498
 499         for fid, f in self.updates.iteritems():
 500             path = f['path']
 501
 502             if 'deleted' in f:
 503                 remove_entry(fid, path, 'file')
 504                 continue
 505
 506             if path in self.base_files:
 507                 update_entry(fid, path, 'file', f['mode'])
 508             else:
 509                 add_entry(fid, path, 'file', f['mode'])
 510
 511         return changes
 512
 513     def get_file_with_stat(self, file_id, path=None):
 514         return (StringIO.StringIO(self.updates[file_id]['data']), None)
 515
 516     def get_symlink_target(self, file_id):
 517         return self.updates[file_id]['data']
 518
 519 def c_style_unescape(string):
 520     if string[0] == string[-1] == '"':
 521         return string.decode('string-escape')[1:-1]
 522     return string
 523
 524 def parse_commit(parser):
 525     global marks, blob_marks, parsed_refs
 526     global mode
 527
 528     parents = []
 529
 530     ref = parser[1]
 531     parser.next()
 532
 533     if ref != 'refs/heads/master':
 534         die("bzr doesn't support multiple branches; use 'master'")
 535
 536     commit_mark = parser.get_mark()
 537     parser.next()
 538     author = parser.get_author()
 539     parser.next()
 540     committer = parser.get_author()
 541     parser.next()
 542     data = parser.get_data()
 543     parser.next()
 544     if parser.check('from'):
 545         parents.append(parser.get_mark())
 546         parser.next()
 547     while parser.check('merge'):
 548         parents.append(parser.get_mark())
 549         parser.next()
 550
 551     files = {}
 552
 553     for line in parser:
 554         if parser.check('M'):
 555             t, m, mark_ref, path = line.split(' ', 3)
 556             mark = int(mark_ref[1:])
 557             f = { 'mode' : m, 'data' : blob_marks[mark] }
 558         elif parser.check('D'):
 559             t, path = line.split(' ')
 560             f = { 'deleted' : True }
 561         else:
 562             die('Unknown file command: %s' % line)
 563         path = c_style_unescape(path).decode('utf-8')
 564         files[path] = f
 565
 566     repo = parser.repo
 567
 568     committer, date, tz = committer
 569     parents = [str(mark_to_rev(p)) for p in parents]
 570     revid = bzrlib.generate_ids.gen_revision_id(committer, date)
 571     props = {}
 572     props['branch-nick'] = repo.nick
 573
 574     mtree = CustomTree(repo, revid, parents, files)
 575     changes = mtree.iter_changes()
 576
 577     repo.lock_write()
 578     try:
 579         builder = repo.get_commit_builder(parents, None, date, tz, committer, props, revid)
 580         try:
 581             list(builder.record_iter_changes(mtree, mtree.last_revision(), changes))
 582             builder.finish_inventory()
 583             builder.commit(data.decode('utf-8', 'replace'))
 584         except Exception, e:
 585             builder.abort()
 586             raise
 587     finally:
 588         repo.unlock()
 589
 590     parsed_refs[ref] = revid
 591     marks.new_mark(revid, commit_mark)
 592
 593 def parse_reset(parser):
 594     global parsed_refs
 595
 596     ref = parser[1]
 597     parser.next()
 598
 599     if ref != 'refs/heads/master':
 600         die("bzr doesn't support multiple branches; use 'master'")
 601
 602     # ugh
 603     if parser.check('commit'):
 604         parse_commit(parser)
 605         return
 606     if not parser.check('from'):
 607         return
 608     from_mark = parser.get_mark()
 609     parser.next()
 610
 611     parsed_refs[ref] = mark_to_rev(from_mark)
 612
 613 def do_export(parser):
 614     global parsed_refs, dirname, peer
 615
 616     parser.next()
 617
 618     for line in parser.each_block('done'):
 619         if parser.check('blob'):
 620             parse_blob(parser)
 621         elif parser.check('commit'):
 622             parse_commit(parser)
 623         elif parser.check('reset'):
 624             parse_reset(parser)
 625         elif parser.check('tag'):
 626             pass
 627         elif parser.check('feature'):
 628             pass
 629         else:
 630             die('unhandled export command: %s' % line)
 631
 632     repo = parser.repo
 633
 634     for ref, revid in parsed_refs.iteritems():
 635         if ref == 'refs/heads/master':
 636             repo.generate_revision_history(revid, marks.get_tip('master'))
 637             if peer:
 638                 try:
 639                     repo.push(peer, stop_revision=revid)
 640                 except bzrlib.errors.DivergedBranches:
 641                     print "error %s non-fast forward" % ref
 642                     continue
 643             else:
 644                 wt = repo.bzrdir.open_workingtree()
 645                 wt.update()
 646         print "ok %s" % ref
 647
 648     print
 649
 650 def do_capabilities(parser):
 651     global dirname
 652
 653     print "import"
 654     print "export"
 655     print "refspec refs/heads/*:%s/heads/*" % prefix
 656     print "refspec refs/tags/*:%s/tags/*" % prefix
 657
 658     path = os.path.join(dirname, 'marks-git')
 659
 660     if os.path.exists(path):
 661         print "*import-marks %s" % path
 662     print "*export-marks %s" % path
 663
 664     print
 665
 666 def ref_is_valid(name):
 667     return not True in [c in name for c in '~^: \\']
 668
 669 def do_list(parser):
 670     global tags
 671     print "? refs/heads/%s" % 'master'
 672
 673     branch = parser.repo
 674     branch.lock_read()
 675     for tag, revid in branch.tags.get_tag_dict().items():
 676         try:
 677             branch.revision_id_to_dotted_revno(revid)
 678         except bzrlib.errors.NoSuchRevision:
 679             continue
 680         if not ref_is_valid(tag):
 681             continue
 682         print "? refs/tags/%s" % tag
 683         tags[tag] = revid
 684     branch.unlock()
 685     print "@refs/heads/%s HEAD" % 'master'
 686     print
 687
 688 def get_repo(url, alias):
 689     global dirname, peer
 690
 691     origin = bzrlib.bzrdir.BzrDir.open(url)
 692     branch = origin.open_branch()
 693
 694     if not isinstance(origin.transport, bzrlib.transport.local.LocalTransport):
 695         clone_path = os.path.join(dirname, 'clone')
 696         remote_branch = branch
 697         if os.path.exists(clone_path):
 698             # pull
 699             d = bzrlib.bzrdir.BzrDir.open(clone_path)
 700             branch = d.open_branch()
 701             result = branch.pull(remote_branch, [], None, False)
 702         else:
 703             # clone
 704             d = origin.sprout(clone_path, None,
 705                     hardlink=True, create_tree_if_local=False,
 706                     source_branch=remote_branch)
 707             branch = d.open_branch()
 708             branch.bind(remote_branch)
 709
 710         peer = remote_branch
 711     else:
 712         peer = None
 713
 714     return branch
 715
 716 def main(args):
 717     global marks, prefix, dirname
 718     global tags, filenodes
 719     global blob_marks
 720     global parsed_refs
 721     global files_cache
 722
 723     alias = args[1]
 724     url = args[2]
 725
 726     prefix = 'refs/bzr/%s' % alias
 727     tags = {}
 728     filenodes = {}
 729     blob_marks = {}
 730     parsed_refs = {}
 731     files_cache = {}
 732     marks = None
 733
 734     gitdir = os.environ['GIT_DIR']
 735     dirname = os.path.join(gitdir, 'bzr', alias)
 736
 737     if not os.path.exists(dirname):
 738         os.makedirs(dirname)
 739
 740     repo = get_repo(url, alias)
 741
 742     marks_path = os.path.join(dirname, 'marks-int')
 743     marks = Marks(marks_path)
 744
 745     parser = Parser(repo)
 746     for line in parser:
 747         if parser.check('capabilities'):
 748             do_capabilities(parser)
 749         elif parser.check('list'):
 750             do_list(parser)
 751         elif parser.check('import'):
 752             do_import(parser)
 753         elif parser.check('export'):
 754             do_export(parser)
 755         else:
 756             die('unhandled command: %s' % line)
 757         sys.stdout.flush()
 758
 759 def bye():
 760     if not marks:
 761         return
 762     marks.store()
 763
 764 atexit.register(bye)
 765 sys.exit(main(sys.argv))