contrib/mtn2git/mtn2git.py

   1 #!/usr/bin/env python
   2
   3 """
   4   Copyright (C) 2006, 2007, 2008 Holger Hans Peter Freyther
   5
   6   Permission is hereby granted, free of charge, to any person obtaining a copy
   7   of this software and associated documentation files (the "Software"), to deal
   8   in the Software without restriction, including without limitation the rights
   9   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10   copies of the Software, and to permit persons to whom the Software is
  11   furnished to do so, subject to the following conditions:
  12
  13   The above copyright notice and this permission notice shall be included in
  14   all copies or substantial portions of the Software.
  15
  16   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22   THE SOFTWARE.
  23 """
  24
  25 ####
  26 # TODO:
  27 #   -tag handling
  28 #   -work with n-merges
  29 #
  30
  31 import mtn
  32 import os
  33 import sys
  34 import datetime
  35 import email.Utils
  36
  37 import status
  38
  39 # Interesting revisions:
  40 # Rename with dest==src: 24cba5923360fef7c5cc81d51000e30b90355eb9
  41 # Recursive rename: fca159c5c00ae4158c289f5aabce995378d4e41b
  42 # Delete+Rename: 91da98265a39c93946e00adf5d7bf92b341de847
  43 #
  44 #
  45 #
  46
  47 # Our manifest/tree fifo construct
  48 cached_tree = {}
  49 cached_fifo = []
  50
  51 def get_mark(revision):
  52     """
  53     Get a mark for a specific revision. If the revision is known the former
  54     mark will be returned. Otherwise a new mark will be allocated and stored
  55     to the mark file.
  56     """
  57     if revision in status.marks:
  58         return status.marks[revision]
  59     status.last_mark += 1
  60     status.marks[revision] = status.last_mark
  61     print >> status.mark_file, "%d: %s" % (status.last_mark, revision)
  62     status.mark_file.flush()
  63     return status.last_mark
  64
  65 def has_mark(revision):
  66     return revision in status.marks
  67
  68 def get_branch_name(revision):
  69     """
  70     TODO for unnamed branches (e.g. as we lack the certs) we might want to follow
  71     the parents until we end up at a item with a branch name and then use the last
  72     item without a name...
  73     """
  74     if "branch" in revision:
  75         branch = revision["branch"]
  76     else:
  77         #branch = "initial-%s" % revision["revision"]
  78         branch = "mtn-rev-%s" % revision["revision"]
  79     return branch
  80
  81 def reset_git(ops, revision):
  82     """
  83     Find the name of the branch of this revision
  84     """
  85     branch = get_branch_name(revision)
  86
  87     cmd = []
  88     cmd += ["reset refs/heads/%s" % branch]
  89     cmd += ["from :%s" % get_mark(revision["revision"])]
  90     cmd += [""]
  91     print "\n".join(cmd)
  92
  93 def checkpoint():
  94     """
  95     Force git to checkpoint the import
  96     """
  97     cmd = []
  98     cmd += ["checkpoint"]
  99     cmd += [""]
 100     print "\n".join(cmd)
 101
 102 def get_git_date(revision):
 103     """
 104     Convert the "date" cert of monotone to a time understandable by git. No timezone
 105     conversions are done.
 106     """
 107     dt = datetime.datetime.strptime(revision["date"], "%Y-%m-%dT%H:%M:%S").strftime("%a, %d %b %Y %H:%M:%S +0000")
 108     return dt
 109
 110 def is_executable_attribute_set(attributes, rev):
 111     assert(len(attributes) % 3 == 0), rev
 112
 113     if len(attributes) >= 3:
 114         for i in range(0, len(attributes)%3+1):
 115             if attributes[i] == "attr" and attributes[i+1] == "mtn:execute" and attributes[i+2] == "true":
 116                 return True
 117     return False
 118
 119 def build_tree(manifest, rev):
 120     """Assemble a filesystem tree from a given manifest"""
 121
 122     class tree:
 123         def __init__(self):
 124             self.dirs = {}
 125             self.files= {}
 126
 127     tree = tree()
 128     for line in manifest:
 129         if line[0] == "file":
 130             tree.files[line[1]] = (line[3], is_executable_attribute_set(line[4:], rev))
 131         elif line[0] == "dir":
 132             tree.dirs[line[1]] = 1
 133         elif line[0] != "format_version":
 134             assert(False), "Rev: %s: Line[0]: '%s'" % (rev, line[0])
 135
 136     return tree
 137
 138 def get_and_cache_tree(ops, revision):
 139     """Simple FIFO to cache a number of trees"""
 140     global cached_tree, cached_fifo
 141
 142     if revision in cached_tree:
 143         return cached_tree[revision]
 144
 145     tree = build_tree([line for line in ops.get_manifest_of(revision)], revision)
 146     cached_tree[revision] = tree
 147     cached_fifo.append(revision)
 148
 149     # Shrink
 150     if len(cached_fifo) > 100:
 151         old_name = cached_fifo[0]
 152         cached_fifo = cached_fifo[1:]
 153         del cached_tree[old_name]
 154
 155     return tree
 156
 157 def diff_manifest(old_tree, new_tree):
 158     """Find additions, modifications and deletions"""
 159     added = set()
 160     modified = set()
 161     deleted = set()
 162
 163     # Removed dirs
 164     for dir in old_tree.dirs.keys():
 165         if not dir in new_tree.dirs:
 166             deleted.add((dir,True))
 167
 168     # New dirs
 169     for dir in new_tree.dirs.keys():
 170         if not dir in old_tree.dirs:
 171             added.add(dir)
 172
 173     # Deleted files
 174     for file in old_tree.files.keys():
 175         if not file in new_tree.files:
 176             deleted.add((file,False))
 177
 178     # Added files, goes to modifications
 179     for file in new_tree.files.keys():
 180         if not file in old_tree.files:
 181             modified.add((file, new_tree.files[file][0]))
 182             continue
 183
 184         # The file changed, either contents or executable attribute
 185         old = old_tree.files[file]
 186         new = new_tree.files[file]
 187         if old != new:
 188             modified.add((file, new_tree.files[file][0]))
 189
 190     return (added, modified, deleted)
 191
 192 def fast_import(ops, revision):
 193     """Import a revision into git using git-fast-import.
 194
 195     First convert the revision to something git-fast-import
 196     can understand
 197     """
 198     assert("revision" in revision)
 199     assert("author" in revision)
 200     assert("committer" in revision)
 201     assert("parent" in revision)
 202
 203     branch = get_branch_name(revision)
 204
 205     # Use the manifest to find dirs and files
 206     current_tree = get_and_cache_tree(ops, revision["revision"])
 207
 208     # Now diff the manifests
 209     if len(revision["parent"]) == 0:
 210         merge_from = None
 211         merge_other = []
 212         (added, modified, deleted) = diff_manifest(build_tree([],""), current_tree)
 213     else:
 214         # The first parent is our from.
 215         merge_from = revision["parent"][0]
 216         merge_other = revision["parent"][1:]
 217         (added, modified, deleted) = diff_manifest(get_and_cache_tree(ops, merge_from), current_tree)
 218
 219     # TODO:
 220     # Readd the sanity check to see if we deleted and modified an entry. This
 221     # could probably happen if we have more than one parent (on a merge)?
 222
 223     cmd = []
 224     if len(revision["parent"]) == 0:
 225         cmd += ["reset refs/heads/%s" % branch]
 226     cmd += ["commit refs/heads/%s" % branch]
 227     cmd += ["mark :%s" % get_mark(revision["revision"])]
 228     cmd += ["author  <%s> %s" % (revision["author"], get_git_date(revision))]
 229     cmd += ["committer  <%s> %s" % (revision["committer"], get_git_date(revision))]
 230     cmd += ["data  %d" % len(revision["changelog"])]
 231     cmd += ["%s" % revision["changelog"]]
 232
 233     if not merge_from is None:
 234         cmd += ["from :%s" % get_mark(merge_from)]
 235
 236     for parent in merge_other:
 237         cmd += ["merge :%s" % get_mark(parent)]
 238
 239     for dir_name in added:
 240         cmd += ["M 644 inline %s" % os.path.join(dir_name, ".mtn2git_empty")]
 241         cmd += ["data <<EOF"]
 242         cmd += ["EOF"]
 243         cmd += [""]
 244
 245     for (file_name, file_revision) in modified:
 246         (mode, file) = get_file_and_mode(ops, current_tree, file_name, file_revision, revision["revision"])
 247         cmd += ["M %d inline %s" % (mode, file_name)]
 248         cmd += ["data %d" % len(file)]
 249         cmd += ["%s" % file]
 250
 251     for (path, is_dir) in deleted:
 252         if is_dir:
 253             cmd += ["D %s" % os.path.join(path, ".mtn2git_empty")]
 254         else:
 255             cmd += ["D %s" % path]
 256
 257     cmd += [""]
 258     print "\n".join(cmd)
 259
 260 def is_trusted(operations, revision):
 261     for cert in operations.certs(revision):
 262         if cert[0] != 'key' or cert[3] != 'ok' or cert[8] != 'trust' or cert[9] != 'trusted':
 263             print >> sys.stderr, "Cert untrusted?, this must be bad", cert
 264             return False
 265     return True
 266
 267 def get_file_and_mode(operations, file_tree, file_name, _file_revision, rev = None):
 268     assert file_name in file_tree.files, "get_file_and_mode: Revision '%s', file_name='%s' " % (rev, file_name)
 269
 270     (file_revision, executable) = file_tree.files[file_name]
 271     if _file_revision:
 272         assert _file_revision == file_revision, "Same filerevision for file_name='%s' in rev='%s' (%s,%s)" % (file_name, rev, file_revision, _file_revision)
 273
 274     if executable:
 275         mode = 755
 276     else:
 277         mode = 644
 278
 279     file = "".join([file for file in operations.get_file(file_revision)])
 280     return (mode, file)
 281
 282 def parse_revision(operations, revision):
 283     """
 284     Parse a revision as of mtn automate get_revision
 285
 286     Return a tuple with the current version, a list of parents,
 287     a list of operations and their revision
 288     """
 289     if not is_trusted(operations, revision):
 290         raise Exception("Revision %s is not trusted!" % revision)
 291
 292     # The order of certain operations, e.g rename matter so don't use a set
 293     revision_description = {}
 294     revision_description["revision"] = revision
 295     revision_description["added_dirs"] = []
 296     revision_description["added_files"] = []
 297     revision_description["removed"] = []
 298     revision_description["modified"] = []
 299     revision_description["renamed"] = []
 300     revision_description["set_attributes"] = []
 301     revision_description["clear_attributes"] = []
 302
 303     old_rev = None
 304
 305     for line in operations.get_revision(revision):
 306         if line[0] == "format_version":
 307             assert(line[1] == "1")
 308         elif line[0] == "old_revision":
 309             if not "parent" in revision_description:
 310                 revision_description["parent"] = []
 311             if len(line[1]) != 0:
 312                 revision_description["parent"].append(line[1])
 313             old_rev = line[1]
 314         elif line[0] == "new_manifest":
 315             revision_description["manifest"] = line[1]
 316         elif line[0] == "clear":
 317             revision_description["clear_attributes"].append((line[1], line[3], old_rev))
 318         elif line[0] == "set":
 319             revision_description["set_attributes"].append((line[1], line[3], line[5], old_rev))
 320         elif line[0] in ["rename", "patch", "delete", "add_dir", "add_file"]:
 321             pass
 322         else:
 323             print >> sys.stderr, line
 324             assert(False)
 325
 326     for cert in operations.certs(revision):
 327         # Known cert names used by mtn, we can ignore them as they can't be converted to git
 328         if cert[5] in ["suspend", "testresult", "file-comment", "comment", "release-candidate"]:
 329             pass
 330         elif cert[5] in ["author", "changelog", "date", "branch", "tag"]:
 331             revision_description[cert[5]] = cert[7]
 332             if cert[5] == "author":
 333                 revision_description["committer"] = cert[1]
 334         else:
 335             print >> sys.stderr, "Unknown Cert: Ignoring", cert[5], cert[7]
 336             #assert(False)
 337
 338     return revision_description
 339
 340 def tests(ops, revs):
 341     """Load a bunch of revisions and exit"""
 342     for rev in revs:
 343         print >> sys.stderr, rev
 344         fast_import(ops, parse_revision(ops, rev))
 345
 346     sys.exit()
 347
 348 def main(mtn_cli, db, rev):
 349     if not db:
 350         print >> sys.stderr, "You need to specifiy a monotone db"
 351         sys.exit()
 352
 353     ops = mtn.Operations([mtn_cli, db])
 354
 355     # Double rename in mtn
 356     #tests(ops, ["fca159c5c00ae4158c289f5aabce995378d4e41b"])
 357
 358     # Rename and remove in OE
 359     #tests(ops, ["74db43a4ad2bccd5f2fd59339e4ece0092f8dcb0"])
 360
 361     # Rename + Dele
 362     #tests(ops, ["91da98265a39c93946e00adf5d7bf92b341de847"])
 363
 364     # Issue with renaming in OE
 365     #tests(ops, ["c81294b86c62ee21791776732f72f4646f402445"])
 366
 367     # Unterminated inner renames
 368     #tests(ops, ["d813a779ef7157f88dade0b8ccef32f28ff34a6e", "4d027b6bcd69e7eb5b64b2e720c9953d5378d845", "af5ffd789f2852e635aa4af88b56a893b7a83a79"])
 369
 370     # Broken rename in OE. double replacing of the directory command
 371     #tests(ops, ["11f85aab185581dcbff7dce29e44f7c1f0572a27"])
 372
 373     if rev:
 374         tests(ops, [rev])
 375         sys.exit()
 376
 377     branches = [branch.name for branch in ops.branches()]
 378     ops.automate.stop()
 379
 380     all_revs = []
 381     branch_heads = {}
 382     for branch in branches:
 383         heads = [head for head in ops.heads(branch)]
 384         if len(heads) != 1:
 385             print >> sys.stderr, "Skipping branch '%s' due multiple heads" % (branch)
 386             continue
 387
 388         if branch in status.former_heads:
 389             old_heads = status.former_heads[branch]
 390         else:
 391             old_heads = []
 392
 393         for head in heads:
 394             print >> sys.stderr, old_heads, head
 395             all_revs += ops.ancestry_difference(head, old_heads)
 396         status.former_heads[branch] = heads
 397
 398     counter = 0
 399     sorted_revs = [rev for rev in ops.toposort(all_revs)]
 400     for rev in sorted_revs:
 401         if has_mark(rev):
 402             print >> sys.stderr, "B: Already having commit '%s'" % rev
 403         else:
 404             print >> sys.stderr, "Going to import revision ", rev
 405             fast_import(ops, parse_revision(ops, rev))
 406             if counter % 1000 == 0:
 407                 checkpoint()
 408             counter += 1
 409
 410 if __name__ == "__main__":
 411     import optparse
 412     parser = optparse.OptionParser()
 413     parser.add_option("-d", "--db", dest="database",
 414                       help="The monotone database to use")
 415     parser.add_option("-m", "--marks", dest="marks", default="mtn2git-marks",
 416                       help="The marks allocated by the mtn2git command")
 417     parser.add_option("-t", "--mtn", dest="mtn", default="mtn",
 418                       help="The name of the mtn command to use")
 419     parser.add_option("-s", "--status", dest="status", default="mtn2git.status.v2",
 420                       help="The status file as used by %prog")
 421     parser.add_option("-r", "--revision", dest="rev", default=None,
 422                       help="Import a single revision to help debugging.")
 423
 424     (options,_) = parser.parse_args(sys.argv)
 425     status.mark_file = file(options.marks, "a")
 426
 427     try:
 428         status.load(options.status)
 429     except IOError:
 430         print >> sys.stderr, "Failed to open the status file"
 431     main(options.mtn, options.database, options.rev)
 432     status.store(options.status)
 433