exporters/svn-fast-export.py

   1 #!/usr/bin/python
   2 #
   3 # svn-fast-export.py
   4 # ----------
   5 #  Walk through each revision of a local Subversion repository and export it
   6 #  in a stream that git-fast-import can consume.
   7 #
   8 # Author: Chris Lee <clee@kde.org>
   9 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
  10
  11 trunk_path = '/trunk/'
  12 branches_path = '/branches/'
  13 tags_path = '/tags/'
  14 address = 'localhost'
  15
  16 first_rev = 1
  17 final_rev = 0
  18
  19 import gc, sys, os.path
  20 from optparse import OptionParser
  21 from time import sleep, mktime, localtime, strftime, strptime
  22 from svn.fs import svn_fs_dir_entries, svn_fs_file_length, svn_fs_file_contents, svn_fs_is_dir, svn_fs_revision_root, svn_fs_youngest_rev, svn_fs_revision_proplist, svn_fs_revision_prop, svn_fs_paths_changed
  23 from svn.core import svn_pool_create, svn_pool_clear, svn_pool_destroy, svn_stream_read, svn_stream_for_stdout, svn_stream_copy, svn_stream_close, run_app
  24 from svn.repos import svn_repos_open, svn_repos_fs
  25
  26 ct_short = ['M', 'A', 'D', 'R', 'X']
  27
  28 def dump_file_blob(root, full_path, pool):
  29     stream_length = svn_fs_file_length(root, full_path, pool)
  30     stream = svn_fs_file_contents(root, full_path, pool)
  31     sys.stdout.write("data %s\n" % stream_length)
  32     sys.stdout.flush()
  33     ostream = svn_stream_for_stdout(pool)
  34     svn_stream_copy(stream, ostream, pool)
  35     svn_stream_close(ostream)
  36     sys.stdout.write("\n")
  37
  38
  39 class Matcher(object):
  40
  41     branch = None
  42
  43     def __init__(self, trunk_path):
  44         self.trunk_path = trunk_path
  45
  46     def branchname(self):
  47         return self.branch
  48
  49     def __str__(self):
  50         return super(Matcher, self).__str__() + ":" + self.trunk_path
  51
  52     @staticmethod
  53     def getMatcher(trunk_path):
  54         if trunk_path.startswith("regex:"):
  55             return RegexStringMatcher(trunk_path)
  56         else:
  57             return StaticStringMatcher(trunk_path)
  58
  59 class StaticStringMatcher(Matcher):
  60
  61     branch = "master"
  62
  63     def matches(self, path):
  64         return path.startswith(trunk_path)
  65
  66     def replace(self, path):
  67         return path.replace(self.trunk_path, '')
  68
  69 class RegexStringMatcher(Matcher):
  70
  71     def __init__(self, trunk_path):
  72         super(RegexStringMatcher, self).__init__(trunk_path)
  73         import re
  74         self.matcher = re.compile(self.trunk_path[len("regex:"):])
  75
  76     def matches(self, path):
  77         match = self.matcher.match(path)
  78         if match:
  79             self.branch = match.group(1)
  80             return True
  81         else:
  82             return False
  83
  84     def replace(self, path):
  85         return self.matcher.sub("\g<2>", path)
  86
  87 MATCHER = None
  88
  89 def export_revision(rev, repo, fs, pool):
  90     sys.stderr.write("Exporting revision %s... " % rev)
  91
  92     revpool = svn_pool_create(pool)
  93     svn_pool_clear(revpool)
  94
  95     # Open a root object representing the youngest (HEAD) revision.
  96     root = svn_fs_revision_root(fs, rev, revpool)
  97
  98     # And the list of what changed in this revision.
  99     changes = svn_fs_paths_changed(root, revpool)
 100
 101     i = 1
 102     marks = {}
 103     file_changes = []
 104
 105     for path, change_type in changes.iteritems():
 106         c_t = ct_short[change_type.change_kind]
 107         if svn_fs_is_dir(root, path, revpool):
 108             continue
 109         if not MATCHER.matches(path):
 110             # We don't handle branches. Or tags. Yet.
 111             pass
 112         else:
 113             if c_t == 'D':
 114                 file_changes.append("D %s" % MATCHER.replace(path).lstrip("/"))
 115             else:
 116                 marks[i] = MATCHER.replace(path)
 117                 file_changes.append("M 644 :%s %s" % (i, marks[i].lstrip("/")))
 118                 sys.stdout.write("blob\nmark :%s\n" % i)
 119                 dump_file_blob(root, path, revpool)
 120                 i += 1
 121
 122     # Get the commit author and message
 123     props = svn_fs_revision_proplist(fs, rev, revpool)
 124
 125     # Do the recursive crawl.
 126     if props.has_key('svn:author'):
 127         author = "%s <%s@%s>" % (props['svn:author'], props['svn:author'], address)
 128     else:
 129         author = 'nobody <nobody@users.sourceforge.net>'
 130
 131     if len(file_changes) == 0:
 132         svn_pool_destroy(revpool)
 133         sys.stderr.write("skipping.\n")
 134         return
 135
 136     svndate = props['svn:date'][0:-8]
 137     commit_time = mktime(strptime(svndate, '%Y-%m-%dT%H:%M:%S'))
 138     sys.stdout.write("commit refs/heads/%s\n" % MATCHER.branchname())
 139     sys.stdout.write("committer %s %s -0000\n" % (author, int(commit_time)))
 140     sys.stdout.write("data %s\n" % len(props['svn:log']))
 141     sys.stdout.write(props['svn:log'])
 142     sys.stdout.write("\n")
 143     sys.stdout.write('\n'.join(file_changes))
 144     sys.stdout.write("\n\n")
 145
 146     svn_pool_destroy(revpool)
 147
 148     sys.stderr.write("done!\n")
 149
 150     #if rev % 1000 == 0:
 151     #    sys.stderr.write("gc: %s objects\n" % len(gc.get_objects()))
 152     #    sleep(5)
 153
 154
 155 def crawl_revisions(pool, repos_path):
 156     """Open the repository at REPOS_PATH, and recursively crawl all its
 157     revisions."""
 158     global final_rev
 159
 160     # Open the repository at REPOS_PATH, and get a reference to its
 161     # versioning filesystem.
 162     repos_obj = svn_repos_open(repos_path, pool)
 163     fs_obj = svn_repos_fs(repos_obj)
 164
 165     # Query the current youngest revision.
 166     youngest_rev = svn_fs_youngest_rev(fs_obj, pool)
 167
 168
 169     if final_rev == 0:
 170         final_rev = youngest_rev
 171     for rev in xrange(first_rev, final_rev + 1):
 172         export_revision(rev, repos_obj, fs_obj, pool)
 173
 174
 175 if __name__ == '__main__':
 176     usage = '%prog [options] REPOS_PATH'
 177     parser = OptionParser()
 178     parser.set_usage(usage)
 179     parser.add_option('-f', '--final-rev', help='Final revision to import',
 180                       dest='final_rev', metavar='FINAL_REV', type='int')
 181     parser.add_option('-r', '--first-rev', help='First revision to import',
 182                       dest='first_rev', metavar='FIRST_REV', type='int')
 183     parser.add_option('-t', '--trunk-path', help="Path in repo to /trunk, may be `regex:/cvs/(trunk)/proj1/(.*)`\nFirst group is used as branchname, second to match files",
 184                       dest='trunk_path', metavar='TRUNK_PATH')
 185     parser.add_option('-b', '--branches-path', help='Path in repo to /branches',
 186                       dest='branches_path', metavar='BRANCHES_PATH')
 187     parser.add_option('-T', '--tags-path', help='Path in repo to /tags',
 188                       dest='tags_path', metavar='TAGS_PATH')
 189     parser.add_option('-a', '--address', help='Domain to put on users for their mail address',
 190                       dest='address', metavar='hostname', type='string')
 191     (options, args) = parser.parse_args()
 192
 193     if options.trunk_path != None:
 194         trunk_path = options.trunk_path
 195     if options.branches_path != None:
 196         branches_path = options.branches_path
 197     if options.tags_path != None:
 198         tags_path = options.tags_path
 199     if options.final_rev != None:
 200         final_rev = options.final_rev
 201     if options.first_rev != None:
 202         first_rev = options.first_rev
 203     if options.address != None:
 204         address = options.address
 205
 206     MATCHER = Matcher.getMatcher(trunk_path)
 207     sys.stderr.write("%s\n" % MATCHER)
 208     if len(args) != 1:
 209         parser.print_help()
 210         sys.exit(2)
 211
 212     # Canonicalize (enough for Subversion, at least) the repository path.
 213     repos_path = os.path.normpath(args[0])
 214     if repos_path == '.':
 215         repos_path = ''
 216
 217     try:
 218         import msvcrt
 219         msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 220     except ImportError:
 221         pass
 222
 223     # Call the app-wrapper, which takes care of APR initialization/shutdown
 224     # and the creation and cleanup of our top-level memory pool.
 225     run_app(crawl_revisions, repos_path)