From 47d330de83055efc5d4eeec6d0593d7aa3640555 Mon Sep 17 00:00:00 2001 From: Johannes Carlsson Date: Wed, 2 Jan 2019 12:38:34 +0100 Subject: [PATCH] Add support for mercurial subrepos This adds a new command line option (--subrepo-map) that will map mercurial subrepos to git submodules. The --subrepo-map takes a mapping file as an argument that will be used to map a subrepo folder to a git submodule. For more information see the README-SUBMODULES.md. This commit is inspired by the changes made by daolis in PR#38 that was never merged. Closes: #51 Closes: #147 --- README-SUBMODULES.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 5 ++++ hg-fast-export.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 README-SUBMODULES.md diff --git a/README-SUBMODULES.md b/README-SUBMODULES.md new file mode 100644 index 0000000..f55797a --- /dev/null +++ b/README-SUBMODULES.md @@ -0,0 +1,65 @@ +# How to convert Mercurial Repositories with subrepos + +## Introduction + +Subrepositories must first be converted in order for the conversion of +the super repository to know how hg commits map to git commits in the +sub repositories. When all subrepositories have been converted, a +mapping file that maps the mercurial subrepository path to a converted +git submodule path must be created. The format for this file is: + +""="" +""="" +... + +The path of this mapping file is then provided with the --subrepo-map +command line option. + +## Example + +Example mercurial repo folder structure (~/mercurial): + src/... + subrepo/subrepo1 + subrepo/subrepo2 + +### Setup +Create an empty new folder where all the converted git modules will be imported: + mkdir ~/imported-gits + cd ~/imported-gits + +### Convert all submodules to git: + mkdir submodule1 + cd submodule1 + git init + hg-fast-export.sh -r ~/mercurial/subrepo1 + cd .. + mkdir submodule2 + cd submodule2 + git init + hg-fast-export.sh -r ~/mercurial/subrepo2 + +### Create mapping file + cd ~/imported-gits + cat > submodule-mappings << EOF + "subrepo/subrepo1"="../submodule1" + "subrepo/subrepo2"="../submodule2" + EOF + +### Convert main repository + cd ~/imported-gits + mkdir git-main-repo + cd git-main-repo + git init + hg-fast-export.sh -r ~/mercurial --subrepo-map=../submodule-mappings + +### Result +The resulting repository will now contain the subrepo/subrepo1 and +subrepo/subrepo1 submodules. The created .gitmodules file will look +like: + + [submodule "subrepo/subrepo1"] + path = subrepo/subrepo1 + url = ../submodule1 + [submodule "subrepo/subrepo2"] + path = subrepo/subrepo2 + url = ../submodule2 diff --git a/README.md b/README.md index 34c6bf0..e6fb20b 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,11 @@ can be modified by any filter. `file_ctx` is the filecontext from the mercurial python library. After all filters have been run, the values are used to add the file to the git commit. +Submodules +---------- +See README-SUBMODULES.md for how to convert subrepositories into git +submodules. + Notes/Limitations ----------------- diff --git a/hg-fast-export.py b/hg-fast-export.py index e53b5dd..ef87ded 100755 --- a/hg-fast-export.py +++ b/hg-fast-export.py @@ -28,6 +28,9 @@ cfg_checkpoint_count=0 # write some progress message every this many file contents written cfg_export_boundary=1000 +subrepo_cache={} +submodule_mappings=None + def gitmode(flags): return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644' @@ -128,6 +131,48 @@ def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}): count=0 max=len(files) for file in files: + if submodule_mappings and ctx.substate and file==".hgsubstate": + # Remove all submodules as we don't detect deleted submodules properly + # in any other way. We will add the ones not deleted back again below. + for module in submodule_mappings.keys(): + wr('D %s' % module) + + # Read .hgsubstate file in order to find the revision of each subrepo + data=ctx.filectx(file).data() + subHashes={} + for line in data.split('\n'): + if line.strip()=="": + continue + cols=line.split(' ') + subHashes[cols[1]]=cols[0] + + gitmodules="" + # Create the .gitmodules file and all submodules + for name in ctx.substate: + gitRepoLocation=submodule_mappings[name] + "/.git" + + # Populate the cache to map mercurial revision to git revision + if not name in subrepo_cache: + subrepo_cache[name]=(load_cache(gitRepoLocation+"/hg2git-mapping"), + load_cache(gitRepoLocation+"/hg2git-marks", + lambda s: int(s)-1)) + + (mapping_cache, marks_cache)=subrepo_cache[name] + if subHashes[name] in mapping_cache: + revnum=mapping_cache[subHashes[name]] + gitSha=marks_cache[int(revnum)] + wr('M 160000 %s %s' % (gitSha, name)) + sys.stderr.write("Adding submodule %s, revision %s->%s\n" + % (name,subHashes[name],gitSha)) + gitmodules+='[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name, name, submodule_mappings[name]) + else: + sys.stderr.write("Warning: Could not find hg revision %s for %s in git %s\n" % (subHashes[name],name,gitRepoLocation)) + + if len(gitmodules): + wr('M 100644 inline .gitmodules') + wr('data %d' % (len(gitmodules)+1)) + wr(gitmodules) + # Skip .hgtags files. They only get us in trouble. if not hgtags and file == ".hgtags": sys.stderr.write('Skip %s\n' % (file)) @@ -443,6 +488,15 @@ def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile, (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors) mapping_cache[revnode.encode('hex_codec')] = str(rev) + if submodule_mappings: + # Make sure that all submodules are registered in the submodule-mappings file + for rev in range(0,max): + ctx=revsymbol(repo,str(rev)) + if ctx.substate: + for key in ctx.substate: + if key not in submodule_mappings: + sys.stderr.write("Error: %s not found in submodule-mappings\n" % (key)) + return 1 c=0 brmap={} @@ -515,6 +569,8 @@ if __name__=='__main__': help="Additional search path for plugins ") parser.add_option("--plugin", action="append", type="string", dest="plugins", help="Add a plugin with the given init string ") + parser.add_option("--subrepo-map", type="string", dest="subrepo_map", + help="Provide a mapping file between the subrepository name and the submodule name") (options,args)=parser.parse_args() @@ -527,6 +583,14 @@ if __name__=='__main__': if options.statusfile==None: bail(parser,'--status') if options.repourl==None: bail(parser,'--repo') + if options.subrepo_map: + if not os.path.exists(options.subrepo_map): + sys.stderr.write('Subrepo mapping file not found %s\n' + % options.subrepo_map) + sys.exit(1) + submodule_mappings=load_mapping('subrepo mappings', + options.subrepo_map,False) + a={} if options.authorfile!=None: a=load_mapping('authors', options.authorfile, options.raw_mappings) -- 2.11.4.GIT