hg-fast-export.py

   1 #!/usr/bin/env python2
   2
   3 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
   4 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   5
   6 from mercurial import node
   7 from mercurial.scmutil import revsymbol
   8 from hg2git import setup_repo,fixup_user,get_branch,get_changeset
   9 from hg2git import load_cache,save_cache,get_git_sha1,set_default_branch,set_origin_name
  10 from optparse import OptionParser
  11 import re
  12 import sys
  13 import os
  14 from binascii import hexlify
  15 import pluginloader
  16 PY2 = sys.version_info.major == 2
  17 if PY2:
  18   str = unicode
  19
  20 if PY2 and sys.platform == "win32":
  21   # On Windows, sys.stdout is initially opened in text mode, which means that
  22   # when a LF (\n) character is written to sys.stdout, it will be converted
  23   # into CRLF (\r\n).  That makes git blow up, so use this platform-specific
  24   # code to change the mode of sys.stdout to binary.
  25   import msvcrt
  26   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  27
  28 # silly regex to catch Signed-off-by lines in log message
  29 sob_re=re.compile(b'^Signed-[Oo]ff-[Bb]y: (.+)$')
  30 # insert 'checkpoint' command after this many commits or none at all if 0
  31 cfg_checkpoint_count=0
  32 # write some progress message every this many file contents written
  33 cfg_export_boundary=1000
  34
  35 subrepo_cache={}
  36 submodule_mappings=None
  37
  38 # True if fast export should automatically try to sanitize
  39 # author/branch/tag names.
  40 auto_sanitize = None
  41
  42 stdout_buffer = sys.stdout if PY2 else sys.stdout.buffer
  43 stderr_buffer = sys.stderr if PY2 else sys.stderr.buffer
  44
  45 def gitmode(flags):
  46   return b'l' in flags and b'120000' or b'x' in flags and b'100755' or b'100644'
  47
  48 def wr_no_nl(msg=b''):
  49   assert isinstance(msg, bytes)
  50   if msg:
  51     stdout_buffer.write(msg)
  52
  53 def wr(msg=b''):
  54   wr_no_nl(msg)
  55   stdout_buffer.write(b'\n')
  56   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
  57
  58 def checkpoint(count):
  59   count=count+1
  60   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
  61     stderr_buffer.write(b"Checkpoint after %d commits\n" % count)
  62     wr(b'checkpoint')
  63     wr()
  64   return count
  65
  66 def revnum_to_revref(rev, old_marks):
  67   """Convert an hg revnum to a git-fast-import rev reference (an SHA1
  68   or a mark)"""
  69   return old_marks.get(rev) or b':%d' % (rev+1)
  70
  71 def file_mismatch(f1,f2):
  72   """See if two revisions of a file are not equal."""
  73   return node.hex(f1)!=node.hex(f2)
  74
  75 def split_dict(dleft,dright,l=[],c=[],r=[],match=file_mismatch):
  76   """Loop over our repository and find all changed and missing files."""
  77   for left in dleft.keys():
  78     right=dright.get(left,None)
  79     if right==None:
  80       # we have the file but our parent hasn't: add to left set
  81       l.append(left)
  82     elif match(dleft[left],right) or gitmode(dleft.flags(left))!=gitmode(dright.flags(left)):
  83       # we have it but checksums mismatch: add to center set
  84       c.append(left)
  85   for right in dright.keys():
  86     left=dleft.get(right,None)
  87     if left==None:
  88       # if parent has file but we don't: add to right set
  89       r.append(right)
  90     # change is already handled when comparing child against parent
  91   return l,c,r
  92
  93 def get_filechanges(repo,revision,parents,mleft):
  94   """Given some repository and revision, find all changed/deleted files."""
  95   l,c,r=[],[],[]
  96   for p in parents:
  97     if p<0: continue
  98     mright=revsymbol(repo,b"%d" %p).manifest()
  99     l,c,r=split_dict(mleft,mright,l,c,r)
 100   l.sort()
 101   c.sort()
 102   r.sort()
 103   return l,c,r
 104
 105 def get_author(logmessage,committer,authors):
 106   """As git distincts between author and committer of a patch, try to
 107   extract author by detecting Signed-off-by lines.
 108
 109   This walks from the end of the log message towards the top skipping
 110   empty lines. Upon the first non-empty line, it walks all Signed-off-by
 111   lines upwards to find the first one. For that (if found), it extracts
 112   authorship information the usual way (authors table, cleaning, etc.)
 113
 114   If no Signed-off-by line is found, this defaults to the committer.
 115
 116   This may sound stupid (and it somehow is), but in log messages we
 117   accidentially may have lines in the middle starting with
 118   "Signed-off-by: foo" and thus matching our detection regex. Prevent
 119   that."""
 120
 121   loglines=logmessage.split(b'\n')
 122   i=len(loglines)
 123   # from tail walk to top skipping empty lines
 124   while i>=0:
 125     i-=1
 126     if len(loglines[i].strip())==0: continue
 127     break
 128   if i>=0:
 129     # walk further upwards to find first sob line, store in 'first'
 130     first=None
 131     while i>=0:
 132       m=sob_re.match(loglines[i])
 133       if m==None: break
 134       first=m
 135       i-=1
 136     # if the last non-empty line matches our Signed-Off-by regex: extract username
 137     if first!=None:
 138       r=fixup_user(first.group(1),authors)
 139       return r
 140   return committer
 141
 142 def remove_gitmodules(ctx):
 143   """Removes all submodules of ctx parents"""
 144   # Removing all submoduies coming from all parents is safe, as the submodules
 145   # of the current commit will be re-added below. A possible optimization would
 146   # be to only remove the submodules of the first parent.
 147   for parent_ctx in ctx.parents():
 148     for submodule in parent_ctx.substate.keys():
 149       wr(b'D %s' % submodule)
 150   wr(b'D .gitmodules')
 151
 152 def refresh_git_submodule(name,subrepo_info):
 153   wr(b'M 160000 %s %s' % (subrepo_info[1],name))
 154   stderr_buffer.write(
 155     b"Adding/updating submodule %s, revision %s\n" % (name, subrepo_info[1])
 156   )
 157   return b'[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name, name, subrepo_info[0])
 158
 159 def refresh_hg_submodule(name,subrepo_info):
 160   gitRepoLocation=submodule_mappings[name] + b"/.git"
 161
 162   # Populate the cache to map mercurial revision to git revision
 163   if not name in subrepo_cache:
 164     subrepo_cache[name]=(load_cache(gitRepoLocation+b"/hg2git-mapping"),
 165                          load_cache(gitRepoLocation+b"/hg2git-marks",
 166                                     lambda s: int(s)-1))
 167
 168   (mapping_cache,marks_cache)=subrepo_cache[name]
 169   subrepo_hash=subrepo_info[1]
 170   if subrepo_hash in mapping_cache:
 171     revnum=mapping_cache[subrepo_hash]
 172     gitSha=marks_cache[int(revnum)]
 173     wr(b'M 160000 %s %s' % (gitSha,name))
 174     stderr_buffer.write(
 175       b"Adding/updating submodule %s, revision %s->%s\n"
 176       % (name, subrepo_hash, gitSha)
 177     )
 178     return b'[submodule "%s"]\n\tpath = %s\n\turl = %s\n' % (name,name,
 179       submodule_mappings[name])
 180   else:
 181     stderr_buffer.write(
 182       b"Warning: Could not find hg revision %s for %s in git %s\n"
 183       % (subrepo_hash, name, gitRepoLocation,)
 184     )
 185     return b''
 186
 187 def refresh_gitmodules(ctx):
 188   """Updates list of ctx submodules according to .hgsubstate file"""
 189   remove_gitmodules(ctx)
 190   gitmodules=b""
 191   # Create the .gitmodules file and all submodules
 192   for name,subrepo_info in ctx.substate.items():
 193     if subrepo_info[2]==b'git':
 194       gitmodules+=refresh_git_submodule(name,subrepo_info)
 195     elif submodule_mappings and name in submodule_mappings:
 196       gitmodules+=refresh_hg_submodule(name,subrepo_info)
 197
 198   if len(gitmodules):
 199     wr(b'M 100644 inline .gitmodules')
 200     wr(b'data %d' % (len(gitmodules)+1))
 201     wr(gitmodules)
 202
 203 def export_file_contents(ctx,manifest,files,hgtags,encoding='',plugins={}):
 204   count=0
 205   max=len(files)
 206   is_submodules_refreshed=False
 207   for file in files:
 208     if not is_submodules_refreshed and (file==b'.hgsub' or file==b'.hgsubstate'):
 209       is_submodules_refreshed=True
 210       refresh_gitmodules(ctx)
 211     # Skip .hgtags files. They only get us in trouble.
 212     if not hgtags and file == b".hgtags":
 213       stderr_buffer.write(b'Skip %s\n' % file)
 214       continue
 215     if encoding:
 216       filename=file.decode(encoding).encode('utf8')
 217     else:
 218       filename=file
 219     if b'.git' in filename.split(b'/'): # Even on Windows, the path separator is / here.
 220       stderr_buffer.write(
 221         b'Ignoring file %s which cannot be tracked by git\n' % filename
 222       )
 223       continue
 224     file_ctx=ctx.filectx(file)
 225     d=file_ctx.data()
 226
 227     if plugins and plugins['file_data_filters']:
 228       file_data = {'filename':filename,'file_ctx':file_ctx,'data':d}
 229       for filter in plugins['file_data_filters']:
 230         filter(file_data)
 231       d=file_data['data']
 232       filename=file_data['filename']
 233       file_ctx=file_data['file_ctx']
 234
 235     wr(b'M %s inline %s' % (gitmode(manifest.flags(file)),
 236                            strip_leading_slash(filename)))
 237     wr(b'data %d' % len(d)) # had some trouble with size()
 238     wr(d)
 239     count+=1
 240     if count%cfg_export_boundary==0:
 241       stderr_buffer.write(b'Exported %d/%d files\n' % (count,max))
 242   if max>cfg_export_boundary:
 243     stderr_buffer.write(b'Exported %d/%d files\n' % (count,max))
 244
 245 def sanitize_name(name,what="branch", mapping={}):
 246   """Sanitize input roughly according to git-check-ref-format(1)"""
 247
 248   # NOTE: Do not update this transform to work around
 249   # incompatibilities on your platform. If you change it and it starts
 250   # modifying names which previously were not touched it will break
 251   # preexisting setups which are doing incremental imports.
 252   #
 253   # Fast-export tries to not inflict arbitrary naming policy on the
 254   # user, instead it aims to provide mechanisms allowing the user to
 255   # apply their own policy. Therefore do not add a transform which can
 256   # already be implemented with the -B and -T options to mangle branch
 257   # and tag names. If you have a source repository where this is too
 258   # much work to do manually, write a tool that does it for you.
 259   #
 260
 261   def dot(name):
 262     if not name: return name
 263     if name[0:1] == b'.': return b'_'+name[1:]
 264     return name
 265
 266   if not auto_sanitize:
 267     return mapping.get(name,name)
 268   n=mapping.get(name,name)
 269   p=re.compile(b'([[ ~^:?\\\\*]|\.\.)')
 270   n=p.sub(b'_', n)
 271   if n[-1:] in (b'/', b'.'): n=n[:-1]+b'_'
 272   n=b'/'.join([dot(s) for s in n.split(b'/')])
 273   p=re.compile(b'_+')
 274   n=p.sub(b'_', n)
 275
 276   if n!=name:
 277     stderr_buffer.write(
 278       b'Warning: sanitized %s [%s] to [%s]\n' % (what.encode(), name, n)
 279     )
 280   return n
 281
 282 def strip_leading_slash(filename):
 283   if filename[0:1] == b'/':
 284     return filename[1:]
 285   return filename
 286
 287 def export_commit(ui,repo,revision,old_marks,max,count,authors,
 288                   branchesmap,sob,brmap,hgtags,encoding='',fn_encoding='',
 289                   plugins={}):
 290   def get_branchname(name):
 291     if name in brmap:
 292       return brmap[name]
 293     n=sanitize_name(name, "branch", branchesmap)
 294     brmap[name]=n
 295     return n
 296
 297   (revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors,encoding)
 298   if repo[revnode].hidden():
 299     return count
 300
 301   branch=get_branchname(branch)
 302
 303   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 304   author = get_author(desc,user,authors)
 305
 306   if plugins and plugins['commit_message_filters']:
 307     commit_data = {'branch': branch, 'parents': parents, 'author': author, 'desc': desc}
 308     for filter in plugins['commit_message_filters']:
 309       filter(commit_data)
 310     branch = commit_data['branch']
 311     parents = commit_data['parents']
 312     author = commit_data['author']
 313     desc = commit_data['desc']
 314
 315   if len(parents)==0 and revision != 0:
 316     wr(b'reset refs/heads/%s' % branch)
 317
 318   wr(b'commit refs/heads/%s' % branch)
 319   wr(b'mark :%d' % (revision+1))
 320   if sob:
 321     wr(b'author %s %d %s' % (author,time,timezone))
 322   wr(b'committer %s %d %s' % (user,time,timezone))
 323   wr(b'data %d' % (len(desc)+1)) # wtf?
 324   wr(desc)
 325   wr()
 326
 327   ctx=revsymbol(repo, b"%d" % revision)
 328   man=ctx.manifest()
 329   added,changed,removed,type=[],[],[],''
 330
 331   if len(parents) == 0:
 332     # first revision: feed in full manifest
 333     added=man.keys()
 334     added.sort()
 335     type='full'
 336   else:
 337     wr(b'from %s' % revnum_to_revref(parents[0], old_marks))
 338     if len(parents) == 1:
 339       # later non-merge revision: feed in changed manifest
 340       # if we have exactly one parent, just take the changes from the
 341       # manifest without expensively comparing checksums
 342       f=repo.status(parents[0],revnode)
 343       added,changed,removed=f.added,f.modified,f.removed
 344       type='simple delta'
 345     else: # a merge with two parents
 346       wr(b'merge %s' % revnum_to_revref(parents[1], old_marks))
 347       # later merge revision: feed in changed manifest
 348       # for many files comparing checksums is expensive so only do it for
 349       # merges where we really need it due to hg's revlog logic
 350       added,changed,removed=get_filechanges(repo,revision,parents,man)
 351       type='thorough delta'
 352
 353   stderr_buffer.write(
 354     b'%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n'
 355     % (branch, type.encode(), revision + 1, max, len(added), len(changed), len(removed))
 356   )
 357
 358   for filename in removed:
 359     if fn_encoding:
 360       filename=filename.decode(fn_encoding).encode('utf8')
 361     filename=strip_leading_slash(filename)
 362     if filename==b'.hgsub':
 363       remove_gitmodules(ctx)
 364     wr(b'D %s' % filename)
 365
 366   export_file_contents(ctx,man,added,hgtags,fn_encoding,plugins)
 367   export_file_contents(ctx,man,changed,hgtags,fn_encoding,plugins)
 368   wr()
 369
 370   return checkpoint(count)
 371
 372 def export_note(ui,repo,revision,count,authors,encoding,is_first):
 373   (revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
 374   if repo[revnode].hidden():
 375     return count
 376
 377   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 378
 379   wr(b'commit refs/notes/hg')
 380   wr(b'committer %s %d %s' % (user,time,timezone))
 381   wr(b'data 0')
 382   if is_first:
 383     wr(b'from refs/notes/hg^0')
 384   wr(b'N inline :%d' % (revision+1))
 385   hg_hash=revsymbol(repo,b"%d" % revision).hex()
 386   wr(b'data %d' % (len(hg_hash)))
 387   wr_no_nl(hg_hash)
 388   wr()
 389   return checkpoint(count)
 390
 391 def export_tags(ui,repo,old_marks,mapping_cache,count,authors,tagsmap):
 392   l=repo.tagslist()
 393   for tag,node in l:
 394     # Remap the branch name
 395     tag=sanitize_name(tag,"tag",tagsmap)
 396     # ignore latest revision
 397     if tag==b'tip': continue
 398     # ignore tags to nodes that are missing (ie, 'in the future')
 399     if hexlify(node) not in mapping_cache:
 400       stderr_buffer.write(b'Tag %s refers to unseen node %s\n' % (tag, hexlify(node)))
 401       continue
 402
 403     rev=int(mapping_cache[hexlify(node)])
 404
 405     ref=revnum_to_revref(rev, old_marks)
 406     if ref==None:
 407       stderr_buffer.write(
 408         b'Failed to find reference for creating tag %s at r%d\n' % (tag, rev)
 409       )
 410       continue
 411     stderr_buffer.write(b'Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag, rev, ref))
 412     wr(b'reset refs/tags/%s' % tag)
 413     wr(b'from %s' % ref)
 414     wr()
 415     count=checkpoint(count)
 416   return count
 417
 418 def load_mapping(name, filename, mapping_is_raw):
 419   raw_regexp=re.compile(b'^([^=]+)[ ]*=[ ]*(.+)$')
 420   string_regexp=b'"(((\\.)|(\\")|[^"])*)"'
 421   quoted_regexp=re.compile(b'^'+string_regexp+b'[ ]*=[ ]*'+string_regexp+b'$')
 422
 423   def parse_raw_line(line):
 424     m=raw_regexp.match(line)
 425     if m==None:
 426       return None
 427     return (m.group(1).strip(), m.group(2).strip())
 428
 429   def parse_quoted_line(line):
 430     m=quoted_regexp.match(line)
 431     if m==None:
 432       return None
 433     return (m.group(1).decode('unicode_escape').encode('utf8'),
 434             m.group(5).decode('unicode_escape').encode('utf8'))
 435
 436   cache={}
 437   if not os.path.exists(filename):
 438     sys.stderr.write('Could not open mapping file [%s]\n' % (filename))
 439     return cache
 440   f=open(filename,'rb')
 441   l=0
 442   a=0
 443   for line in f.readlines():
 444     l+=1
 445     line=line.strip()
 446     if l==1 and line[0:1]==b'#' and line==b'# quoted-escaped-strings':
 447       continue
 448     elif line==b'' or line[0:1]==b'#':
 449       continue
 450     m=parse_raw_line(line) if mapping_is_raw else parse_quoted_line(line)
 451     if m==None:
 452       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
 453       continue
 454     # put key:value in cache, key without ^:
 455     cache[m[0]]=m[1]
 456     a+=1
 457   f.close()
 458   sys.stderr.write('Loaded %d %s\n' % (a, name))
 459   return cache
 460
 461 def branchtip(repo, heads):
 462   '''return the tipmost branch head in heads'''
 463   tip = heads[-1]
 464   for h in reversed(heads):
 465     if 'close' not in repo.changelog.read(h)[5]:
 466       tip = h
 467       break
 468   return tip
 469
 470 def verify_heads(ui,repo,cache,force,branchesmap):
 471   branches={}
 472   for bn, heads in repo.branchmap().items():
 473     branches[bn] = branchtip(repo, heads)
 474   l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
 475   l.sort()
 476
 477   # get list of hg's branches to verify, don't take all git has
 478   for _,_,b in l:
 479     b=get_branch(b)
 480     sanitized_name=sanitize_name(b,"branch",branchesmap)
 481     sha1=get_git_sha1(sanitized_name)
 482     c=cache.get(sanitized_name)
 483     if sha1!=c:
 484       stderr_buffer.write(
 485         b'Error: Branch [%s] modified outside hg-fast-export:'
 486         b'\n%s (repo) != %s (cache)\n' % (b, b'<None>' if sha1 is None else sha1, c)
 487       )
 488       if not force: return False
 489
 490   # verify that branch has exactly one head
 491   t={}
 492   for h in repo.filtered(b'visible').heads():
 493     (_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
 494     if t.get(branch,False):
 495       stderr_buffer.write(
 496         b'Error: repository has at least one unnamed head: hg r%d\n'
 497         % repo.changelog.rev(h)
 498       )
 499       if not force: return False
 500     t[branch]=True
 501
 502   return True
 503
 504 def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
 505            authors={},branchesmap={},tagsmap={},
 506            sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding='',
 507            plugins={}):
 508   def check_cache(filename, contents):
 509     if len(contents) == 0:
 510       sys.stderr.write('Warning: %s does not contain any data, this will probably make an incremental import fail\n' % filename)
 511
 512   _max=int(m)
 513
 514   old_marks=load_cache(marksfile,lambda s: int(s)-1)
 515   mapping_cache=load_cache(mappingfile)
 516   heads_cache=load_cache(headsfile)
 517   state_cache=load_cache(tipfile)
 518
 519   if len(state_cache) != 0:
 520     for (name, data) in [(marksfile, old_marks),
 521                          (mappingfile, mapping_cache),
 522                          (headsfile, state_cache)]:
 523       check_cache(name, data)
 524
 525   ui,repo=setup_repo(repourl)
 526
 527   if not verify_heads(ui,repo,heads_cache,force,branchesmap):
 528     return 1
 529
 530   try:
 531     tip=repo.changelog.count()
 532   except AttributeError:
 533     tip=len(repo)
 534
 535   min=int(state_cache.get('tip',0))
 536   max=_max
 537   if _max<0 or max>tip:
 538     max=tip
 539
 540   for rev in range(0,max):
 541     (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
 542     if repo[revnode].hidden():
 543       continue
 544     mapping_cache[hexlify(revnode)] = b"%d" % rev
 545
 546   if submodule_mappings:
 547     # Make sure that all submodules are registered in the submodule-mappings file
 548     for rev in range(0,max):
 549       ctx=revsymbol(repo,b"%d" % rev)
 550       if ctx.hidden():
 551         continue
 552       if ctx.substate:
 553         for key in ctx.substate:
 554           if key not in submodule_mappings:
 555             sys.stderr.write("Error: %s not found in submodule-mappings\n" % (key))
 556             return 1
 557
 558   c=0
 559   brmap={}
 560   for rev in range(min,max):
 561     c=export_commit(ui,repo,rev,old_marks,max,c,authors,branchesmap,
 562                     sob,brmap,hgtags,encoding,fn_encoding,
 563                     plugins)
 564   if notes:
 565     for rev in range(min,max):
 566       c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
 567
 568   state_cache['tip']=max
 569   state_cache['repo']=repourl
 570   save_cache(tipfile,state_cache)
 571   save_cache(mappingfile,mapping_cache)
 572
 573   c=export_tags(ui,repo,old_marks,mapping_cache,c,authors,tagsmap)
 574
 575   sys.stderr.write('Issued %d commands\n' % c)
 576
 577   return 0
 578
 579 if __name__=='__main__':
 580   def bail(parser,opt):
 581     sys.stderr.write('Error: No %s option given\n' % opt)
 582     parser.print_help()
 583     sys.exit(2)
 584
 585   parser=OptionParser()
 586
 587   parser.add_option("-n", "--no-auto-sanitize",action="store_false",
 588       dest="auto_sanitize",default=True,
 589       help="Do not perform built-in (broken in many cases) sanitizing of names")
 590   parser.add_option("-m","--max",type="int",dest="max",
 591       help="Maximum hg revision to import")
 592   parser.add_option("--mapping",dest="mappingfile",
 593       help="File to read last run's hg-to-git SHA1 mapping")
 594   parser.add_option("--marks",dest="marksfile",
 595       help="File to read git-fast-import's marks from")
 596   parser.add_option("--heads",dest="headsfile",
 597       help="File to read last run's git heads from")
 598   parser.add_option("--status",dest="statusfile",
 599       help="File to read status from")
 600   parser.add_option("-r","--repo",dest="repourl",
 601       help="URL of repo to import")
 602   parser.add_option("-s",action="store_true",dest="sob",
 603       default=False,help="Enable parsing Signed-off-by lines")
 604   parser.add_option("--hgtags",action="store_true",dest="hgtags",
 605       default=False,help="Enable exporting .hgtags files")
 606   parser.add_option("-A","--authors",dest="authorfile",
 607       help="Read authormap from AUTHORFILE")
 608   parser.add_option("-B","--branches",dest="branchesfile",
 609       help="Read branch map from BRANCHESFILE")
 610   parser.add_option("-T","--tags",dest="tagsfile",
 611       help="Read tags map from TAGSFILE")
 612   parser.add_option("-f","--force",action="store_true",dest="force",
 613       default=False,help="Ignore validation errors by force")
 614   parser.add_option("-M","--default-branch",dest="default_branch",
 615       help="Set the default branch")
 616   parser.add_option("-o","--origin",dest="origin_name",
 617       help="use <name> as namespace to track upstream")
 618   parser.add_option("--hg-hash",action="store_true",dest="notes",
 619       default=False,help="Annotate commits with the hg hash as git notes in the hg namespace")
 620   parser.add_option("-e",dest="encoding",
 621       help="Assume commit and author strings retrieved from Mercurial are encoded in <encoding>")
 622   parser.add_option("--fe",dest="fn_encoding",
 623       help="Assume file names from Mercurial are encoded in <filename_encoding>")
 624   parser.add_option("--mappings-are-raw",dest="raw_mappings", default=False,
 625       help="Assume mappings are raw <key>=<value> lines")
 626   parser.add_option("--filter-contents",dest="filter_contents",
 627       help="Pipe contents of each exported file through FILTER_CONTENTS <file-path> <hg-hash> <is-binary>")
 628   parser.add_option("--plugin-path", type="string", dest="pluginpath",
 629       help="Additional search path for plugins ")
 630   parser.add_option("--plugin", action="append", type="string", dest="plugins",
 631       help="Add a plugin with the given init string <name=init>")
 632   parser.add_option("--subrepo-map", type="string", dest="subrepo_map",
 633       help="Provide a mapping file between the subrepository name and the submodule name")
 634
 635   (options,args)=parser.parse_args()
 636
 637   m=-1
 638   auto_sanitize = options.auto_sanitize
 639   if options.max!=None: m=options.max
 640
 641   if options.marksfile==None: bail(parser,'--marks')
 642   if options.mappingfile==None: bail(parser,'--mapping')
 643   if options.headsfile==None: bail(parser,'--heads')
 644   if options.statusfile==None: bail(parser,'--status')
 645   if options.repourl==None: bail(parser,'--repo')
 646
 647   if options.subrepo_map:
 648       if not os.path.exists(options.subrepo_map):
 649         sys.stderr.write('Subrepo mapping file not found %s\n'
 650                          % options.subrepo_map)
 651         sys.exit(1)
 652       submodule_mappings=load_mapping('subrepo mappings',
 653                                       options.subrepo_map,False)
 654
 655   a={}
 656   if options.authorfile!=None:
 657     a=load_mapping('authors', options.authorfile, options.raw_mappings)
 658
 659   b={}
 660   if options.branchesfile!=None:
 661     b=load_mapping('branches', options.branchesfile, options.raw_mappings)
 662
 663   t={}
 664   if options.tagsfile!=None:
 665     t=load_mapping('tags', options.tagsfile, options.raw_mappings)
 666
 667   if options.default_branch!=None:
 668     set_default_branch(options.default_branch)
 669
 670   if options.origin_name!=None:
 671     set_origin_name(options.origin_name)
 672
 673   encoding=''
 674   if options.encoding!=None:
 675     encoding=options.encoding
 676
 677   fn_encoding=encoding
 678   if options.fn_encoding!=None:
 679     fn_encoding=options.fn_encoding
 680
 681   plugins=[]
 682   if options.plugins!=None:
 683     plugins+=options.plugins
 684
 685   if options.filter_contents!=None:
 686     plugins+=['shell_filter_file_contents='+options.filter_contents]
 687
 688   plugins_dict={}
 689   plugins_dict['commit_message_filters']=[]
 690   plugins_dict['file_data_filters']=[]
 691
 692   if plugins and options.pluginpath:
 693     sys.stderr.write('Using additional plugin path: ' + options.pluginpath + '\n')
 694
 695   for plugin in plugins:
 696     split = plugin.split('=')
 697     name, opts = split[0], '='.join(split[1:])
 698     i = pluginloader.get_plugin(name,options.pluginpath)
 699     sys.stderr.write('Loaded plugin ' + i['name'] + ' from path: ' + i['path'] +' with opts: ' + opts + '\n')
 700     plugin = pluginloader.load_plugin(i).build_filter(opts)
 701     if hasattr(plugin,'file_data_filter') and callable(plugin.file_data_filter):
 702       plugins_dict['file_data_filters'].append(plugin.file_data_filter)
 703     if hasattr(plugin, 'commit_message_filter') and callable(plugin.commit_message_filter):
 704       plugins_dict['commit_message_filters'].append(plugin.commit_message_filter)
 705
 706   sys.exit(hg2git(options.repourl,m,options.marksfile,options.mappingfile,
 707                   options.headsfile, options.statusfile,
 708                   authors=a,branchesmap=b,tagsmap=t,
 709                   sob=options.sob,force=options.force,hgtags=options.hgtags,
 710                   notes=options.notes,encoding=encoding,fn_encoding=fn_encoding,
 711                   plugins=plugins_dict))