hg-fast-export.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
   4 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   5
   6 from mercurial import node
   7 from hg2git import setup_repo,fixup_user,get_branch,get_changeset
   8 from hg2git import load_cache,save_cache,get_git_sha1,set_default_branch,set_origin_name
   9 from optparse import OptionParser
  10 import re
  11 import sys
  12 import os
  13
  14 if sys.platform == "win32":
  15   # On Windows, sys.stdout is initially opened in text mode, which means that
  16   # when a LF (\n) character is written to sys.stdout, it will be converted
  17   # into CRLF (\r\n).  That makes git blow up, so use this platform-specific
  18   # code to change the mode of sys.stdout to binary.
  19   import msvcrt
  20   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  21
  22 # silly regex to catch Signed-off-by lines in log message
  23 sob_re=re.compile('^Signed-[Oo]ff-[Bb]y: (.+)$')
  24 # insert 'checkpoint' command after this many commits or none at all if 0
  25 cfg_checkpoint_count=0
  26 # write some progress message every this many file contents written
  27 cfg_export_boundary=1000
  28
  29 def gitmode(flags):
  30   return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644'
  31
  32 def wr_no_nl(msg=''):
  33   if msg:
  34     sys.stdout.write(msg)
  35
  36 def wr(msg=''):
  37   wr_no_nl(msg)
  38   sys.stdout.write('\n')
  39   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
  40
  41 def checkpoint(count):
  42   count=count+1
  43   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
  44     sys.stderr.write("Checkpoint after %d commits\n" % count)
  45     wr('checkpoint')
  46     wr()
  47   return count
  48
  49 def revnum_to_revref(rev, old_marks):
  50   """Convert an hg revnum to a git-fast-import rev reference (an SHA1
  51   or a mark)"""
  52   return old_marks.get(rev) or ':%d' % (rev+1)
  53
  54 def file_mismatch(f1,f2):
  55   """See if two revisions of a file are not equal."""
  56   return node.hex(f1)!=node.hex(f2)
  57
  58 def split_dict(dleft,dright,l=[],c=[],r=[],match=file_mismatch):
  59   """Loop over our repository and find all changed and missing files."""
  60   for left in dleft.keys():
  61     right=dright.get(left,None)
  62     if right==None:
  63       # we have the file but our parent hasn't: add to left set
  64       l.append(left)
  65     elif match(dleft[left],right) or gitmode(dleft.flags(left))!=gitmode(dright.flags(left)):
  66       # we have it but checksums mismatch: add to center set
  67       c.append(left)
  68   for right in dright.keys():
  69     left=dleft.get(right,None)
  70     if left==None:
  71       # if parent has file but we don't: add to right set
  72       r.append(right)
  73     # change is already handled when comparing child against parent
  74   return l,c,r
  75
  76 def get_filechanges(repo,revision,parents,mleft):
  77   """Given some repository and revision, find all changed/deleted files."""
  78   l,c,r=[],[],[]
  79   for p in parents:
  80     if p<0: continue
  81     mright=repo.changectx(p).manifest()
  82     l,c,r=split_dict(mleft,mright,l,c,r)
  83   l.sort()
  84   c.sort()
  85   r.sort()
  86   return l,c,r
  87
  88 def get_author(logmessage,committer,authors):
  89   """As git distincts between author and committer of a patch, try to
  90   extract author by detecting Signed-off-by lines.
  91
  92   This walks from the end of the log message towards the top skipping
  93   empty lines. Upon the first non-empty line, it walks all Signed-off-by
  94   lines upwards to find the first one. For that (if found), it extracts
  95   authorship information the usual way (authors table, cleaning, etc.)
  96
  97   If no Signed-off-by line is found, this defaults to the committer.
  98
  99   This may sound stupid (and it somehow is), but in log messages we
 100   accidentially may have lines in the middle starting with
 101   "Signed-off-by: foo" and thus matching our detection regex. Prevent
 102   that."""
 103
 104   loglines=logmessage.split('\n')
 105   i=len(loglines)
 106   # from tail walk to top skipping empty lines
 107   while i>=0:
 108     i-=1
 109     if len(loglines[i].strip())==0: continue
 110     break
 111   if i>=0:
 112     # walk further upwards to find first sob line, store in 'first'
 113     first=None
 114     while i>=0:
 115       m=sob_re.match(loglines[i])
 116       if m==None: break
 117       first=m
 118       i-=1
 119     # if the last non-empty line matches our Signed-Off-by regex: extract username
 120     if first!=None:
 121       r=fixup_user(first.group(1),authors)
 122       return r
 123   return committer
 124
 125 def export_file_contents(ctx,manifest,files,hgtags,encoding=''):
 126   count=0
 127   max=len(files)
 128   for file in files:
 129     # Skip .hgtags files. They only get us in trouble.
 130     if not hgtags and file == ".hgtags":
 131       sys.stderr.write('Skip %s\n' % (file))
 132       continue
 133     d=ctx.filectx(file).data()
 134     if encoding:
 135       filename=file.decode(encoding).encode('utf8')
 136     else:
 137       filename=file
 138     wr('M %s inline %s' % (gitmode(manifest.flags(file)),
 139                            strip_leading_slash(filename)))
 140     wr('data %d' % len(d)) # had some trouble with size()
 141     wr(d)
 142     count+=1
 143     if count%cfg_export_boundary==0:
 144       sys.stderr.write('Exported %d/%d files\n' % (count,max))
 145   if max>cfg_export_boundary:
 146     sys.stderr.write('Exported %d/%d files\n' % (count,max))
 147
 148 def sanitize_name(name,what="branch", mapping={}):
 149   """Sanitize input roughly according to git-check-ref-format(1)"""
 150
 151   # NOTE: Do not update this transform to work around
 152   # incompatibilities on your platform. If you change it and it starts
 153   # modifying names which previously were not touched it will break
 154   # preexisting setups which are doing incremental imports.
 155   #
 156   # Use the -B and -T options to mangle branch and tag names
 157   # instead. If you have a source repository where this is too much
 158   # work to do manually, write a tool that does it for you.
 159
 160   def dot(name):
 161     if not name: return name
 162     if name[0] == '.': return '_'+name[1:]
 163     return name
 164
 165   n=mapping.get(name,name)
 166   p=re.compile('([[ ~^:?\\\\*]|\.\.)')
 167   n=p.sub('_', n)
 168   if n[-1] in ('/', '.'): n=n[:-1]+'_'
 169   n='/'.join(map(dot,n.split('/')))
 170   p=re.compile('_+')
 171   n=p.sub('_', n)
 172
 173   if n!=name:
 174     sys.stderr.write('Warning: sanitized %s [%s] to [%s]\n' % (what,name,n))
 175   return n
 176
 177 def strip_leading_slash(filename):
 178   if filename[0] == '/':
 179     return filename[1:]
 180   return filename
 181
 182 def export_commit(ui,repo,revision,old_marks,max,count,authors,
 183                   branchesmap,sob,brmap,hgtags,encoding='',fn_encoding=''):
 184   def get_branchname(name):
 185     if brmap.has_key(name):
 186       return brmap[name]
 187     n=sanitize_name(name, "branch", branchesmap)
 188     brmap[name]=n
 189     return n
 190
 191   (revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors,encoding)
 192
 193   branch=get_branchname(branch)
 194
 195   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 196
 197   if len(parents)==0 and revision != 0:
 198     wr('reset refs/heads/%s' % branch)
 199
 200   wr('commit refs/heads/%s' % branch)
 201   wr('mark :%d' % (revision+1))
 202   if sob:
 203     wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
 204   wr('committer %s %d %s' % (user,time,timezone))
 205   wr('data %d' % (len(desc)+1)) # wtf?
 206   wr(desc)
 207   wr()
 208
 209   ctx=repo.changectx(str(revision))
 210   man=ctx.manifest()
 211   added,changed,removed,type=[],[],[],''
 212
 213   if len(parents) == 0:
 214     # first revision: feed in full manifest
 215     added=man.keys()
 216     added.sort()
 217     type='full'
 218   else:
 219     wr('from %s' % revnum_to_revref(parents[0], old_marks))
 220     if len(parents) == 1:
 221       # later non-merge revision: feed in changed manifest
 222       # if we have exactly one parent, just take the changes from the
 223       # manifest without expensively comparing checksums
 224       f=repo.status(repo.lookup(parents[0]),revnode)[:3]
 225       added,changed,removed=f[1],f[0],f[2]
 226       type='simple delta'
 227     else: # a merge with two parents
 228       wr('merge %s' % revnum_to_revref(parents[1], old_marks))
 229       # later merge revision: feed in changed manifest
 230       # for many files comparing checksums is expensive so only do it for
 231       # merges where we really need it due to hg's revlog logic
 232       added,changed,removed=get_filechanges(repo,revision,parents,man)
 233       type='thorough delta'
 234
 235   sys.stderr.write('%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n' %
 236       (branch,type,revision+1,max,len(added),len(changed),len(removed)))
 237
 238   if fn_encoding:
 239     removed=[r.decode(fn_encoding).encode('utf8') for r in removed]
 240
 241   removed=[strip_leading_slash(x) for x in removed]
 242
 243   map(lambda r: wr('D %s' % r),removed)
 244   export_file_contents(ctx,man,added,hgtags,fn_encoding)
 245   export_file_contents(ctx,man,changed,hgtags,fn_encoding)
 246   wr()
 247
 248   return checkpoint(count)
 249
 250 def export_note(ui,repo,revision,count,authors,encoding,is_first):
 251   (revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
 252
 253   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 254
 255   wr('commit refs/notes/hg')
 256   wr('committer %s %d %s' % (user,time,timezone))
 257   wr('data 0')
 258   if is_first:
 259     wr('from refs/notes/hg^0')
 260   wr('N inline :%d' % (revision+1))
 261   hg_hash=repo.changectx(str(revision)).hex()
 262   wr('data %d' % (len(hg_hash)))
 263   wr_no_nl(hg_hash)
 264   wr()
 265   return checkpoint(count)
 266
 267   wr('data %d' % (len(desc)+1)) # wtf?
 268   wr(desc)
 269   wr()
 270
 271 def export_tags(ui,repo,old_marks,mapping_cache,count,authors,tagsmap):
 272   l=repo.tagslist()
 273   for tag,node in l:
 274     # Remap the branch name
 275     tag=sanitize_name(tag,"tag",tagsmap)
 276     # ignore latest revision
 277     if tag=='tip': continue
 278     # ignore tags to nodes that are missing (ie, 'in the future')
 279     if node.encode('hex_codec') not in mapping_cache:
 280       sys.stderr.write('Tag %s refers to unseen node %s\n' % (tag, node.encode('hex_codec')))
 281       continue
 282
 283     rev=int(mapping_cache[node.encode('hex_codec')])
 284
 285     ref=revnum_to_revref(rev, old_marks)
 286     if ref==None:
 287       sys.stderr.write('Failed to find reference for creating tag'
 288           ' %s at r%d\n' % (tag,rev))
 289       continue
 290     sys.stderr.write('Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag,rev,ref))
 291     wr('reset refs/tags/%s' % tag)
 292     wr('from %s' % ref)
 293     wr()
 294     count=checkpoint(count)
 295   return count
 296
 297 def load_mapping(name, filename):
 298   cache={}
 299   if not os.path.exists(filename):
 300     sys.stderr.write('Could not open mapping file [%s]\n' % (filename))
 301     return cache
 302   f=open(filename,'r')
 303   l=0
 304   a=0
 305   lre=re.compile('^([^=]+)[ ]*=[ ]*(.+)$')
 306   for line in f.readlines():
 307     l+=1
 308     line=line.strip()
 309     if line=='' or line[0]=='#':
 310       continue
 311     m=lre.match(line)
 312     if m==None:
 313       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
 314       continue
 315     # put key:value in cache, key without ^:
 316     cache[m.group(1).strip()]=m.group(2).strip()
 317     a+=1
 318   f.close()
 319   sys.stderr.write('Loaded %d %s\n' % (a, name))
 320   return cache
 321
 322 def branchtip(repo, heads):
 323   '''return the tipmost branch head in heads'''
 324   tip = heads[-1]
 325   for h in reversed(heads):
 326     if 'close' not in repo.changelog.read(h)[5]:
 327       tip = h
 328       break
 329   return tip
 330
 331 def verify_heads(ui,repo,cache,force,branchesmap):
 332   branches={}
 333   for bn, heads in repo.branchmap().iteritems():
 334     branches[bn] = branchtip(repo, heads)
 335   l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
 336   l.sort()
 337
 338   # get list of hg's branches to verify, don't take all git has
 339   for _,_,b in l:
 340     b=get_branch(b)
 341     sanitized_name=sanitize_name(b,"branch",branchesmap)
 342     sha1=get_git_sha1(sanitized_name)
 343     c=cache.get(sanitized_name)
 344     if sha1!=c:
 345       sys.stderr.write('Error: Branch [%s] modified outside hg-fast-export:'
 346         '\n%s (repo) != %s (cache)\n' % (b,sha1,c))
 347       if not force: return False
 348
 349   # verify that branch has exactly one head
 350   t={}
 351   for h in repo.heads():
 352     (_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
 353     if t.get(branch,False):
 354       sys.stderr.write('Error: repository has at least one unnamed head: hg r%s\n' %
 355           repo.changelog.rev(h))
 356       if not force: return False
 357     t[branch]=True
 358
 359   return True
 360
 361 def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
 362            authors={},branchesmap={},tagsmap={},
 363            sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding=''):
 364   def check_cache(filename, contents):
 365     if len(contents) == 0:
 366       sys.stderr.write('Warning: %s does not contain any data, this will probably make an incremental import fail\n' % filename)
 367
 368   _max=int(m)
 369
 370   old_marks=load_cache(marksfile,lambda s: int(s)-1)
 371   mapping_cache=load_cache(mappingfile)
 372   heads_cache=load_cache(headsfile)
 373   state_cache=load_cache(tipfile)
 374
 375   if len(state_cache) != 0:
 376     for (name, data) in [(marksfile, old_marks),
 377                          (mappingfile, mapping_cache),
 378                          (headsfile, state_cache)]:
 379       check_cache(name, data)
 380
 381   ui,repo=setup_repo(repourl)
 382
 383   if not verify_heads(ui,repo,heads_cache,force,branchesmap):
 384     return 1
 385
 386   try:
 387     tip=repo.changelog.count()
 388   except AttributeError:
 389     tip=len(repo)
 390
 391   min=int(state_cache.get('tip',0))
 392   max=_max
 393   if _max<0 or max>tip:
 394     max=tip
 395
 396   for rev in range(0,max):
 397         (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
 398         mapping_cache[revnode.encode('hex_codec')] = str(rev)
 399
 400
 401   c=0
 402   brmap={}
 403   for rev in range(min,max):
 404     c=export_commit(ui,repo,rev,old_marks,max,c,authors,branchesmap,
 405                     sob,brmap,hgtags,encoding,fn_encoding)
 406   if notes:
 407     for rev in range(min,max):
 408       c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
 409
 410   state_cache['tip']=max
 411   state_cache['repo']=repourl
 412   save_cache(tipfile,state_cache)
 413   save_cache(mappingfile,mapping_cache)
 414
 415   c=export_tags(ui,repo,old_marks,mapping_cache,c,authors,tagsmap)
 416
 417   sys.stderr.write('Issued %d commands\n' % c)
 418
 419   return 0
 420
 421 if __name__=='__main__':
 422   def bail(parser,opt):
 423     sys.stderr.write('Error: No %s option given\n' % opt)
 424     parser.print_help()
 425     sys.exit(2)
 426
 427   parser=OptionParser()
 428
 429   parser.add_option("-m","--max",type="int",dest="max",
 430       help="Maximum hg revision to import")
 431   parser.add_option("--mapping",dest="mappingfile",
 432       help="File to read last run's hg-to-git SHA1 mapping")
 433   parser.add_option("--marks",dest="marksfile",
 434       help="File to read git-fast-import's marks from")
 435   parser.add_option("--heads",dest="headsfile",
 436       help="File to read last run's git heads from")
 437   parser.add_option("--status",dest="statusfile",
 438       help="File to read status from")
 439   parser.add_option("-r","--repo",dest="repourl",
 440       help="URL of repo to import")
 441   parser.add_option("-s",action="store_true",dest="sob",
 442       default=False,help="Enable parsing Signed-off-by lines")
 443   parser.add_option("--hgtags",action="store_true",dest="hgtags",
 444       default=False,help="Enable exporting .hgtags files")
 445   parser.add_option("-A","--authors",dest="authorfile",
 446       help="Read authormap from AUTHORFILE")
 447   parser.add_option("-B","--branches",dest="branchesfile",
 448       help="Read branch map from BRANCHESFILE")
 449   parser.add_option("-T","--tags",dest="tagsfile",
 450       help="Read tags map from TAGSFILE")
 451   parser.add_option("-f","--force",action="store_true",dest="force",
 452       default=False,help="Ignore validation errors by force")
 453   parser.add_option("-M","--default-branch",dest="default_branch",
 454       help="Set the default branch")
 455   parser.add_option("-o","--origin",dest="origin_name",
 456       help="use <name> as namespace to track upstream")
 457   parser.add_option("--hg-hash",action="store_true",dest="notes",
 458       default=False,help="Annotate commits with the hg hash as git notes in the hg namespace")
 459   parser.add_option("-e",dest="encoding",
 460       help="Assume commit and author strings retrieved from Mercurial are encoded in <encoding>")
 461   parser.add_option("--fe",dest="fn_encoding",
 462       help="Assume file names from Mercurial are encoded in <filename_encoding>")
 463
 464   (options,args)=parser.parse_args()
 465
 466   m=-1
 467   if options.max!=None: m=options.max
 468
 469   if options.marksfile==None: bail(parser,'--marks')
 470   if options.mappingfile==None: bail(parser,'--mapping')
 471   if options.headsfile==None: bail(parser,'--heads')
 472   if options.statusfile==None: bail(parser,'--status')
 473   if options.repourl==None: bail(parser,'--repo')
 474
 475   a={}
 476   if options.authorfile!=None:
 477     a=load_mapping('authors', options.authorfile)
 478
 479   b={}
 480   if options.branchesfile!=None:
 481     b=load_mapping('branches', options.branchesfile)
 482
 483   t={}
 484   if options.tagsfile!=None:
 485     t=load_mapping('tags', options.tagsfile)
 486
 487   if options.default_branch!=None:
 488     set_default_branch(options.default_branch)
 489
 490   if options.origin_name!=None:
 491     set_origin_name(options.origin_name)
 492
 493   encoding=''
 494   if options.encoding!=None:
 495     encoding=options.encoding
 496
 497   fn_encoding=encoding
 498   if options.fn_encoding!=None:
 499     fn_encoding=options.fn_encoding
 500
 501   sys.exit(hg2git(options.repourl,m,options.marksfile,options.mappingfile,
 502                   options.headsfile, options.statusfile,
 503                   authors=a,branchesmap=b,tagsmap=t,
 504                   sob=options.sob,force=options.force,hgtags=options.hgtags,
 505                   notes=options.notes,encoding=encoding,fn_encoding=fn_encoding))