hg-fast-export.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2007, 2008 Rocco Rutte <pdmef@gmx.net> and others.
   4 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   5
   6 from mercurial import node
   7 from hg2git import setup_repo,fixup_user,get_branch,get_changeset
   8 from hg2git import load_cache,save_cache,get_git_sha1,set_default_branch,set_origin_name
   9 from optparse import OptionParser
  10 import re
  11 import sys
  12 import os
  13
  14 if sys.platform == "win32":
  15   # On Windows, sys.stdout is initially opened in text mode, which means that
  16   # when a LF (\n) character is written to sys.stdout, it will be converted
  17   # into CRLF (\r\n).  That makes git blow up, so use this platform-specific
  18   # code to change the mode of sys.stdout to binary.
  19   import msvcrt
  20   msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
  21
  22 # silly regex to catch Signed-off-by lines in log message
  23 sob_re=re.compile('^Signed-[Oo]ff-[Bb]y: (.+)$')
  24 # insert 'checkpoint' command after this many commits or none at all if 0
  25 cfg_checkpoint_count=0
  26 # write some progress message every this many file contents written
  27 cfg_export_boundary=1000
  28
  29 def gitmode(flags):
  30   return 'l' in flags and '120000' or 'x' in flags and '100755' or '100644'
  31
  32 def wr_no_nl(msg=''):
  33   if msg:
  34     sys.stdout.write(msg)
  35
  36 def wr(msg=''):
  37   wr_no_nl(msg)
  38   sys.stdout.write('\n')
  39   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
  40
  41 def checkpoint(count):
  42   count=count+1
  43   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
  44     sys.stderr.write("Checkpoint after %d commits\n" % count)
  45     wr('checkpoint')
  46     wr()
  47   return count
  48
  49 def revnum_to_revref(rev, old_marks):
  50   """Convert an hg revnum to a git-fast-import rev reference (an SHA1
  51   or a mark)"""
  52   return old_marks.get(rev) or ':%d' % (rev+1)
  53
  54 def file_mismatch(f1,f2):
  55   """See if two revisions of a file are not equal."""
  56   return node.hex(f1)!=node.hex(f2)
  57
  58 def split_dict(dleft,dright,l=[],c=[],r=[],match=file_mismatch):
  59   """Loop over our repository and find all changed and missing files."""
  60   for left in dleft.keys():
  61     right=dright.get(left,None)
  62     if right==None:
  63       # we have the file but our parent hasn't: add to left set
  64       l.append(left)
  65     elif match(dleft[left],right) or gitmode(dleft.flags(left))!=gitmode(dright.flags(left)):
  66       # we have it but checksums mismatch: add to center set
  67       c.append(left)
  68   for right in dright.keys():
  69     left=dleft.get(right,None)
  70     if left==None:
  71       # if parent has file but we don't: add to right set
  72       r.append(right)
  73     # change is already handled when comparing child against parent
  74   return l,c,r
  75
  76 def get_filechanges(repo,revision,parents,mleft):
  77   """Given some repository and revision, find all changed/deleted files."""
  78   l,c,r=[],[],[]
  79   for p in parents:
  80     if p<0: continue
  81     mright=repo.changectx(p).manifest()
  82     l,c,r=split_dict(mleft,mright,l,c,r)
  83   l.sort()
  84   c.sort()
  85   r.sort()
  86   return l,c,r
  87
  88 def get_author(logmessage,committer,authors):
  89   """As git distincts between author and committer of a patch, try to
  90   extract author by detecting Signed-off-by lines.
  91
  92   This walks from the end of the log message towards the top skipping
  93   empty lines. Upon the first non-empty line, it walks all Signed-off-by
  94   lines upwards to find the first one. For that (if found), it extracts
  95   authorship information the usual way (authors table, cleaning, etc.)
  96
  97   If no Signed-off-by line is found, this defaults to the committer.
  98
  99   This may sound stupid (and it somehow is), but in log messages we
 100   accidentially may have lines in the middle starting with
 101   "Signed-off-by: foo" and thus matching our detection regex. Prevent
 102   that."""
 103
 104   loglines=logmessage.split('\n')
 105   i=len(loglines)
 106   # from tail walk to top skipping empty lines
 107   while i>=0:
 108     i-=1
 109     if len(loglines[i].strip())==0: continue
 110     break
 111   if i>=0:
 112     # walk further upwards to find first sob line, store in 'first'
 113     first=None
 114     while i>=0:
 115       m=sob_re.match(loglines[i])
 116       if m==None: break
 117       first=m
 118       i-=1
 119     # if the last non-empty line matches our Signed-Off-by regex: extract username
 120     if first!=None:
 121       r=fixup_user(first.group(1),authors)
 122       return r
 123   return committer
 124
 125 def export_file_contents(ctx,manifest,files,hgtags,encoding=''):
 126   count=0
 127   max=len(files)
 128   for file in files:
 129     # Skip .hgtags files. They only get us in trouble.
 130     if not hgtags and file == ".hgtags":
 131       sys.stderr.write('Skip %s\n' % (file))
 132       continue
 133     d=ctx.filectx(file).data()
 134     if encoding:
 135       filename=file.decode(encoding).encode('utf8')
 136     else:
 137       filename=file
 138     wr('M %s inline %s' % (gitmode(manifest.flags(file)),
 139                            strip_leading_slash(filename)))
 140     wr('data %d' % len(d)) # had some trouble with size()
 141     wr(d)
 142     count+=1
 143     if count%cfg_export_boundary==0:
 144       sys.stderr.write('Exported %d/%d files\n' % (count,max))
 145   if max>cfg_export_boundary:
 146     sys.stderr.write('Exported %d/%d files\n' % (count,max))
 147
 148 def sanitize_name(name,what="branch", mapping={}):
 149   """Sanitize input roughly according to git-check-ref-format(1)"""
 150
 151   # NOTE: Do not update this transform to work around
 152   # incompatibilities on your platform. If you change it and it starts
 153   # modifying names which previously were not touched it will break
 154   # preexisting setups which are doing incremental imports.
 155   #
 156   # Use the -B and -T options to mangle branch and tag names
 157   # instead. If you have a source repository where this is too much
 158   # work to do manually, write a tool that does it for you.
 159
 160   def dot(name):
 161     if name[0] == '.': return '_'+name[1:]
 162     return name
 163
 164   n=mapping.get(name,name)
 165   p=re.compile('([[ ~^:?\\\\*]|\.\.)')
 166   n=p.sub('_', n)
 167   if n[-1] in ('/', '.'): n=n[:-1]+'_'
 168   n='/'.join(map(dot,n.split('/')))
 169   p=re.compile('_+')
 170   n=p.sub('_', n)
 171
 172   if n!=name:
 173     sys.stderr.write('Warning: sanitized %s [%s] to [%s]\n' % (what,name,n))
 174   return n
 175
 176 def strip_leading_slash(filename):
 177   if filename[0] == '/':
 178     return filename[1:]
 179   return filename
 180
 181 def export_commit(ui,repo,revision,old_marks,max,count,authors,
 182                   branchesmap,sob,brmap,hgtags,encoding='',fn_encoding=''):
 183   def get_branchname(name):
 184     if brmap.has_key(name):
 185       return brmap[name]
 186     n=sanitize_name(name, "branch", branchesmap)
 187     brmap[name]=n
 188     return n
 189
 190   (revnode,_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors,encoding)
 191
 192   branch=get_branchname(branch)
 193
 194   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 195
 196   if len(parents)==0 and revision != 0:
 197     wr('reset refs/heads/%s' % branch)
 198
 199   wr('commit refs/heads/%s' % branch)
 200   wr('mark :%d' % (revision+1))
 201   if sob:
 202     wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
 203   wr('committer %s %d %s' % (user,time,timezone))
 204   wr('data %d' % (len(desc)+1)) # wtf?
 205   wr(desc)
 206   wr()
 207
 208   ctx=repo.changectx(str(revision))
 209   man=ctx.manifest()
 210   added,changed,removed,type=[],[],[],''
 211
 212   if len(parents) == 0:
 213     # first revision: feed in full manifest
 214     added=man.keys()
 215     added.sort()
 216     type='full'
 217   else:
 218     wr('from %s' % revnum_to_revref(parents[0], old_marks))
 219     if len(parents) == 1:
 220       # later non-merge revision: feed in changed manifest
 221       # if we have exactly one parent, just take the changes from the
 222       # manifest without expensively comparing checksums
 223       f=repo.status(repo.lookup(parents[0]),revnode)[:3]
 224       added,changed,removed=f[1],f[0],f[2]
 225       type='simple delta'
 226     else: # a merge with two parents
 227       wr('merge %s' % revnum_to_revref(parents[1], old_marks))
 228       # later merge revision: feed in changed manifest
 229       # for many files comparing checksums is expensive so only do it for
 230       # merges where we really need it due to hg's revlog logic
 231       added,changed,removed=get_filechanges(repo,revision,parents,man)
 232       type='thorough delta'
 233
 234   sys.stderr.write('%s: Exporting %s revision %d/%d with %d/%d/%d added/changed/removed files\n' %
 235       (branch,type,revision+1,max,len(added),len(changed),len(removed)))
 236
 237   if fn_encoding:
 238     removed=[r.decode(fn_encoding).encode('utf8') for r in removed]
 239
 240   removed=[strip_leading_slash(x) for x in removed]
 241
 242   map(lambda r: wr('D %s' % r),removed)
 243   export_file_contents(ctx,man,added,hgtags,fn_encoding)
 244   export_file_contents(ctx,man,changed,hgtags,fn_encoding)
 245   wr()
 246
 247   return checkpoint(count)
 248
 249 def export_note(ui,repo,revision,count,authors,encoding,is_first):
 250   (revnode,_,user,(time,timezone),_,_,_,_)=get_changeset(ui,repo,revision,authors,encoding)
 251
 252   parents = [p for p in repo.changelog.parentrevs(revision) if p >= 0]
 253
 254   wr('commit refs/notes/hg')
 255   wr('committer %s %d %s' % (user,time,timezone))
 256   wr('data 0')
 257   if is_first:
 258     wr('from refs/notes/hg^0')
 259   wr('N inline :%d' % (revision+1))
 260   hg_hash=repo.changectx(str(revision)).hex()
 261   wr('data %d' % (len(hg_hash)))
 262   wr_no_nl(hg_hash)
 263   wr()
 264   return checkpoint(count)
 265
 266   wr('data %d' % (len(desc)+1)) # wtf?
 267   wr(desc)
 268   wr()
 269
 270 def export_tags(ui,repo,old_marks,mapping_cache,count,authors,tagsmap):
 271   l=repo.tagslist()
 272   for tag,node in l:
 273     # Remap the branch name
 274     tag=sanitize_name(tag,"tag",tagsmap)
 275     # ignore latest revision
 276     if tag=='tip': continue
 277     # ignore tags to nodes that are missing (ie, 'in the future')
 278     if node.encode('hex_codec') not in mapping_cache:
 279       sys.stderr.write('Tag %s refers to unseen node %s\n' % (tag, node.encode('hex_codec')))
 280       continue
 281
 282     rev=int(mapping_cache[node.encode('hex_codec')])
 283
 284     ref=revnum_to_revref(rev, old_marks)
 285     if ref==None:
 286       sys.stderr.write('Failed to find reference for creating tag'
 287           ' %s at r%d\n' % (tag,rev))
 288       continue
 289     sys.stderr.write('Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag,rev,ref))
 290     wr('reset refs/tags/%s' % tag)
 291     wr('from %s' % ref)
 292     wr()
 293     count=checkpoint(count)
 294   return count
 295
 296 def load_mapping(name, filename):
 297   cache={}
 298   if not os.path.exists(filename):
 299     sys.stderr.write('Could not open mapping file [%s]\n' % (filename))
 300     return cache
 301   f=open(filename,'r')
 302   l=0
 303   a=0
 304   lre=re.compile('^([^=]+)[ ]*=[ ]*(.+)$')
 305   for line in f.readlines():
 306     l+=1
 307     line=line.strip()
 308     if line=='' or line[0]=='#':
 309       continue
 310     m=lre.match(line)
 311     if m==None:
 312       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
 313       continue
 314     # put key:value in cache, key without ^:
 315     cache[m.group(1).strip()]=m.group(2).strip()
 316     a+=1
 317   f.close()
 318   sys.stderr.write('Loaded %d %s\n' % (a, name))
 319   return cache
 320
 321 def branchtip(repo, heads):
 322   '''return the tipmost branch head in heads'''
 323   tip = heads[-1]
 324   for h in reversed(heads):
 325     if 'close' not in repo.changelog.read(h)[5]:
 326       tip = h
 327       break
 328   return tip
 329
 330 def verify_heads(ui,repo,cache,force,branchesmap):
 331   branches={}
 332   for bn, heads in repo.branchmap().iteritems():
 333     branches[bn] = branchtip(repo, heads)
 334   l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
 335   l.sort()
 336
 337   # get list of hg's branches to verify, don't take all git has
 338   for _,_,b in l:
 339     b=get_branch(b)
 340     sha1=get_git_sha1(sanitize_name(b,"branch",branchesmap))
 341     c=cache.get(b)
 342     if sha1!=c:
 343       sys.stderr.write('Error: Branch [%s] modified outside hg-fast-export:'
 344         '\n%s (repo) != %s (cache)\n' % (b,sha1,c))
 345       if not force: return False
 346
 347   # verify that branch has exactly one head
 348   t={}
 349   for h in repo.heads():
 350     (_,_,_,_,_,_,branch,_)=get_changeset(ui,repo,h)
 351     if t.get(branch,False):
 352       sys.stderr.write('Error: repository has at least one unnamed head: hg r%s\n' %
 353           repo.changelog.rev(h))
 354       if not force: return False
 355     t[branch]=True
 356
 357   return True
 358
 359 def hg2git(repourl,m,marksfile,mappingfile,headsfile,tipfile,
 360            authors={},branchesmap={},tagsmap={},
 361            sob=False,force=False,hgtags=False,notes=False,encoding='',fn_encoding=''):
 362   def check_cache(filename, contents):
 363     if len(contents) == 0:
 364       sys.stderr.write('Warning: %s does not contain any data, this will probably make an incremental import fail\n' % filename)
 365
 366   _max=int(m)
 367
 368   old_marks=load_cache(marksfile,lambda s: int(s)-1)
 369   mapping_cache=load_cache(mappingfile)
 370   heads_cache=load_cache(headsfile)
 371   state_cache=load_cache(tipfile)
 372
 373   if len(state_cache) != 0:
 374     for (name, data) in [(marksfile, old_marks),
 375                          (mappingfile, mapping_cache),
 376                          (headsfile, state_cache)]:
 377       check_cache(name, data)
 378
 379   ui,repo=setup_repo(repourl)
 380
 381   if not verify_heads(ui,repo,heads_cache,force,branchesmap):
 382     return 1
 383
 384   try:
 385     tip=repo.changelog.count()
 386   except AttributeError:
 387     tip=len(repo)
 388
 389   min=int(state_cache.get('tip',0))
 390   max=_max
 391   if _max<0 or max>tip:
 392     max=tip
 393
 394   for rev in range(0,max):
 395         (revnode,_,_,_,_,_,_,_)=get_changeset(ui,repo,rev,authors)
 396         mapping_cache[revnode.encode('hex_codec')] = str(rev)
 397
 398
 399   c=0
 400   brmap={}
 401   for rev in range(min,max):
 402     c=export_commit(ui,repo,rev,old_marks,max,c,authors,branchesmap,
 403                     sob,brmap,hgtags,encoding,fn_encoding)
 404   if notes:
 405     for rev in range(min,max):
 406       c=export_note(ui,repo,rev,c,authors, encoding, rev == min and min != 0)
 407
 408   state_cache['tip']=max
 409   state_cache['repo']=repourl
 410   save_cache(tipfile,state_cache)
 411   save_cache(mappingfile,mapping_cache)
 412
 413   c=export_tags(ui,repo,old_marks,mapping_cache,c,authors,tagsmap)
 414
 415   sys.stderr.write('Issued %d commands\n' % c)
 416
 417   return 0
 418
 419 if __name__=='__main__':
 420   def bail(parser,opt):
 421     sys.stderr.write('Error: No %s option given\n' % opt)
 422     parser.print_help()
 423     sys.exit(2)
 424
 425   parser=OptionParser()
 426
 427   parser.add_option("-m","--max",type="int",dest="max",
 428       help="Maximum hg revision to import")
 429   parser.add_option("--mapping",dest="mappingfile",
 430       help="File to read last run's hg-to-git SHA1 mapping")
 431   parser.add_option("--marks",dest="marksfile",
 432       help="File to read git-fast-import's marks from")
 433   parser.add_option("--heads",dest="headsfile",
 434       help="File to read last run's git heads from")
 435   parser.add_option("--status",dest="statusfile",
 436       help="File to read status from")
 437   parser.add_option("-r","--repo",dest="repourl",
 438       help="URL of repo to import")
 439   parser.add_option("-s",action="store_true",dest="sob",
 440       default=False,help="Enable parsing Signed-off-by lines")
 441   parser.add_option("--hgtags",action="store_true",dest="hgtags",
 442       default=False,help="Enable exporting .hgtags files")
 443   parser.add_option("-A","--authors",dest="authorfile",
 444       help="Read authormap from AUTHORFILE")
 445   parser.add_option("-B","--branches",dest="branchesfile",
 446       help="Read branch map from BRANCHESFILE")
 447   parser.add_option("-T","--tags",dest="tagsfile",
 448       help="Read tags map from TAGSFILE")
 449   parser.add_option("-f","--force",action="store_true",dest="force",
 450       default=False,help="Ignore validation errors by force")
 451   parser.add_option("-M","--default-branch",dest="default_branch",
 452       help="Set the default branch")
 453   parser.add_option("-o","--origin",dest="origin_name",
 454       help="use <name> as namespace to track upstream")
 455   parser.add_option("--hg-hash",action="store_true",dest="notes",
 456       default=False,help="Annotate commits with the hg hash as git notes in the hg namespace")
 457   parser.add_option("-e",dest="encoding",
 458       help="Assume commit and author strings retrieved from Mercurial are encoded in <encoding>")
 459   parser.add_option("--fe",dest="fn_encoding",
 460       help="Assume file names from Mercurial are encoded in <filename_encoding>")
 461
 462   (options,args)=parser.parse_args()
 463
 464   m=-1
 465   if options.max!=None: m=options.max
 466
 467   if options.marksfile==None: bail(parser,'--marks')
 468   if options.mappingfile==None: bail(parser,'--mapping')
 469   if options.headsfile==None: bail(parser,'--heads')
 470   if options.statusfile==None: bail(parser,'--status')
 471   if options.repourl==None: bail(parser,'--repo')
 472
 473   a={}
 474   if options.authorfile!=None:
 475     a=load_mapping('authors', options.authorfile)
 476
 477   b={}
 478   if options.branchesfile!=None:
 479     b=load_mapping('branches', options.branchesfile)
 480
 481   t={}
 482   if options.tagsfile!=None:
 483     t=load_mapping('tags', options.tagsfile)
 484
 485   if options.default_branch!=None:
 486     set_default_branch(options.default_branch)
 487
 488   if options.origin_name!=None:
 489     set_origin_name(options.origin_name)
 490
 491   encoding=''
 492   if options.encoding!=None:
 493     encoding=options.encoding
 494
 495   fn_encoding=encoding
 496   if options.fn_encoding!=None:
 497     fn_encoding=options.fn_encoding
 498
 499   sys.exit(hg2git(options.repourl,m,options.marksfile,options.mappingfile,
 500                   options.headsfile, options.statusfile,
 501                   authors=a,branchesmap=b,tagsmap=t,
 502                   sob=options.sob,force=options.force,hgtags=options.hgtags,
 503                   notes=options.notes,encoding=encoding,fn_encoding=fn_encoding))