hg2git.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2007 Rocco Rutte <pdmef@gmx.net>
   4 # License: GPLv2
   5
   6 """hg2git.py - A mercurial-to-git filter for git-fast-import(1)
   7 Usage: hg2git.py <hg repo url> <marks file> <heads file> <tip file>
   8 """
   9
  10 from mercurial import repo,hg,cmdutil,util,ui,revlog,node
  11 from tempfile import mkstemp
  12 from optparse import OptionParser
  13 import re
  14 import sys
  15 import os
  16
  17 # silly regex to catch Signed-off-by lines in log message
  18 sob_re=re.compile('^Signed-[Oo]ff-[Bb]y: (.+)$')
  19 # silly regex to see if user field has email address
  20 user_re=re.compile('([^<]+) (<[^>]+>)$')
  21 # silly regex to clean out user names
  22 user_clean_re=re.compile('^["]([^"]+)["]$')
  23 # git branch for hg's default 'HEAD' branch
  24 cfg_master='master'
  25 # insert 'checkpoint' command after this many commits or none at all if 0
  26 cfg_checkpoint_count=0
  27 # write some progress message every this many file contents written
  28 cfg_export_boundary=1000
  29
  30 def usage(ret):
  31   sys.stderr.write(__doc__)
  32   return ret
  33
  34 def setup_repo(url):
  35   myui=ui.ui()
  36   return myui,hg.repository(myui,url)
  37
  38 def fixup_user(user,authors):
  39   if authors!=None:
  40     # if we have an authors table, try to get mapping
  41     # by defaulting to the current value of 'user'
  42     user=authors.get(user,user)
  43   name,mail,m='','',user_re.match(user)
  44   if m==None:
  45     # if we don't have 'Name <mail>' syntax, use 'user
  46     # <devnull@localhost>' if use contains no at and
  47     # 'user <user>' otherwise
  48     name=user
  49     if '@' not in user:
  50       mail='<devnull@localhost>'
  51     else:
  52       mail='<%s>' % user
  53   else:
  54     # if we have 'Name <mail>' syntax, everything is fine :)
  55     name,mail=m.group(1),m.group(2)
  56
  57   # remove any silly quoting from username
  58   m2=user_clean_re.match(name)
  59   if m2!=None:
  60     name=m2.group(1)
  61   return '%s %s' % (name,mail)
  62
  63 def get_branch(name):
  64   if name=='HEAD':
  65     name=cfg_master
  66   return name
  67
  68 def get_changeset(ui,repo,revision,authors):
  69   node=repo.lookup(revision)
  70   (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(node)
  71   tz="%+03d%02d" % (-timezone / 3600, ((-timezone % 3600) / 60))
  72   branch=get_branch(extra.get('branch','master'))
  73   return (manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
  74
  75 def gitmode(x):
  76   return x and '100755' or '100644'
  77
  78 def wr(msg=''):
  79   print msg
  80   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
  81
  82 def checkpoint(count):
  83   count=count+1
  84   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
  85     sys.stderr.write("Checkpoint after %d commits\n" % count)
  86     wr('checkpoint')
  87     wr()
  88   return count
  89
  90 def get_parent_mark(parent,marks):
  91   """Get the mark for some parent.
  92   If we saw it in the current session, return :%d syntax and
  93   otherwise the SHA1 from the cache."""
  94   return marks.get(str(parent+1),':%d' % (parent+1))
  95
  96 def mismatch(f1,f2):
  97   """See if two revisions of a file are not equal."""
  98   return node.hex(f1)!=node.hex(f2)
  99
 100 def outer_set(dleft,dright,l,c,r):
 101   """Loop over our repository and find all changed and missing files."""
 102   for left in dleft.keys():
 103     right=dright.get(left,None)
 104     if right==None:
 105       # we have the file but our parent hasn't: add to left set
 106       l.append(left)
 107     elif mismatch(dleft[left],right):
 108       # we have it but checksums mismatch: add to center set
 109       c.append(left)
 110   for right in dright.keys():
 111     left=dleft.get(right,None)
 112     if left==None:
 113       # if parent has file but we don't: add to right set
 114       r.append(right)
 115     # change is already handled when comparing child against parent
 116   return l,c,r
 117
 118 def get_filechanges(repo,revision,parents,mleft):
 119   """Given some repository and revision, find all changed/deleted files."""
 120   l,c,r=[],[],[]
 121   for p in parents:
 122     if p<0: continue
 123     mright=repo.changectx(p).manifest()
 124     dleft=mleft.keys()
 125     dleft.sort()
 126     dright=mright.keys()
 127     dright.sort()
 128     l,c,r=outer_set(mleft,mright,l,c,r)
 129   return l,c,r
 130
 131 def get_author(logmessage,committer,authors):
 132   """As git distincts between author and committer of a patch, try to
 133   extract author by detecting Signed-off-by lines.
 134
 135   This walks from the end of the log message towards the top skipping
 136   empty lines. Upon the first non-empty line, it walks all Signed-off-by
 137   lines upwards to find the first one. For that (if found), it extracts
 138   authorship information the usual way (authors table, cleaning, etc.)
 139
 140   If no Signed-off-by line is found, this defaults to the committer.
 141
 142   This may sound stupid (and it somehow is), but in log messages we
 143   accidentially may have lines in the middle starting with
 144   "Signed-off-by: foo" and thus matching our detection regex. Prevent
 145   that."""
 146
 147   loglines=logmessage.split('\n')
 148   i=len(loglines)
 149   # from tail walk to top skipping empty lines
 150   while i>=0:
 151     i-=1
 152     if len(loglines[i].strip())==0: continue
 153     break
 154   if i>=0:
 155     # walk further upwards to find first sob line, store in 'first'
 156     first=None
 157     while i>=0:
 158       m=sob_re.match(loglines[i])
 159       if m==None: break
 160       first=m
 161       i-=1
 162     # if the last non-empty line matches our Signed-Off-by regex: extract username
 163     if first!=None:
 164       r=fixup_user(first.group(1),authors)
 165       return r
 166   return committer
 167
 168 def export_file_contents(ctx,manifest,files):
 169   count=0
 170   max=len(files)
 171   for file in files:
 172     fctx=ctx.filectx(file)
 173     d=fctx.data()
 174     wr('M %s inline %s' % (gitmode(manifest.execf(file)),file))
 175     wr('data %d' % len(d)) # had some trouble with size()
 176     wr(d)
 177     count+=1
 178     if count%cfg_export_boundary==0:
 179       sys.stderr.write('Exported %d/%d files\n' % (count,max))
 180   if max>cfg_export_boundary:
 181     sys.stderr.write('Exported %d/%d files\n' % (count,max))
 182
 183 def export_commit(ui,repo,revision,marks,heads,last,max,count,authors,sob):
 184   (_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors)
 185   parents=repo.changelog.parentrevs(revision)
 186
 187   wr('commit refs/heads/%s' % branch)
 188   wr('mark :%d' % (revision+1))
 189   if sob:
 190     wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
 191   wr('committer %s %d %s' % (user,time,timezone))
 192   wr('data %d' % (len(desc)+1)) # wtf?
 193   wr(desc)
 194   wr()
 195
 196   src=heads.get(branch,'')
 197   link=''
 198   if src!='':
 199     # if we have a cached head, this is an incremental import: initialize it
 200     # and kill reference so we won't init it again
 201     wr('from %s' % src)
 202     heads[branch]=''
 203     sys.stderr.write('Initializing branch [%s] to parent [%s]\n' %
 204         (branch,src))
 205     link=src # avoid making a merge commit for incremental import
 206   elif link=='' and not heads.has_key(branch) and revision>0:
 207     # newly created branch and not the first one: connect to parent
 208     tmp=get_parent_mark(parents[0],marks)
 209     wr('from %s' % tmp)
 210     sys.stderr.write('Link new branch [%s] to parent [%s]\n' %
 211         (branch,tmp))
 212     link=tmp # avoid making a merge commit for branch fork
 213
 214   if parents:
 215     l=last.get(branch,revision)
 216     for p in parents:
 217       # 1) as this commit implicitely is the child of the most recent
 218       #    commit of this branch, ignore this parent
 219       # 2) ignore nonexistent parents
 220       # 3) merge otherwise
 221       if p==l or p==revision or p<0:
 222         continue
 223       tmp=get_parent_mark(p,marks)
 224       # if we fork off a branch, don't merge with our parent via 'merge'
 225       # as we have 'from' already above
 226       if tmp==link:
 227         continue
 228       sys.stderr.write('Merging branch [%s] with parent [%s] from [r%d]\n' %
 229           (branch,tmp,p))
 230       wr('merge %s' % tmp)
 231
 232   last[branch]=revision
 233   heads[branch]=''
 234   # we need this later to write out tags
 235   marks[str(revision)]=':%d'%(revision+1)
 236
 237   ctx=repo.changectx(str(revision))
 238   man=ctx.manifest()
 239
 240   if revision==0:
 241     # first revision: feed in full manifest
 242     sys.stderr.write('Exporting full revision %d/%d with %d added files\n' %
 243         (revision,max,len(man.keys())))
 244     export_file_contents(ctx,man,man.keys())
 245   else:
 246     # later revision: feed in changed manifest
 247     added,changed,removed=get_filechanges(repo,revision,parents,man)
 248     sys.stderr.write('Exporting delta revision %d/%d with %d/%d/%d added/changed/removed files\n' %
 249         (revision,max,len(added),len(changed),len(removed)))
 250     export_file_contents(ctx,man,added+changed)
 251     for r in removed:
 252       wr('D %s' % r)
 253
 254   wr()
 255   return checkpoint(count)
 256
 257 def export_tags(ui,repo,marks_cache,start,end,count,authors):
 258   l=repo.tagslist()
 259   for tag,node in l:
 260     # ignore latest revision
 261     if tag=='tip': continue
 262     rev=repo.changelog.rev(node)
 263     # ignore those tags not in our import range
 264     if rev<start or rev>=end: continue
 265
 266     ref=marks_cache.get(str(rev),None)
 267     if ref==None:
 268       sys.stderr.write('Failed to find reference for creating tag'
 269           ' %s at r%d\n' % (tag,rev))
 270       continue
 271     (_,user,(time,timezone),_,desc,branch,_)=get_changeset(ui,repo,rev,authors)
 272     sys.stderr.write('Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag,rev,ref))
 273     wr('tag %s' % tag)
 274     wr('from %s' % ref)
 275     wr('tagger %s %d %s' % (user,time,timezone))
 276     msg='hg2git created tag %s for hg revision %d on branch %s on (summary):\n\t%s' % (tag,
 277         rev,branch,desc.split('\n')[0])
 278     wr('data %d' % (len(msg)+1))
 279     wr(msg)
 280     wr()
 281     count=checkpoint(count)
 282   return count
 283
 284 def load_authors(filename):
 285   cache={}
 286   if not os.path.exists(filename):
 287     return cache
 288   f=open(filename,'r')
 289   l=0
 290   lre=re.compile('^([^=]+)[ ]*=[ ]*(.+)$')
 291   for line in f.readlines():
 292     l+=1
 293     m=lre.match(line)
 294     if m==None:
 295       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
 296       continue
 297     # put key:value in cache, key without ^:
 298     cache[m.group(1).strip()]=m.group(2).strip()
 299   f.close()
 300   sys.stderr.write('Loaded %d authors\n' % l)
 301   return cache
 302
 303 def load_cache(filename):
 304   cache={}
 305   if not os.path.exists(filename):
 306     return cache
 307   f=open(filename,'r')
 308   l=0
 309   for line in f.readlines():
 310     l+=1
 311     fields=line.split(' ')
 312     if fields==None or not len(fields)==2 or fields[0][0]!=':':
 313       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
 314       continue
 315     # put key:value in cache, key without ^:
 316     cache[fields[0][1:]]=fields[1].split('\n')[0]
 317   f.close()
 318   return cache
 319
 320 def save_cache(filename,cache):
 321   f=open(filename,'w+')
 322   map(lambda x: f.write(':%s %s\n' % (str(x),str(cache.get(x)))),cache.keys())
 323   f.close()
 324
 325 def verify_heads(ui,repo,cache):
 326   def getsha1(branch):
 327     try:
 328       f=open(os.getenv('GIT_DIR','/dev/null')+'/refs/heads/'+branch)
 329       sha1=f.readlines()[0].split('\n')[0]
 330       f.close()
 331       return sha1
 332     except IOError:
 333       return None
 334
 335   # get list of hg's branches to verify, don't take all git has
 336   branches=repo.branchtags()
 337   l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
 338   l.sort()
 339
 340   for _,_,b in l:
 341     b=get_branch(b)
 342     sys.stderr.write('Verifying branch [%s]\n' % b)
 343     sha1=getsha1(b)
 344     c=cache.get(b)
 345     if sha1!=c:
 346       sys.stderr.write('Warning: Branch [%s] modified outside hg2git:'
 347         '\n%s (repo) != %s (cache)\n' % (b,sha1,c))
 348   return True
 349
 350 def hg2git(repourl,m,marksfile,headsfile,tipfile,authors={},sob=False):
 351   _max=int(m)
 352
 353   marks_cache=load_cache(marksfile)
 354   heads_cache=load_cache(headsfile)
 355   state_cache=load_cache(tipfile)
 356
 357   ui,repo=setup_repo(repourl)
 358
 359   if not verify_heads(ui,repo,heads_cache):
 360     return 1
 361
 362   tip=repo.changelog.count()
 363
 364   min=int(state_cache.get('tip',0))
 365   max=_max
 366   if _max<0:
 367     max=tip
 368
 369   c=0
 370   last={}
 371   for rev in range(min,max):
 372     c=export_commit(ui,repo,rev,marks_cache,heads_cache,last,max,c,authors,sob)
 373
 374   c=export_tags(ui,repo,marks_cache,min,max,c,authors)
 375
 376   sys.stderr.write('Issued %d commands\n' % c)
 377
 378   state_cache['tip']=max
 379   state_cache['repo']=repourl
 380   save_cache(tipfile,state_cache)
 381
 382   return 0
 383
 384 if __name__=='__main__':
 385   def bail(parser,opt):
 386     sys.stderr.write('Error: No %s option given\n' % opt)
 387     parser.print_help()
 388     sys.exit(2)
 389
 390   parser=OptionParser()
 391
 392   parser.add_option("-m","--max",type="int",dest="max",
 393       help="Maximum hg revision to import")
 394   parser.add_option("--marks",dest="marksfile",
 395       help="File to read git-fast-import's marks from")
 396   parser.add_option("--heads",dest="headsfile",
 397       help="File to read last run's git heads from")
 398   parser.add_option("--status",dest="statusfile",
 399       help="File to read status from")
 400   parser.add_option("-r","--repo",dest="repourl",
 401       help="URL of repo to import")
 402   parser.add_option("-s",action="store_true",dest="sob",
 403       default=False,help="Enable parsing Signed-off-by lines")
 404   parser.add_option("-A","--authors",dest="authorfile",
 405       help="Read authormap from AUTHORFILE")
 406
 407   (options,args)=parser.parse_args()
 408
 409   m=-1
 410   if options.max!=None: m=options.max
 411
 412   if options.marksfile==None: bail(parser,'--marks')
 413   if options.marksfile==None: bail(parser,'--heads')
 414   if options.marksfile==None: bail(parser,'--status')
 415   if options.marksfile==None: bail(parser,'--repo')
 416
 417   a={}
 418   if options.authorfile!=None:
 419     a=load_authors(options.authorfile)
 420
 421   sys.exit(hg2git(options.repourl,m,options.marksfile,options.headsfile,
 422     options.statusfile,authors=a,sob=options.sob))