hg2git.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2007 Rocco Rutte <pdmef@gmx.net>
   4 # License: GPLv2
   5
   6 """hg2git.py - A mercurial-to-git filter for git-fast-import(1)
   7 Usage: hg2git.py <hg repo url> <marks file> <heads file> <tip file>
   8 """
   9
  10 from mercurial import repo,hg,cmdutil,util,ui,revlog,node
  11 from tempfile import mkstemp
  12 from optparse import OptionParser
  13 import re
  14 import sys
  15 import os
  16
  17 # silly regex to catch Signed-off-by lines in log message
  18 sob_re=re.compile('^Signed-[Oo]ff-[Bb]y: (.+)$')
  19 # silly regex to see if user field has email address
  20 user_re=re.compile('([^<]+) (<[^>]+>)$')
  21 # silly regex to clean out user names
  22 user_clean_re=re.compile('^["]([^"]+)["]$')
  23 # git branch for hg's default 'HEAD' branch
  24 cfg_master='master'
  25 # insert 'checkpoint' command after this many commits or none at all if 0
  26 cfg_checkpoint_count=0
  27
  28 def usage(ret):
  29   sys.stderr.write(__doc__)
  30   return ret
  31
  32 def setup_repo(url):
  33   myui=ui.ui()
  34   return myui,hg.repository(myui,url)
  35
  36 def fixup_user(user,authors):
  37   if authors!=None:
  38     # if we have an authors table, try to get mapping
  39     # by defaulting to the current value of 'user'
  40     user=authors.get(user,user)
  41   name,mail,m='','',user_re.match(user)
  42   if m==None:
  43     # if we don't have 'Name <mail>' syntax, use 'user
  44     # <devnull@localhost>' if use contains no at and
  45     # 'user <user>' otherwise
  46     name=user
  47     if '@' not in user:
  48       mail='<devnull@localhost>'
  49     else:
  50       mail='<%s>' % user
  51   else:
  52     # if we have 'Name <mail>' syntax, everything is fine :)
  53     name,mail=m.group(1),m.group(2)
  54
  55   # remove any silly quoting from username
  56   m2=user_clean_re.match(name)
  57   if m2!=None:
  58     name=m2.group(1)
  59   return '%s %s' % (name,mail)
  60
  61 def get_branch(name):
  62   if name=='HEAD':
  63     name=cfg_master
  64   return name
  65
  66 def get_changeset(ui,repo,revision,authors):
  67   node=repo.lookup(revision)
  68   (manifest,user,(time,timezone),files,desc,extra)=repo.changelog.read(node)
  69   tz="%+03d%02d" % (-timezone / 3600, ((-timezone % 3600) / 60))
  70   branch=get_branch(extra.get('branch','master'))
  71   return (manifest,fixup_user(user,authors),(time,tz),files,desc,branch,extra)
  72
  73 def gitmode(x):
  74   return x and '100755' or '100644'
  75
  76 def wr(msg=''):
  77   print msg
  78   #map(lambda x: sys.stderr.write('\t[%s]\n' % x),msg.split('\n'))
  79
  80 def checkpoint(count):
  81   count=count+1
  82   if cfg_checkpoint_count>0 and count%cfg_checkpoint_count==0:
  83     sys.stderr.write("Checkpoint after %d commits\n" % count)
  84     wr('checkpoint')
  85     wr()
  86   return count
  87
  88 def get_parent_mark(parent,marks):
  89   """Get the mark for some parent.
  90   If we saw it in the current session, return :%d syntax and
  91   otherwise the SHA1 from the cache."""
  92   return marks.get(str(parent+1),':%d' % (parent+1))
  93
  94 def mismatch(f1,f2):
  95   """See if two revisions of a file are not equal."""
  96   return node.hex(f1)!=node.hex(f2)
  97
  98 def outer_set(dleft,dright,l,c,r):
  99   """Loop over our repository and find all changed and missing files."""
 100   for left in dleft.keys():
 101     right=dright.get(left,None)
 102     if right==None:
 103       # we have the file but our parent hasn't: add to left set
 104       l.append(left)
 105     elif mismatch(dleft[left],right):
 106       # we have it but checksums mismatch: add to center set
 107       c.append(left)
 108   for right in dright.keys():
 109     left=dleft.get(right,None)
 110     if left==None:
 111       # if parent has file but we don't: add to right set
 112       r.append(right)
 113     # change is already handled when comparing child against parent
 114   return l,c,r
 115
 116 def get_filechanges(repo,revision,parents,mleft):
 117   """Given some repository and revision, find all changed/deleted files."""
 118   l,c,r=[],[],[]
 119   for p in parents:
 120     if p<0: continue
 121     mright=repo.changectx(p).manifest()
 122     dleft=mleft.keys()
 123     dleft.sort()
 124     dright=mright.keys()
 125     dright.sort()
 126     l,c,r=outer_set(mleft,mright,l,c,r)
 127   return l,c,r
 128
 129 def get_author(logmessage,committer,authors):
 130   """As git distincts between author and committer of a patch, try to
 131   extract author by detecting Signed-off-by lines.
 132
 133   This walks from the end of the log message towards the top skipping
 134   empty lines. Upon the first non-empty line, it walks all Signed-off-by
 135   lines upwards to find the first one. For that (if found), it extracts
 136   authorship information the usual way (authors table, cleaning, etc.)
 137
 138   If no Signed-off-by line is found, this defaults to the committer.
 139
 140   This may sound stupid (and it somehow is), but in log messages we
 141   accidentially may have lines in the middle starting with
 142   "Signed-off-by: foo" and thus matching our detection regex. Prevent
 143   that."""
 144
 145   loglines=logmessage.split('\n')
 146   i=len(loglines)
 147   # from tail walk to top skipping empty lines
 148   while i>=0:
 149     i-=1
 150     if len(loglines[i].strip())==0: continue
 151     break
 152   if i>=0:
 153     # walk further upwards to find first sob line, store in 'first'
 154     first=None
 155     while i>=0:
 156       m=sob_re.match(loglines[i])
 157       if m==None: break
 158       first=m
 159       i-=1
 160     # if the last non-empty line matches our Signed-Off-by regex: extract username
 161     if first!=None:
 162       r=fixup_user(first.group(1),authors)
 163       return r
 164   return committer
 165
 166 def export_commit(ui,repo,revision,marks,heads,last,max,count,authors):
 167   (_,user,(time,timezone),files,desc,branch,_)=get_changeset(ui,repo,revision,authors)
 168   parents=repo.changelog.parentrevs(revision)
 169
 170   wr('commit refs/heads/%s' % branch)
 171   wr('mark :%d' % (revision+1))
 172   wr('author %s %d %s' % (get_author(desc,user,authors),time,timezone))
 173   wr('committer %s %d %s' % (user,time,timezone))
 174   wr('data %d' % (len(desc)+1)) # wtf?
 175   wr(desc)
 176   wr()
 177
 178   src=heads.get(branch,'')
 179   link=''
 180   if src!='':
 181     # if we have a cached head, this is an incremental import: initialize it
 182     # and kill reference so we won't init it again
 183     wr('from %s' % src)
 184     heads[branch]=''
 185     sys.stderr.write('Initializing branch [%s] to parent [%s]\n' %
 186         (branch,src))
 187     link=src # avoid making a merge commit for incremental import
 188   elif link=='' and not heads.has_key(branch) and revision>0:
 189     # newly created branch and not the first one: connect to parent
 190     tmp=get_parent_mark(parents[0],marks)
 191     wr('from %s' % tmp)
 192     sys.stderr.write('Link new branch [%s] to parent [%s]\n' %
 193         (branch,tmp))
 194     link=tmp # avoid making a merge commit for branch fork
 195
 196   if parents:
 197     l=last.get(branch,revision)
 198     for p in parents:
 199       # 1) as this commit implicitely is the child of the most recent
 200       #    commit of this branch, ignore this parent
 201       # 2) ignore nonexistent parents
 202       # 3) merge otherwise
 203       if p==l or p==revision or p<0:
 204         continue
 205       tmp=get_parent_mark(p,marks)
 206       # if we fork off a branch, don't merge with our parent via 'merge'
 207       # as we have 'from' already above
 208       if tmp==link:
 209         continue
 210       sys.stderr.write('Merging branch [%s] with parent [%s] from [r%d]\n' %
 211           (branch,tmp,p))
 212       wr('merge %s' % tmp)
 213
 214   last[branch]=revision
 215   heads[branch]=''
 216   # we need this later to write out tags
 217   marks[str(revision)]=':%d'%(revision+1)
 218
 219   ctx=repo.changectx(str(revision))
 220   man=ctx.manifest()
 221   added,changed,removed=get_filechanges(repo,revision,parents,man)
 222
 223   sys.stderr.write('Exporting revision %d with %d/%d/%d added/changed/removed files\n' %
 224       (revision,len(added),len(changed),len(removed)))
 225
 226   for a in added+changed:
 227     fctx=ctx.filectx(a)
 228     d=fctx.data()
 229     wr('M %s inline %s' % (gitmode(man.execf(a)),a))
 230     wr('data %d' % len(d)) # had some trouble with size()
 231     wr(d)
 232
 233   for r in removed:
 234     wr('D %s' % r)
 235
 236   wr()
 237   return checkpoint(count)
 238
 239 def export_tags(ui,repo,marks_cache,start,end,count,authors):
 240   l=repo.tagslist()
 241   for tag,node in l:
 242     # ignore latest revision
 243     if tag=='tip': continue
 244     rev=repo.changelog.rev(node)
 245     # ignore those tags not in our import range
 246     if rev<start or rev>=end: continue
 247
 248     ref=marks_cache.get(str(rev),None)
 249     if ref==None:
 250       sys.stderr.write('Failed to find reference for creating tag'
 251           ' %s at r%d\n' % (tag,rev))
 252       continue
 253     (_,user,(time,timezone),_,desc,branch,_)=get_changeset(ui,repo,rev,authors)
 254     sys.stderr.write('Exporting tag [%s] at [hg r%d] [git %s]\n' % (tag,rev,ref))
 255     wr('tag %s' % tag)
 256     wr('from %s' % ref)
 257     wr('tagger %s %d %s' % (user,time,timezone))
 258     msg='hg2git created tag %s for hg revision %d on branch %s on (summary):\n\t%s' % (tag,
 259         rev,branch,desc.split('\n')[0])
 260     wr('data %d' % (len(msg)+1))
 261     wr(msg)
 262     wr()
 263     count=checkpoint(count)
 264   return count
 265
 266 def load_cache(filename):
 267   cache={}
 268   if not os.path.exists(filename):
 269     return cache
 270   f=open(filename,'r')
 271   l=0
 272   for line in f.readlines():
 273     l+=1
 274     fields=line.split(' ')
 275     if fields==None or not len(fields)==2 or fields[0][0]!=':':
 276       sys.stderr.write('Invalid file format in [%s], line %d\n' % (filename,l))
 277       continue
 278     # put key:value in cache, key without ^:
 279     cache[fields[0][1:]]=fields[1].split('\n')[0]
 280   f.close()
 281   return cache
 282
 283 def save_cache(filename,cache):
 284   f=open(filename,'w+')
 285   map(lambda x: f.write(':%s %s\n' % (str(x),str(cache.get(x)))),cache.keys())
 286   f.close()
 287
 288 def verify_heads(ui,repo,cache):
 289   def getsha1(branch):
 290     f=open(os.getenv('GIT_DIR','/dev/null')+'/refs/heads/'+branch)
 291     sha1=f.readlines()[0].split('\n')[0]
 292     f.close()
 293     return sha1
 294
 295   # get list of hg's branches to verify, don't take all git has
 296   branches=repo.branchtags()
 297   l=[(-repo.changelog.rev(n), n, t) for t, n in branches.items()]
 298   l.sort()
 299
 300   for _,_,b in l:
 301     b=get_branch(b)
 302     sys.stderr.write('Verifying branch [%s]\n' % b)
 303     sha1=getsha1(b)
 304     c=cache.get(b)
 305     if sha1!=c:
 306       sys.stderr.write('Warning: Branch [%s] modified outside hg2git:'
 307         '\n%s (repo) != %s (cache)\n' % (b,sha1,c))
 308   return True
 309
 310 def hg2git(repourl,m,marksfile,headsfile,tipfile,authors={}):
 311   _max=int(m)
 312
 313   marks_cache=load_cache(marksfile)
 314   heads_cache=load_cache(headsfile)
 315   state_cache=load_cache(tipfile)
 316
 317   ui,repo=setup_repo(repourl)
 318
 319   if not verify_heads(ui,repo,heads_cache):
 320     return 1
 321
 322   tip=repo.changelog.count()
 323
 324   min=int(state_cache.get('tip',0))
 325   max=_max
 326   if _max<=0:
 327     max=tip
 328
 329   c=0
 330   last={}
 331   for rev in range(min,max):
 332     c=export_commit(ui,repo,rev,marks_cache,heads_cache,last,tip,c,authors)
 333
 334   c=export_tags(ui,repo,marks_cache,min,max,c,authors)
 335
 336   sys.stderr.write('Issued %d commands\n' % c)
 337
 338   state_cache['tip']=max
 339   state_cache['repo']=repourl
 340   save_cache(tipfile,state_cache)
 341
 342   return 0
 343
 344 if __name__=='__main__':
 345   def bail(parser,opt):
 346     sys.stderr.write('Error: No %s option given\n' % opt)
 347     parser.print_help()
 348     sys.exit(2)
 349
 350   parser=OptionParser()
 351
 352   parser.add_option("-m","--max",type="int",dest="max",
 353       help="Maximum hg revision to import")
 354   parser.add_option("--marks",dest="marksfile",
 355       help="File to read git-fast-import's marks from")
 356   parser.add_option("--heads",dest="headsfile",
 357       help="File to read last run's git heads from")
 358   parser.add_option("--status",dest="statusfile",
 359       help="File to read status from")
 360   parser.add_option("-r","--repo",dest="repourl",
 361       help="URL of repo to import")
 362
 363   (options,args)=parser.parse_args()
 364
 365   m=0
 366   if options.max!=None: m=options.max
 367
 368   if options.marksfile==None: bail(parser,'--marks')
 369   if options.marksfile==None: bail(parser,'--heads')
 370   if options.marksfile==None: bail(parser,'--status')
 371   if options.marksfile==None: bail(parser,'--repo')
 372
 373   sys.exit(hg2git(options.repourl,m,options.marksfile,options.headsfile,
 374     options.headsfile))