5 darcs2git -- Darcs to git converter.
7 Copyright (c) 2007 Han-Wen Nienhuys <hanwen@xs4all.nl>
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2, or (at your option)
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
31 # - use binary search to find from-patch in case of conflict.
35 from datetime
import datetime
36 from time
import strptime
38 import distutils
.version
43 import xml
.dom
.minidom
49 ################################################################
55 mail_to_name_dict
= {}
60 ################################################################
63 class PullConflict (Exception):
65 class CommandFailed (Exception):
69 sys
.stderr
.write (s
+ '\n')
71 def get_cli_options ():
72 class MyOP(optparse
.OptionParser
):
74 optparse
.OptionParser
.print_help (self
)
78 This tool is a conversion utility for Darcs repositories, importing
79 them in chronological order. It requires a Git version that has
80 git fast-import. It does not support incremental updating.
84 * repositories with skewed timestamps, or different patches with
85 equal timestamps will confuse darcs2git.
86 * does not respect file modes or time zones.
87 * too slow. See source code for instructions to speed it up.
88 * probably doesn\'t work on partial repositories
90 Report new bugs to hanwen@xs4all.nl
94 Copyright (c) 2007 Han-Wen Nienhuys <hanwen@xs4all.nl>.
95 Distributed under terms of the GNU General Public License
96 This program comes with NO WARRANTY.
101 p
.usage
='''darcs2git [OPTIONS] DARCS-REPO'''
102 p
.description
='''Convert darcs repo to git.'''
104 def update_map (option
, opt
, value
, parser
):
105 for l
in open (value
).readlines ():
106 (mail
, name
) = tuple (l
.strip ().split ('='))
107 mail_to_name_dict
[mail
] = name
109 p
.add_option ('-a', '--authors', action
='callback',
113 help='read a text file, containing EMAIL=NAME lines')
115 p
.add_option ('--checkpoint-frequency', action
='store',
116 dest
='checkpoint_frequency',
119 help='how often should the git importer be synced?\n'
120 'Default is 0 (no limit)'
123 p
.add_option ('-d', '--destination', action
='store',
126 dest
='target_git_repo',
127 help='where to put the resulting Git repo.')
129 p
.add_option ('--verbose', action
='store_true',
132 help='show commands as they are invoked')
134 p
.add_option ('--history-window', action
='store',
135 dest
='history_window',
138 help='Look back this many patches as conflict ancestors.\n'
139 'Default is 0 (no limit)')
141 p
.add_option ('--debug', action
='store_true',
144 help="""add patch numbers to commit messages;
145 don\'t clean conversion repo;
149 options
, args
= p
.parse_args ()
154 if len(urlparse
.urlparse(args
[0])) == 0:
155 raise NotImplementedError, "We support local DARCS repos only."
157 git_version
= distutils
.version
.LooseVersion(
158 os
.popen("git --version","r").read().strip().split(" ")[-1])
159 ideal_version
= distutils
.version
.LooseVersion("1.5.0")
160 if git_version
<ideal_version
:
161 raise RuntimeError,"You need git >= 1.5.0 for this."
163 options
.basename
= os
.path
.basename (
164 os
.path
.normpath (args
[0])).replace ('.darcs', '')
165 if not options
.target_git_repo
:
166 options
.target_git_repo
= options
.basename
+ '.git'
170 name
= options
.target_git_repo
.replace ('.git', '.log')
171 if name
== options
.target_git_repo
:
174 progress ("Shell log to %s" % name
)
175 log_file
= open (name
, 'w')
177 return (options
, args
)
179 def read_pipe (cmd
, ignore_errors
=False):
181 progress ('pipe %s' % cmd
)
182 pipe
= os
.popen (cmd
)
185 if pipe
.close () and not ignore_errors
:
186 raise CommandFailed ("Pipe failed: %s" % cmd
)
190 def system (c
, ignore_error
=0, timed
=0):
197 log_file
.write ('%s\n' % c
)
200 if os
.system (c
) and not ignore_error
:
201 raise CommandFailed ("Command failed: %s" % c
)
203 def darcs_date_to_git (x
):
204 t
= time
.strptime (x
, '%Y%m%d%H%M%S')
205 return '%d' % int (time
.mktime (t
))
207 def darcs_timezone (x
) :
208 time
.strptime (x
, '%a %b %d %H:%M:%S %Z %Y')
213 ################################################################
216 ## http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/521889
218 PATCH_DATE_FORMAT
= '%Y%m%d%H%M%S'
221 \[ # Patch start indicator
222 (?P<name>[^\n]+)\n # Patch name (rest of same line)
223 (?P<author>[^\*]+) # Patch author
224 \* # Author/date separator
225 (?P<inverted>[-\*]) # Inverted patch indicator
226 (?P<date>\d{14}) # Patch date
227 (?:\n(?P<comment>(?:^\ [^\n]*\n)+))? # Optional long comment
228 \] # Patch end indicator
230 patch_re
= re
.compile(patch_pattern
, re
.VERBOSE | re
.MULTILINE
)
231 tidy_comment_re
= re
.compile(r
'^ ', re
.MULTILINE
)
233 def parse_inventory(inventory
):
235 Given the contents of a darcs inventory file, generates ``Patch``
236 objects representing contained patch details.
238 for match
in patch_re
.finditer(inventory
):
239 attrs
= match
.groupdict(None)
240 attrs
['inverted'] = (attrs
['inverted'] == '-')
241 if attrs
['comment'] is not None:
242 attrs
['comment'] = tidy_comment_re
.sub('', attrs
['comment']).strip()
243 yield InventoryPatch(**attrs
)
245 def fix_braindead_darcs_escapes(s
):
246 def insert_hibit(match
):
247 return chr(int(match
.group(1), 16))
249 return re
.sub(r
'\[_\\([0-9a-f][0-9a-f])_\]',
250 insert_hibit
, str(s
))
252 class InventoryPatch
:
254 Patch details, as defined in a darcs inventory file.
256 Attribute names match those generated by the
257 ``darcs changes --xml-output`` command.
260 def __init__(self
, name
, author
, date
, inverted
, comment
=None):
263 self
.date
= datetime(*strptime(date
, PATCH_DATE_FORMAT
)[:6])
264 self
.inverted
= inverted
265 self
.comment
= comment
271 def complete_patch_details(self
):
272 date_str
= self
.date
.strftime(PATCH_DATE_FORMAT
)
273 return '%s%s%s%s%s' % (
274 self
.name
, self
.author
, date_str
,
275 self
.comment
and ''.join([l
.rstrip() for l
in self
.comment
.split('\n')]) or '',
276 self
.inverted
and 't' or 'f')
283 return unicode('%s%s*%s%s' % (self
.name
, self
.author
, inv
, self
.hash.split ('-')[0]), 'UTF-8')
288 Calculates the filename of the gzipped file containing patch
289 contents in the repository's ``patches`` directory.
291 This consists of the patch date, a partial SHA-1 hash of the
292 patch author and a full SHA-1 hash of the complete patch
296 date_str
= self
.date
.strftime(PATCH_DATE_FORMAT
)
297 return '%s-%s-%s.gz' % (date_str
,
298 sha
.new(self
.author
).hexdigest()[:5],
299 sha
.new(self
.complete_patch_details
).hexdigest())
301 ## http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/521889
303 class DarcsConversionRepo
:
304 """Representation of a Darcs repo.
306 The repo is thought to be ordered, and supports methods for
307 going back (obliterate) and forward (pull).
311 def __init__ (self
, dir, patches
):
312 self
.dir = os
.path
.abspath (dir)
313 self
.patches
= patches
314 self
._current
_number
= -1
316 self
._inventory
_dict
= None
317 self
._short
_id
_dict
= dict ((p
.short_id (), p
) for p
in patches
)
320 if not options
.debug
:
321 system ('rm -fr %s' % self
.dir)
323 def is_contiguous (self
):
324 return (len (self
.inventory_dict ()) == self
._current
_number
+ 1
325 and self
.contains_contiguous (self
._current
_number
))
327 def contains_contiguous (self
, num
):
328 if not self
._is
_valid
:
331 darcs_dir
= self
.dir + '/_darcs'
332 if not os
.path
.exists (darcs_dir
):
335 for p
in self
.patches
[:num
+ 1]:
336 if not self
.has_patch (p
):
341 def has_patch (self
, p
):
342 assert self
._is
_valid
344 return p
.short_id () in self
.inventory_dict ()
346 def pristine_tree (self
):
347 return self
.dir + '/_darcs/pristine'
349 def go_back_to (self
, dest
):
351 # at 4, len = 5, go to 2: count == 2
352 count
= len (self
.inventory_dict()) - dest
- 1
354 assert self
._is
_valid
360 progress ('Rewinding %d patches' % count
)
361 system ('cd %(dir)s && darcs revert --all' % locals())
362 system ('cd %(dir)s && yes|darcs obliterate -a --ignore-times --last %(count)d' % locals ())
363 system ('cd %(dir)s && darcs revert -a' % locals())
364 d
= self
.inventory_dict ()
365 for p
in self
.patches
[dest
+1:self
._current
_number
+1]:
371 self
._current
_number
= dest
374 system ('rm -rf %s' % self
.dir)
378 system ('rsync -a %(dir)s/_darcs/pristine/ %(dir)s/' % locals ())
380 def pull (self
, patch
):
381 id = patch
.attributes
['hash']
382 source_repo
= patch
.dir
385 progress ('Pull patch %d' % patch
.number
)
386 system ('cd %(dir)s && darcs revert --all' % locals())
387 system ('cd %(dir)s && darcs pull --ignore-times --quiet --all --match "hash %(id)s" %(source_repo)s ' % locals ())
389 self
._current
_number
= patch
.number
391 ## must reread: the pull may have pulled in others.
392 self
._inventory
_dict
= None
394 def go_forward_to (self
, num
):
395 d
= self
.inventory_dict ()
400 for p
in self
.patches
[0:num
+1]:
401 if not d
.has_key (p
.short_id ()):
405 pull_str
= ' || '.join (['hash %s' % p
.id () for p
in pull_me
])
407 src
= self
.patches
[0].dir
409 progress ('Pulling %d patches to go to %d' % (len (pull_me
), num
))
410 system ('darcs revert --repo %(dir)s --all' % locals ())
411 system ('darcs pull --all --repo %(dir)s --match "%(pull_str)s" %(src)s' % locals ())
413 def create_fresh (self
):
415 system ('rm -rf %(dir)s && mkdir %(dir)s && darcs init --repo %(dir)s'
417 self
._is
_valid
= True
418 self
._current
_number
= -1
419 self
._inventory
_dict
= {}
421 def inventory (self
):
422 darcs_dir
= self
.dir + '/_darcs'
424 for f
in [darcs_dir
+ '/inventory'] + glob
.glob (darcs_dir
+ '/inventories/*'):
425 i
+= open (f
).read ()
428 def inventory_dict (self
):
430 if type (self
._inventory
_dict
) != type ({}):
431 self
._inventory
_dict
= {}
433 for p
in parse_inventory(self
.inventory()):
437 self
._inventory
_dict
[key
] = self
._short
_id
_dict
[key
]
439 print 'key not found', key
440 print self
._short
_id
_dict
443 return self
._inventory
_dict
445 def start_at (self
, num
):
446 """Move the repo to NUM.
448 This uses the fishy technique of writing the inventory and
449 constructing the pristine tree with 'darcs repair'
451 progress ('Starting afresh at %d' % num
)
455 iv
= open (dir + '/_darcs/inventory', 'w')
457 log_file
.write ("# messing with _darcs/inventory")
459 for p
in self
.patches
[:num
+1]:
460 os
.link (p
.filename (), dir + '/_darcs/patches/' + os
.path
.basename (p
.filename ()))
461 iv
.write (p
.header ())
462 self
._inventory
_dict
[p
.short_id ()] = p
465 system ('darcs revert --repo %(dir)s --all' % locals())
466 system ('darcs repair --repo %(dir)s --quiet' % locals ())
468 self
._current
_number
= num
469 self
._is
_valid
= True
471 def go_to (self
, dest
):
472 if not self
._is
_valid
:
474 elif dest
== self
._current
_number
and self
.is_contiguous ():
476 elif (self
.contains_contiguous (dest
)):
477 self
.go_back_to (dest
)
478 elif dest
- len (self
.inventory_dict ()) < dest
/ 100:
479 self
.go_forward_to (dest
)
484 def go_from_to (self
, from_patch
, to_patch
):
486 """Move the repo to FROM_PATCH, then go to TO_PATCH. Raise
487 PullConflict if conflict is detected
490 progress ('Trying %s -> %s' % (from_patch
, to_patch
))
492 source
= to_patch
.dir
495 self
.go_to (from_patch
.number
)
501 success
= 'No conflicts to resolve' in read_pipe ('cd %(dir)s && echo y|darcs resolve' % locals ())
502 except CommandFailed
:
503 self
._is
_valid
= False
504 raise PullConflict ()
507 raise PullConflict ()
511 return 'patch %d' % self
.number
513 def __init__ (self
, xml
, dir):
518 self
._contents
= None
519 for (nm
, value
) in xml
.attributes
.items():
520 self
.attributes
[nm
] = value
522 # fixme: ugh attributes vs. methods.
523 self
.extract_author ()
524 self
.extract_message ()
528 return self
.attributes
['hash']
532 if self
.attributes
['inverted'] == 'True':
535 return '%s%s*%s%s' % (self
.name(), self
.attributes
['author'], inv
, self
.attributes
['hash'].split ('-')[0])
538 return self
.dir + '/_darcs/patches/' + self
.attributes
['hash']
541 if type (self
._contents
) != type (''):
542 f
= gzip
.open (self
.filename ())
543 self
._contents
= f
.read ()
545 return self
._contents
548 lines
= self
.contents ().split ('\n')
551 committer
= lines
[1] + '\n'
552 committer
= re
.sub ('] {\n$', ']\n', committer
)
553 committer
= re
.sub ('] *\n$', ']\n', committer
)
555 if not committer
.endswith (']\n'):
562 header
= name
+ '\n' + committer
566 assert header
[-1] == '\n'
569 def extract_author (self
):
570 mail
= self
.attributes
['author']
572 m
= re
.search ("^(.*) <(.*)>$", mail
)
579 name
= mail_to_name_dict
[mail
]
581 name
= mail
.split ('@')[0]
583 self
.author_name
= name
585 # mail addresses should be plain strings.
586 self
.author_mail
= mail
.encode('utf-8')
588 def extract_time (self
):
589 self
.date
= darcs_date_to_git (self
.attributes
['date']) + ' ' + darcs_timezone (self
.attributes
['local_date'])
594 name_elt
= self
.xml
.getElementsByTagName ('name')[0]
595 patch_name
= unicode(fix_braindead_darcs_escapes(str(name_elt
.childNodes
[0].data
)), 'UTF-8')
600 def extract_message (self
):
601 patch_name
= self
.name ()
602 comment_elts
= self
.xml
.getElementsByTagName ('comment')
605 comment
= comment_elts
[0].childNodes
[0].data
607 if self
.attributes
['inverted'] == 'True':
608 patch_name
= 'UNDO: ' + patch_name
610 self
.message
= '%s\n\n%s' % (patch_name
, comment
)
613 patch_name
= self
.name ()
614 if patch_name
.startswith ("TAG "):
616 tag
= re
.sub (r
'\s', '_', tag
).strip ()
617 tag
= re
.sub (r
':', '_', tag
).strip ()
621 def get_darcs_patches (darcs_repo
):
622 progress ('reading patches.')
624 xml_string
= read_pipe ('darcs changes --xml --reverse --repo ' + darcs_repo
)
626 dom
= xml
.dom
.minidom
.parseString(xml_string
)
627 xmls
= dom
.documentElement
.getElementsByTagName('patch')
629 patches
= [DarcsPatch (x
, darcs_repo
) for x
in xmls
]
638 ################################################################
642 def __init__ (self
, parent
, darcs_patch
):
644 self
.darcs_patch
= darcs_patch
646 self
.depth
= parent
.depth
+ 1
651 return self
.darcs_patch
.number
653 def parent_patch (self
):
655 return self
.parent
.darcs_patch
659 def common_ancestor (a
, b
):
661 if a
.depth
< b
.depth
:
663 elif a
.depth
> b
.depth
:
677 def export_checkpoint (gfi
):
678 gfi
.write ('checkpoint\n\n')
680 def export_tree (tree
, gfi
):
681 tree
= os
.path
.normpath (tree
)
682 gfi
.write ('deleteall\n')
683 for (root
, dirs
, files
) in os
.walk (tree
):
685 rf
= os
.path
.normpath (os
.path
.join (root
, f
))
686 s
= open (rf
).read ()
687 rf
= rf
.replace (tree
+ '/', '')
689 gfi
.write ('M 644 inline %s\n' % rf
)
690 gfi
.write ('data %d\n%s\n' % (len (s
), s
))
694 def export_commit (repo
, patch
, last_patch
, gfi
):
695 gfi
.write ('commit refs/heads/darcstmp%d\n' % patch
.number
)
696 gfi
.write ('mark :%d\n' % (patch
.number
+ 1))
698 raw_name
= patch
.author_name
699 gfi
.write ('committer %s <%s> %s\n' % (raw_name
,
705 msg
+= '\n\n#%d\n' % patch
.number
707 msg
= msg
.encode('utf-8')
708 gfi
.write ('data %d\n%s\n' % (len (msg
), msg
))
710 for (n
, p
) in pending_patches
.items ():
711 if repo
.has_patch (p
):
713 del pending_patches
[n
]
717 and git_commits
.has_key (last_patch
.number
)):
718 mergers
= [last_patch
.number
]
721 gfi
.write ('from :%d\n' % (mergers
[0] + 1))
722 for m
in mergers
[1:]:
723 gfi
.write ('merge :%d\n' % (m
+ 1))
725 pending_patches
[patch
.number
] = patch
726 export_tree (repo
.pristine_tree (), gfi
)
730 n
= last_patch
.number
731 git_commits
[patch
.number
] = GitCommit (git_commits
.get (n
, None),
734 def export_pending (gfi
):
735 if len (pending_patches
.items ()) == 1:
736 gfi
.write ('reset refs/heads/master\n')
737 gfi
.write ('from :%d\n\n' % (pending_patches
.values()[0].number
+1))
739 progress ("Creating branch master")
742 for (n
, p
) in pending_patches
.items ():
743 gfi
.write ('reset refs/heads/master%d\n' % n
)
744 gfi
.write ('from :%d\n\n' % (n
+1))
746 progress ("Creating branch master%d" % n
)
748 patches
= pending_patches
.values()
750 gfi
.write ('commit refs/heads/master\n')
751 gfi
.write ('committer %s <%s> %s\n' % (patch
.author_name
,
755 gfi
.write ('data %d\n%s\n' % (len(msg
), msg
))
756 gfi
.write ('from :%d\n' % (patch
.number
+ 1))
757 for p
in patches
[1:]:
758 gfi
.write ('merge :%d\n' % (p
.number
+ 1))
761 def export_tag (patch
, gfi
):
762 gfi
.write ('tag %s\n' % patch
.tag_name ())
763 gfi
.write ('from :%d\n' % (patch
.number
+ 1))
764 gfi
.write ('tagger %s <%s> %s\n' % (patch
.author_name
,
768 raw_message
= patch
.message
.encode('utf-8')
769 gfi
.write ('data %d\n%s\n' % (len (raw_message
),
772 ################################################################
775 def test_conversion (darcs_repo
, git_repo
):
776 pristine
= '%(darcs_repo)s/_darcs/pristine' % locals ()
777 if not os
.path
.exists (pristine
):
778 progress ("darcs repository does not contain pristine tree?!")
781 gd
= options
.basename
+ '.checkouttmp.git'
782 system ('rm -rf %(gd)s && git clone %(git_repo)s %(gd)s' % locals ())
783 diff_cmd
= 'diff --exclude .git -urN %(gd)s %(pristine)s' % locals ()
784 diff
= read_pipe (diff_cmd
, ignore_errors
=True)
785 system ('rm -rf %(gd)s' % locals ())
788 if len (diff
) > 1024:
789 diff
= diff
[:512] + '\n...\n' + diff
[-512:]
791 progress ("Conversion introduced changes: %s" % diff
)
794 progress ("Checkout matches pristine darcs tree.")
797 (options
, args
) = get_cli_options ()
799 darcs_repo
= os
.path
.abspath (args
[0])
800 git_repo
= os
.path
.abspath (options
.target_git_repo
)
802 if os
.path
.exists (git_repo
):
803 system ('rm -rf %(git_repo)s' % locals ())
805 system ('mkdir %(git_repo)s && cd %(git_repo)s && git --bare init' % locals ())
806 system ('git --git-dir %(git_repo)s repo-config core.logAllRefUpdates false' % locals ())
808 os
.environ
['GIT_DIR'] = git_repo
814 gfi
= os
.popen ('git fast-import %s' % quiet
, 'w')
816 patches
= get_darcs_patches (darcs_repo
)
817 conv_repo
= DarcsConversionRepo (options
.basename
+ ".tmpdarcs", patches
)
818 conv_repo
.start_at (-1)
824 combinations
= [(v
, w
) for v
in pending_patches
.values ()
825 for w
in pending_patches
.values ()]
826 candidates
= [common_ancestor (git_commits
[c
[0].number
], git_commits
[c
[1].number
]) for c
in combinations
]
827 candidates
= sorted ([(-a
.darcs_patch
.number
, a
) for a
in candidates
])
828 for (depth
, c
) in candidates
:
831 conv_repo
.go_from_to (q
, p
)
834 parent_number
= q
.number
835 progress ('Found existing common parent as predecessor')
841 ## no branches found where we could attach.
842 ## try previous commits one by one.
844 parent_number
= p
.number
- 2
846 if parent_number
>= 0:
847 parent_patch
= patches
[parent_number
]
850 conv_repo
.go_from_to (parent_patch
, p
)
854 ## simplistic, may not be enough.
855 progress ('conflict, going one back')
858 if parent_number
< 0:
861 if (options
.history_window
862 and parent_number
< p
.number
- options
.history_window
):
867 if parent_number
>= 0 or p
.number
== 0:
868 progress ('Export %d -> %d (total %d)' % (parent_number
,
869 p
.number
, len (patches
)))
870 export_commit (conv_repo
, p
, parent_patch
, gfi
)
874 if options
.checkpoint_frequency
and p
.number
% options
.checkpoint_frequency
== 0:
875 export_checkpoint (gfi
)
877 progress ("Can't import patch %d, need conflict resolution patch?"
882 for f
in glob
.glob ('%(git_repo)s/refs/heads/darcstmp*' % locals ()):
885 test_conversion (darcs_repo
, git_repo
)
887 if not options
.debug
: