2 * Copyright (C) 2007 Thiago Macieira <thiago@kde.org>
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 * Based on svn-fast-export by Chris Lee <clee@kde.org>
20 * License: MIT <http://www.opensource.org/licenses/mit-license.php>
21 * URL: git://repo.or.cz/fast-import.git http://repo.or.cz/w/fast-export.git
25 #define _LARGEFILE_SUPPORT
26 #define _LARGEFILE64_SUPPORT
37 #include <apr_getopt.h>
38 #include <apr_general.h>
41 #include <svn_pools.h>
42 #include <svn_repos.h>
43 #include <svn_types.h>
48 #include "repository.h"
51 #define SVN_ERR(expr) SVN_INT_ERR(expr)
53 typedef QList
<Rules::Match
> MatchRuleList
;
54 typedef QHash
<QString
, Repository
*> RepositoryHash
;
55 typedef QHash
<QByteArray
, QByteArray
> IdentityHash
;
60 AprAutoPool(const AprAutoPool
&);
61 AprAutoPool
&operator=(const AprAutoPool
&);
63 inline AprAutoPool(apr_pool_t
*parent
= NULL
)
64 { pool
= svn_pool_create(parent
); }
66 { svn_pool_destroy(pool
); }
68 inline void clear() { svn_pool_clear(pool
); }
69 inline apr_pool_t
*data() const { return pool
; }
70 inline operator apr_pool_t
*() const { return pool
; }
76 MatchRuleList matchRules
;
77 RepositoryHash repositories
;
78 IdentityHash identities
;
80 SvnPrivate(const QString
&pathToRepository
);
82 int youngestRevision();
83 int exportRevision(int revnum
);
85 int openRepository(const QString
&pathToRepository
);
88 AprAutoPool global_pool
;
90 svn_revnum_t youngest_rev
;
93 void Svn::initialize()
95 // initialize APR or exit
96 if (apr_initialize() != APR_SUCCESS
) {
97 fprintf(stderr
, "You lose at apr_initialize().\n");
102 static struct Destructor
{ ~Destructor() { apr_terminate(); } } destructor
;
105 Svn::Svn(const QString
&pathToRepository
)
106 : d(new SvnPrivate(pathToRepository
))
115 void Svn::setMatchRules(const MatchRuleList
&matchRules
)
117 d
->matchRules
= matchRules
;
120 void Svn::setRepositories(const RepositoryHash
&repositories
)
122 d
->repositories
= repositories
;
125 void Svn::setIdentityMap(const IdentityHash
&identityMap
)
127 d
->identities
= identityMap
;
130 int Svn::youngestRevision()
132 return d
->youngestRevision();
135 bool Svn::exportRevision(int revnum
)
137 return d
->exportRevision(revnum
) == EXIT_SUCCESS
;
140 SvnPrivate::SvnPrivate(const QString
&pathToRepository
)
143 openRepository(pathToRepository
);
145 // get the youngest revision
146 svn_fs_youngest_rev(&youngest_rev
, fs
, global_pool
);
149 SvnPrivate::~SvnPrivate()
151 svn_pool_destroy(global_pool
);
154 int SvnPrivate::youngestRevision()
159 int SvnPrivate::openRepository(const QString
&pathToRepository
)
162 SVN_ERR(svn_repos_open(&repos
, QFile::encodeName(pathToRepository
), global_pool
));
163 fs
= svn_repos_fs(repos
);
168 enum RuleType
{ AnyRule
= 0, NoIgnoreRule
= 0x01, NoRecurseRule
= 0x02 };
170 static MatchRuleList::ConstIterator
171 findMatchRule(const MatchRuleList
&matchRules
, int revnum
, const QString
¤t
,
172 int ruleMask
= AnyRule
)
174 MatchRuleList::ConstIterator it
= matchRules
.constBegin(),
175 end
= matchRules
.constEnd();
176 for ( ; it
!= end
; ++it
) {
177 if (it
->minRevision
> revnum
)
179 if (it
->maxRevision
!= -1 && it
->maxRevision
< revnum
)
181 if (it
->action
== Rules::Match::Ignore
&& ruleMask
& NoIgnoreRule
)
183 if (it
->action
== Rules::Match::Recurse
&& ruleMask
& NoRecurseRule
)
185 if (it
->rx
.indexIn(current
) == 0)
193 static void splitPathName(const Rules::Match
&rule
, const QString
&pathName
, QString
*svnprefix_p
,
194 QString
*repository_p
, QString
*branch_p
, QString
*path_p
)
196 QString svnprefix
= pathName
;
197 svnprefix
.truncate(rule
.rx
.matchedLength());
199 *svnprefix_p
= svnprefix
;
202 *repository_p
= svnprefix
;
203 repository_p
->replace(rule
.rx
, rule
.repository
);
207 *branch_p
= svnprefix
;
208 branch_p
->replace(rule
.rx
, rule
.branch
);
212 *path_p
= pathName
.mid(svnprefix
.length());
215 static int pathMode(svn_fs_root_t
*fs_root
, const char *pathname
, apr_pool_t
*pool
)
217 svn_string_t
*propvalue
;
218 SVN_ERR(svn_fs_node_prop(&propvalue
, fs_root
, pathname
, "svn:executable", pool
));
226 svn_error_t
*QIODevice_write(void *baton
, const char *data
, apr_size_t
*len
)
228 QIODevice
*device
= reinterpret_cast<QIODevice
*>(baton
);
229 device
->write(data
, *len
);
231 while (device
->bytesToWrite() > 32*1024) {
232 if (!device
->waitForBytesWritten(-1)) {
233 qFatal("Failed to write to process: %s", qPrintable(device
->errorString()));
234 return svn_error_createf(APR_EOF
, SVN_NO_ERROR
, "Failed to write to process: %s",
235 qPrintable(device
->errorString()));
241 static svn_stream_t
*streamForDevice(QIODevice
*device
, apr_pool_t
*pool
)
243 svn_stream_t
*stream
= svn_stream_create(device
, pool
);
244 svn_stream_set_write(stream
, QIODevice_write
);
249 static int dumpBlob(Repository::Transaction
*txn
, svn_fs_root_t
*fs_root
,
250 const char *pathname
, const QString
&finalPathName
, apr_pool_t
*pool
)
252 AprAutoPool
dumppool(pool
);
254 int mode
= pathMode(fs_root
, pathname
, dumppool
);
256 svn_filesize_t stream_length
;
258 SVN_ERR(svn_fs_file_length(&stream_length
, fs_root
, pathname
, dumppool
));
262 svn_stream_t
*in_stream
, *out_stream
;
263 SVN_ERR(svn_fs_file_contents(&in_stream
, fs_root
, pathname
, dumppool
));
266 // maybe it's a symlink?
267 svn_string_t
*propvalue
;
268 SVN_ERR(svn_fs_node_prop(&propvalue
, fs_root
, pathname
, "svn:special", dumppool
));
270 apr_size_t len
= strlen("link ");
274 SVN_ERR(svn_stream_read(in_stream
, buf
.data(), &len
));
275 if (len
!= strlen("link ") || strncmp(buf
, "link ", len
) != 0)
276 qFatal("file %s is svn:special but not a symlink", pathname
);
279 stream_length
-= len
;
282 QIODevice
*io
= txn
->addFile(finalPathName
, mode
, stream_length
);
285 // open a generic svn_stream_t for the QIODevice
286 out_stream
= streamForDevice(io
, dumppool
);
287 SVN_ERR(svn_stream_copy(in_stream
, out_stream
, dumppool
));
288 svn_stream_close(out_stream
);
289 svn_stream_close(in_stream
);
291 // print an ending newline
298 static int recursiveDumpDir(Repository::Transaction
*txn
, svn_fs_root_t
*fs_root
,
299 const QByteArray
&pathname
, const QString
&finalPathName
,
302 // get the dir listing
304 SVN_ERR(svn_fs_dir_entries(&entries
, fs_root
, pathname
, pool
));
305 AprAutoPool
dirpool(pool
);
307 for (apr_hash_index_t
*i
= apr_hash_first(pool
, entries
); i
; i
= apr_hash_next(i
)) {
311 apr_hash_this(i
, &vkey
, NULL
, &value
);
313 svn_fs_dirent_t
*dirent
= reinterpret_cast<svn_fs_dirent_t
*>(value
);
314 QByteArray entryName
= pathname
+ '/' + dirent
->name
;
315 QString entryFinalName
= finalPathName
+ dirent
->name
;
317 if (dirent
->kind
== svn_node_dir
) {
318 entryFinalName
+= '/';
319 if (recursiveDumpDir(txn
, fs_root
, entryName
, entryFinalName
, dirpool
) == EXIT_FAILURE
)
321 } else if (dirent
->kind
== svn_node_file
) {
324 if (dumpBlob(txn
, fs_root
, entryName
, entryFinalName
, dirpool
) == EXIT_FAILURE
)
332 static bool wasDir(svn_fs_t
*fs
, int revnum
, const char *pathname
, apr_pool_t
*pool
)
334 AprAutoPool
subpool(pool
);
335 svn_fs_root_t
*fs_root
;
336 if (svn_fs_revision_root(&fs_root
, fs
, revnum
, subpool
) != SVN_NO_ERROR
)
339 svn_boolean_t is_dir
;
340 if (svn_fs_is_dir(&is_dir
, fs_root
, pathname
, subpool
) != SVN_NO_ERROR
)
346 time_t get_epoch(char *svn_date
)
349 memset(&tm
, 0, sizeof tm
);
350 QByteArray
date(svn_date
, strlen(svn_date
) - 8);
351 strptime(date
, "%Y-%m-%dT%H:%M:%S", &tm
);
359 QHash
<QString
, Repository::Transaction
*> transactions
;
360 MatchRuleList matchRules
;
361 RepositoryHash repositories
;
362 IdentityHash identities
;
365 svn_fs_root_t
*fs_root
;
368 // must call fetchRevProps first:
369 QByteArray authorident
;
373 SvnRevision(int revision
, svn_fs_t
*f
, apr_pool_t
*parent_pool
)
374 : pool(parent_pool
), fs(f
), fs_root(0), revnum(revision
)
380 SVN_ERR(svn_fs_revision_root(&fs_root
, fs
, revnum
, pool
));
384 int prepareTransactions();
388 int exportEntry(const char *path
, const svn_fs_path_change_t
*change
, apr_hash_t
*changes
);
389 int exportDispatch(const char *path
, const svn_fs_path_change_t
*change
,
390 const char *path_from
, svn_revnum_t rev_from
,
391 apr_hash_t
*changes
, const QString
¤t
, const Rules::Match
&rule
,
393 int exportInternal(const char *path
, const svn_fs_path_change_t
*change
,
394 const char *path_from
, svn_revnum_t rev_from
,
395 const QString
¤t
, const Rules::Match
&rule
);
396 int recurse(const char *path
, const svn_fs_path_change_t
*change
,
397 const char *path_from
, svn_revnum_t rev_from
,
398 apr_hash_t
*changes
, apr_pool_t
*pool
);
401 int SvnPrivate::exportRevision(int revnum
)
403 SvnRevision
rev(revnum
, fs
, global_pool
);
404 rev
.matchRules
= matchRules
;
405 rev
.repositories
= repositories
;
406 rev
.identities
= identities
;
408 // open this revision:
409 printf("Exporting revision %d ", revnum
);
412 if (rev
.open() == EXIT_FAILURE
)
415 if (rev
.prepareTransactions() == EXIT_FAILURE
)
418 if (rev
.transactions
.isEmpty()) {
419 printf(" nothing to do\n");
420 return EXIT_SUCCESS
; // no changes?
423 if (rev
.commit() == EXIT_FAILURE
)
430 int SvnRevision::prepareTransactions()
432 // find out what was changed in this revision:
434 SVN_ERR(svn_fs_paths_changed(&changes
, fs_root
, pool
));
435 for (apr_hash_index_t
*i
= apr_hash_first(pool
, changes
); i
; i
= apr_hash_next(i
)) {
438 apr_hash_this(i
, &vkey
, NULL
, &value
);
439 const char *key
= reinterpret_cast<const char *>(vkey
);
440 svn_fs_path_change_t
*change
= reinterpret_cast<svn_fs_path_change_t
*>(value
);
442 if (exportEntry(key
, change
, changes
) == EXIT_FAILURE
)
449 int SvnRevision::fetchRevProps()
451 apr_hash_t
*revprops
;
452 SVN_ERR(svn_fs_revision_proplist(&revprops
, fs
, revnum
, pool
));
453 svn_string_t
*svnauthor
= (svn_string_t
*)apr_hash_get(revprops
, "svn:author", APR_HASH_KEY_STRING
);
454 svn_string_t
*svndate
= (svn_string_t
*)apr_hash_get(revprops
, "svn:date", APR_HASH_KEY_STRING
);
455 svn_string_t
*svnlog
= (svn_string_t
*)apr_hash_get(revprops
, "svn:log", APR_HASH_KEY_STRING
);
457 log
= (char *)svnlog
->data
;
458 authorident
= svnauthor
? identities
.value((char *)svnauthor
->data
) : QByteArray();
459 epoch
= get_epoch((char*)svndate
->data
);
460 if (authorident
.isEmpty()) {
461 if (!svnauthor
|| svn_string_isempty(svnauthor
))
462 authorident
= "nobody <nobody@localhost>";
464 authorident
= svnauthor
->data
+ QByteArray(" <") +
465 svnauthor
->data
+ QByteArray("@localhost>");
470 int SvnRevision::commit()
472 // now create the commit
473 if (fetchRevProps() != EXIT_SUCCESS
)
475 foreach (Repository::Transaction
*txn
, transactions
) {
476 txn
->setAuthor(authorident
);
477 txn
->setDateTime(epoch
);
487 int SvnRevision::exportEntry(const char *key
, const svn_fs_path_change_t
*change
,
490 AprAutoPool
revpool(pool
.data());
491 QString current
= QString::fromUtf8(key
);
493 // was this copied from somewhere?
494 svn_revnum_t rev_from
;
495 const char *path_from
;
496 SVN_ERR(svn_fs_copied_from(&rev_from
, &path_from
, fs_root
, key
, revpool
));
498 // is this a directory?
499 svn_boolean_t is_dir
;
500 SVN_ERR(svn_fs_is_dir(&is_dir
, fs_root
, key
, revpool
));
502 if (path_from
== NULL
) {
503 // no, it's a new directory being added
504 // Git doesn't handle directories, so we don't either
505 //qDebug() << " mkdir ignored:" << key;
510 qDebug() << " " << key
<< "was copied from" << path_from
<< "rev" << rev_from
;
513 // find the first rule that matches this pathname
514 MatchRuleList::ConstIterator match
= findMatchRule(matchRules
, revnum
, current
);
515 if (match
!= matchRules
.constEnd()) {
516 const Rules::Match
&rule
= *match
;
517 return exportDispatch(key
, change
, path_from
, rev_from
, changes
, current
, rule
, revpool
);
520 if (is_dir
&& path_from
!= NULL
) {
521 qDebug() << current
<< "is a copy-with-history, auto-recursing";
522 return recurse(key
, change
, path_from
, rev_from
, changes
, revpool
);
523 } else if (wasDir(fs
, revnum
- 1, key
, revpool
)) {
524 qDebug() << current
<< "was a directory; ignoring";
525 } else if (change
->change_kind
== svn_fs_path_change_delete
) {
526 qDebug() << current
<< "is being deleted but I don't know anything about it; ignoring";
528 qCritical() << current
<< "did not match any rules; cannot continue";
535 int SvnRevision::exportDispatch(const char *key
, const svn_fs_path_change_t
*change
,
536 const char *path_from
, svn_revnum_t rev_from
,
537 apr_hash_t
*changes
, const QString
¤t
,
538 const Rules::Match
&rule
, apr_pool_t
*pool
)
540 switch (rule
.action
) {
541 case Rules::Match::Ignore
:
543 //qDebug() << " " << qPrintable(current) << "rev" << revnum
544 // << "-> ignored (rule" << rule << ")";
547 case Rules::Match::Recurse
:
548 return recurse(key
, change
, path_from
, rev_from
, changes
, pool
);
550 case Rules::Match::Export
:
551 return exportInternal(key
, change
, path_from
, rev_from
, current
, rule
);
558 int SvnRevision::exportInternal(const char *key
, const svn_fs_path_change_t
*change
,
559 const char *path_from
, svn_revnum_t rev_from
,
560 const QString
¤t
, const Rules::Match
&rule
)
562 QString svnprefix
, repository
, branch
, path
;
563 splitPathName(rule
, current
, &svnprefix
, &repository
, &branch
, &path
);
567 // qDebug() << " " << qPrintable(current) << "rev" << revnum << "->"
568 // << qPrintable(repository) << qPrintable(branch) << qPrintable(path);
570 if (path
.isEmpty() && path_from
!= NULL
) {
571 QString previous
= QString::fromUtf8(path_from
) + '/';
572 MatchRuleList::ConstIterator prevmatch
=
573 findMatchRule(matchRules
, rev_from
, previous
, NoIgnoreRule
);
574 if (prevmatch
!= matchRules
.constEnd()) {
575 QString prevsvnprefix
, prevrepository
, prevbranch
, prevpath
;
576 splitPathName(*prevmatch
, previous
, &prevsvnprefix
, &prevrepository
,
577 &prevbranch
, &prevpath
);
579 if (!prevpath
.isEmpty()) {
580 qDebug() << qPrintable(current
) << "is a partial branch of repository"
581 << qPrintable(prevrepository
) << "branch"
582 << qPrintable(prevbranch
) << "subdir"
583 << qPrintable(prevpath
);
584 } else if (prevrepository
!= repository
) {
585 qWarning() << qPrintable(current
) << "rev" << revnum
586 << "is a cross-repository copy (from repository"
587 << qPrintable(prevrepository
) << "branch"
588 << qPrintable(prevbranch
) << "path"
589 << qPrintable(prevpath
) << "rev" << rev_from
<< ")";
590 } else if (prevbranch
== branch
) {
591 // same branch and same repository
592 qDebug() << qPrintable(current
) << "rev" << revnum
593 << "is an SVN rename from"
594 << qPrintable(previous
) << "rev" << rev_from
;
597 // same repository but not same branch
598 // this means this is a plain branch
599 qDebug() << qPrintable(repository
) << ": branch"
600 << qPrintable(branch
) << "is branching from"
601 << qPrintable(prevbranch
);
603 Repository
*repo
= repositories
.value(repository
, 0);
605 qCritical() << "Rule" << rule
606 << "references unknown repository" << repository
;
610 repo
->createBranch(branch
, revnum
, prevbranch
, rev_from
);
612 // create an annotated tag
614 repo
->createAnnotatedTag(branch
, svnprefix
, revnum
, authorident
,
622 Repository::Transaction
*txn
= transactions
.value(repository
+ branch
, 0);
624 Repository
*repo
= repositories
.value(repository
, 0);
626 qCritical() << "Rule" << rule
627 << "references unknown repository" << repository
;
631 txn
= repo
->newTransaction(branch
, svnprefix
, revnum
);
635 transactions
.insert(repository
+ branch
, txn
);
638 if (change
->change_kind
== svn_fs_path_change_delete
) {
639 txn
->deleteFile(path
);
640 } else if (!current
.endsWith('/')) {
641 dumpBlob(txn
, fs_root
, key
, path
, pool
);
643 QString pathNoSlash
= path
;
645 txn
->deleteFile(pathNoSlash
);
646 recursiveDumpDir(txn
, fs_root
, key
, path
, pool
);
652 int SvnRevision::recurse(const char *path
, const svn_fs_path_change_t
*change
,
653 const char *path_from
, svn_revnum_t rev_from
,
654 apr_hash_t
*changes
, apr_pool_t
*pool
)
656 // get the dir listing
658 SVN_ERR(svn_fs_dir_entries(&entries
, fs_root
, path
, pool
));
660 AprAutoPool
dirpool(pool
);
661 for (apr_hash_index_t
*i
= apr_hash_first(pool
, entries
); i
; i
= apr_hash_next(i
)) {
665 apr_hash_this(i
, &vkey
, NULL
, &value
);
667 svn_fs_dirent_t
*dirent
= reinterpret_cast<svn_fs_dirent_t
*>(value
);
668 if (dirent
->kind
!= svn_node_dir
)
669 continue; // not a directory, so can't recurse; skip
671 QByteArray entry
= path
+ QByteArray("/") + dirent
->name
;
672 QByteArray entryFrom
;
674 entryFrom
= path_from
+ QByteArray("/") + dirent
->name
;
676 // check if this entry is in the changelist for this revision already
677 svn_fs_path_change_t
*otherchange
=
678 (svn_fs_path_change_t
*)apr_hash_get(changes
, entry
.constData(), APR_HASH_KEY_STRING
);
679 if (otherchange
&& otherchange
->change_kind
== svn_fs_path_change_add
) {
680 qDebug() << entry
<< "rev" << revnum
681 << "is in the change-list, deferring to that one";
685 QString current
= QString::fromUtf8(entry
);
686 if (dirent
->kind
== svn_node_dir
)
689 // find the first rule that matches this pathname
690 MatchRuleList::ConstIterator match
= findMatchRule(matchRules
, revnum
, current
);
691 if (match
!= matchRules
.constEnd()) {
692 if (exportDispatch(entry
, change
, entryFrom
.isNull() ? 0 : entryFrom
.constData(),
693 rev_from
, changes
, current
, *match
, dirpool
) == EXIT_FAILURE
)
696 qCritical() << current
<< "rev" << revnum
697 << "did not match any rules; cannot continue";