src/repository.cpp

   1 /*
   2  *  Copyright (C) 2007  Thiago Macieira <thiago@kde.org>
   3  *
   4  *  This program is free software: you can redistribute it and/or modify
   5  *  it under the terms of the GNU General Public License as published by
   6  *  the Free Software Foundation, either version 2 of the License, or
   7  *  (at your option) any later version.
   8  *
   9  *  This program is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13  *
  14  *  You should have received a copy of the GNU General Public License
  15  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include "repository.h"
  19 #include <QCoreApplication>
  20 #include <QTextStream>
  21 #include <QDebug>
  22 #include <QLinkedList>
  23
  24 static const int maxSimultaneousProcesses = 100;
  25
  26 class ProcessCache: QLinkedList<Repository *>
  27 {
  28 public:
  29     void touch(Repository *repo)
  30     {
  31         remove(repo);
  32
  33         // if the cache is too big, remove from the front
  34         while (size() >= maxSimultaneousProcesses)
  35             takeFirst()->closeFastImport();
  36
  37         // append to the end
  38         append(repo);
  39     }
  40
  41     inline void remove(Repository *repo)
  42     {
  43         removeOne(repo);
  44     }
  45 };
  46 static ProcessCache processCache;
  47
  48 Repository::Repository(const Rules::Repository &rule)
  49     : name(rule.name), commitCount(0), processHasStarted(false)
  50 {
  51     foreach (Rules::Repository::Branch branchRule, rule.branches) {
  52         Branch branch;
  53         branch.created = 0;     // not created
  54
  55         branches.insert(branchRule.name, branch);
  56     }
  57
  58     // create the default branch
  59     branches["master"].created = 1;
  60 }
  61
  62 Repository::~Repository()
  63 {
  64     closeFastImport();
  65     closeFixup();
  66 }
  67
  68 void Repository::closeFastImport()
  69 {
  70     if (fastImport.state() != QProcess::NotRunning) {
  71         fastImport.write("checkpoint\n");
  72         fastImport.waitForBytesWritten(-1);
  73         fastImport.closeWriteChannel();
  74         if (!fastImport.waitForFinished()) {
  75             fastImport.terminate();
  76             if (!fastImport.waitForFinished(200))
  77                 qWarning() << "git-fast-import for repository" << name << "did not die";
  78         }
  79     }
  80     processHasStarted = false;
  81     processCache.remove(this);
  82 }
  83
  84 void Repository::reloadBranches()
  85 {
  86     QProcess revParse;
  87     revParse.setWorkingDirectory(name);
  88     revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
  89     revParse.waitForFinished(-1);
  90
  91     if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
  92         while (revParse.canReadLine()) {
  93             QByteArray branchName = revParse.readLine().trimmed();
  94
  95             //qDebug() << "Repo" << name << "reloaded branch" << branchName;
  96             branches[branchName].created = 1;
  97             fastImport.write("reset refs/heads/" + branchName +
  98                              "\nfrom refs/heads/" + branchName + "^0\n\n"
  99                              "progress Branch refs/heads/" + branchName + " reloaded\n");
 100         }
 101     }
 102 }
 103
 104 bool Repository::branchExists(const QString &branch)
 105 {
 106     startFastImport();
 107     QHash<QString, Branch>::ConstIterator it = branches.constFind(branch);
 108     return it != branches.constEnd() && it->created > 0;
 109 }
 110
 111 void Repository::createBranch(const QString &branch, int revnum,
 112                               const QString &branchFrom, int)
 113 {
 114     startFastImport();
 115     if (!branches.contains(branch)) {
 116         qWarning() << branch << "is not a known branch in repository" << name << endl
 117                    << "Going to create it automatically";
 118     }
 119
 120     QByteArray branchRef = branch.toUtf8();
 121     if (!branchRef.startsWith("refs/"))
 122         branchRef.prepend("refs/heads/");
 123
 124     Branch &br = branches[branch];
 125     if (br.created && br.created != revnum) {
 126         QByteArray backupBranch = branchRef + '_' + QByteArray::number(revnum);
 127         qWarning() << branch << "already exists; backing up to" << backupBranch;
 128
 129         fastImport.write("reset " + backupBranch + "\nfrom " + branchRef + "\n\n");
 130     }
 131
 132     // now create the branch
 133     br.created = revnum;
 134     QByteArray branchFromRef = branchFrom.toUtf8();
 135     if (!branchFromRef.startsWith("refs/"))
 136         branchFromRef.prepend("refs/heads/");
 137
 138     if (!branches.contains(branchFrom) || !branches.value(branchFrom).created) {
 139         qCritical() << branch << "in repository" << name
 140                     << "is branching from branch" << branchFrom
 141                     << "but the latter doesn't exist. Can't continue.";
 142         exit(1);
 143     }
 144
 145     fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n"
 146         "progress Branch " + branchRef + " created from " + branchFromRef + "\n\n");
 147 }
 148
 149 void Repository::startFastImport()
 150 {
 151     if (fastImport.state() == QProcess::NotRunning) {
 152         closeFixup();
 153         if (processHasStarted)
 154             qFatal("git-fast-import has been started once and crashed?");
 155         processHasStarted = true;
 156
 157         // start the process
 158         QString outputFile = name;
 159         outputFile.replace('/', '_');
 160         outputFile.prepend("log-");
 161         fastImport.setStandardOutputFile(outputFile, QIODevice::Append);
 162         fastImport.setProcessChannelMode(QProcess::MergedChannels);
 163         fastImport.setWorkingDirectory(name);
 164
 165 #ifndef DRY_RUN
 166         fastImport.start("git", QStringList() << "fast-import");
 167 #else
 168         fastImport.start("/bin/cat", QStringList());
 169 #endif
 170
 171         reloadBranches();
 172     }
 173 }
 174
 175 void Repository::executeFixups(const QList<Fixup> &fixups, const QByteArray &branch)
 176 {
 177     closeFastImport();
 178
 179     // same as above:
 180     QString outputFile = name;
 181     outputFile.replace('/', '_');
 182     outputFile.prepend("log-");
 183     fixupProcess.setStandardOutputFile(outputFile, QIODevice::Append);
 184     fixupProcess.setProcessChannelMode(QProcess::MergedChannels);
 185
 186 #ifndef DRY_RUN
 187     fixupProcess.start(QCoreApplication::applicationDirPath() + "/scripts/fixup",
 188                        QStringList() << name << branch);
 189 #else
 190     fixupProcess.start("/bin/cat", QStringList());
 191 #endif
 192
 193     {
 194         QTextStream s(&fixupProcess);
 195         foreach (const Fixup &f, fixups) {
 196             if (!f.valid)
 197                 continue;
 198             f.source->fastImport.write("checkpoint\n");
 199             while (f.source->fastImport.bytesToWrite() &&
 200                    f.source->fastImport.waitForBytesWritten(-1))
 201                 ;
 202             s << "fixup " << f.source->name << ' ' << f.branch << ' '
 203               << f.rev << ':' << f.path << endl;
 204         }
 205     }
 206
 207     // flush
 208     fixupProcess.closeWriteChannel();
 209     while (fixupProcess.bytesToWrite())
 210         if (!fixupProcess.waitForBytesWritten(-1))
 211             qFatal("Failed to write to process: %s", qPrintable(fixupProcess.errorString()));
 212 }
 213
 214 void Repository::closeFixup()
 215 {
 216     if (fixupProcess.state() != QProcess::NotRunning) {
 217         // wait for the fixup process to finish running
 218         if (!fixupProcess.waitForFinished(-1) ||
 219             fixupProcess.exitCode() != 0)
 220             qFatal("Fixup script crashed?");
 221     }
 222 }
 223
 224 Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix,
 225                                                     int revnum)
 226 {
 227     startFastImport();
 228     if (!branches.contains(branch)) {
 229         qWarning() << branch << "is not a known branch in repository" << name << endl
 230                    << "Going to create it automatically";
 231     }
 232
 233     Transaction *txn = new Transaction;
 234     txn->repository = this;
 235     txn->branch = branch.toUtf8();
 236     txn->svnprefix = svnprefix.toUtf8();
 237     txn->datetime = 0;
 238     txn->revnum = revnum;
 239     txn->lastmark = revnum;
 240
 241     if ((++commitCount % 10000) == 0)
 242         // write everything to disk every 10000 commits
 243         fastImport.write("checkpoint\n");
 244     return txn;
 245 }
 246
 247 Repository::Transaction::~Transaction()
 248 {
 249 }
 250
 251 void Repository::Transaction::addFixup(const Fixup &fixup)
 252 {
 253     Q_ASSERT(fixup.source != repository);
 254
 255     QList<Fixup>::Iterator it = fixups.begin();
 256     for ( ; it != fixups.end(); ++it) {
 257         if (fixup.source == it->source && fixup.branch == it->branch) {
 258             // fixup for this repository-branch already exists
 259             if (!it->valid)
 260                 return;         // can't fix an invalid
 261
 262             if (fixup.rev > it->rev)
 263                 it->rev = fixup.rev;
 264
 265             // figure out the common path between the two
 266             int lastSlash = -1;
 267             int max = qMin(fixup.path.length(), it->path.length());
 268             const QChar *p1 = fixup.path.constData(),
 269                         *p2 = it->path.constData();
 270             for (int i = 0; p1[i] == p2[i] && i < max; ++i)
 271                 if (p1[i].unicode() == '/')
 272                     lastSlash = i;
 273
 274             it->path.truncate(lastSlash + 1);
 275             it->valid = lastSlash != -1;
 276             return;
 277         }
 278     }
 279
 280     // new fixup from this repository-branch
 281     fixups.append(fixup);
 282 }
 283
 284 void Repository::Transaction::addMergeBranch(const QString &branch)
 285 {
 286     mergeBranches.insert(branch.toUtf8());
 287 }
 288
 289 void Repository::Transaction::setAuthor(const QByteArray &a)
 290 {
 291     author = a;
 292 }
 293
 294 void Repository::Transaction::setDateTime(uint dt)
 295 {
 296     datetime = dt;
 297 }
 298
 299 void Repository::Transaction::setLog(const QByteArray &l)
 300 {
 301     log = l;
 302 }
 303
 304 void Repository::Transaction::deleteFile(const QString &path)
 305 {
 306     deletedFiles.append(path);
 307 }
 308
 309 QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length)
 310 {
 311     FileProperties fp;
 312     fp.mode = mode;
 313     fp.mark = ++lastmark;
 314
 315 #ifndef DRY_RUN
 316     repository->fastImport.write("blob\nmark :");
 317     repository->fastImport.write(QByteArray::number(fp.mark));
 318     repository->fastImport.write("\ndata ");
 319     repository->fastImport.write(QByteArray::number(length));
 320     repository->fastImport.write("\n", 1);
 321 #endif
 322
 323     modifiedFiles.insert(path, fp);
 324     return &repository->fastImport;
 325 }
 326
 327 void Repository::Transaction::commit()
 328 {
 329     processCache.touch(repository);
 330
 331     // create the commit message
 332     QByteArray message = log;
 333     if (!message.endsWith('\n'))
 334         message += '\n';
 335     message += "\nsvn path=" + svnprefix + "; revision=" + QByteArray::number(revnum) + "\n";
 336
 337     {
 338         QByteArray branchRef = branch;
 339         if (!branchRef.startsWith("refs/"))
 340             branchRef.prepend("refs/heads/");
 341
 342         QTextStream s(&repository->fastImport);
 343         s << "commit " << branchRef << endl;
 344         foreach (const QByteArray &mb, mergeBranches)
 345             s << "merge "
 346               << (mb.startsWith("refs/") ? "" : "refs/heads/")
 347               << mb
 348               << endl;
 349         s << "mark :" << revnum << endl;
 350         s << "committer " << QString::fromUtf8(author) << ' ' << datetime << " -0000" << endl;
 351
 352         Branch &br = repository->branches[branch];
 353         if (!br.created) {
 354             qWarning() << "Branch" << branch << "in repository" << repository->name << "doesn't exist at revision"
 355                        << revnum << "-- did you resume from the wrong revision?";
 356             br.created = revnum;
 357         }
 358
 359         s << "data " << message.length() << endl;
 360     }
 361
 362     repository->startFastImport();
 363     repository->fastImport.write(message);
 364     repository->fastImport.putChar('\n');
 365
 366     // write the file deletions
 367     if (deletedFiles.contains(""))
 368         repository->fastImport.write("deleteall\n");
 369     else
 370         foreach (QString df, deletedFiles)
 371             repository->fastImport.write("D " + df.toUtf8() + "\n");
 372
 373     // write the file modifications
 374     QHash<QString, FileProperties>::ConstIterator it = modifiedFiles.constBegin();
 375     for ( ; it != modifiedFiles.constEnd(); ++it) {
 376         repository->fastImport.write("M ", 2);
 377         repository->fastImport.write(QByteArray::number(it->mode, 8));
 378         repository->fastImport.write(" :", 2);
 379         repository->fastImport.write(QByteArray::number(it->mark));
 380         repository->fastImport.write(" ", 1);
 381         repository->fastImport.write(it.key().toUtf8());
 382         repository->fastImport.write("\n", 1);
 383     }
 384
 385     repository->fastImport.write("\nprogress Commit #" +
 386                                  QByteArray::number(repository->commitCount) +
 387                                  " branch " + branch +
 388                                  " = SVN r" + QByteArray::number(revnum) + "\n\n");
 389     printf(" %d modifications to \"%s\"",
 390            deletedFiles.count() + modifiedFiles.count(),
 391            qPrintable(repository->name));
 392
 393     while (repository->fastImport.bytesToWrite())
 394         if (!repository->fastImport.waitForBytesWritten(-1))
 395             qFatal("Failed to write to process: %s", qPrintable(repository->fastImport.errorString()));
 396
 397     // do we have to do fixups now?
 398     if (!fixups.isEmpty()) {
 399         printf(" (fixup required)");
 400         repository->executeFixups(fixups, branch);
 401     }
 402 }