src/dataloader.cpp

   1 /*
   2         Description: async stream reader, used to load repository data on startup
   3
   4         Author: Marco Costalba (C) 2005-2007
   5
   6         Copyright: See COPYING file that comes with this distribution
   7
   8 */
   9 #include <QDir>
  10 #include <QTemporaryFile>
  11 #include "FileHistory.h"
  12 #include "git.h"
  13 #include "dataloader.h"
  14
  15 #define GUI_UPDATE_INTERVAL 500
  16 #define READ_BLOCK_SIZE     65535
  17
  18 class UnbufferedTemporaryFile : public QTemporaryFile {
  19 public:
  20         explicit UnbufferedTemporaryFile(QObject* p) : QTemporaryFile(p) {}
  21         bool unbufOpen() { return open(QIODevice::ReadOnly | QIODevice::Unbuffered); }
  22 };
  23
  24 DataLoader::DataLoader(Git* g, FileHistory* f) : QProcess(g), git(g), fh(f) {
  25
  26         canceling = parsing = false;
  27         isProcExited = true;
  28         halfChunk = NULL;
  29         dataFile = NULL;
  30         loadedBytes = 0;
  31         guiUpdateTimer.setSingleShot(true);
  32
  33         connect(git, SIGNAL(cancelAllProcesses()), this, SLOT(on_cancel()));
  34         connect(&guiUpdateTimer, SIGNAL(timeout()), this, SLOT(on_timeout()));
  35 }
  36
  37 DataLoader::~DataLoader() {
  38
  39         // avoid a Qt warning in case we are
  40         // destroyed while still running
  41         waitForFinished(1000);
  42 }
  43
  44 void DataLoader::on_cancel(const FileHistory* f) {
  45
  46         if (f == fh)
  47                 on_cancel();
  48 }
  49
  50 void DataLoader::on_cancel() {
  51
  52         if (!canceling) { // just once
  53                 canceling = true;
  54                 kill(); // SIGKILL (Unix and Mac), TerminateProcess (Windows)
  55         }
  56 }
  57
  58 bool DataLoader::start(SCList args, SCRef wd, SCRef buf) {
  59
  60         if (!isProcExited) {
  61                 dbs("ASSERT in DataLoader::start(), called while processing");
  62                 return false;
  63         }
  64         isProcExited = false;
  65         setWorkingDirectory(wd);
  66
  67         connect(this, SIGNAL(finished(int, QProcess::ExitStatus)),
  68                 this, SLOT(on_finished(int, QProcess::ExitStatus)));
  69
  70         if (!createTemporaryFile() || !QGit::startProcess(this, args, buf)) {
  71                 deleteLater();
  72                 return false;
  73         }
  74         loadTime.start();
  75         guiUpdateTimer.start(GUI_UPDATE_INTERVAL);
  76         return true;
  77 }
  78
  79 void DataLoader::on_finished(int, QProcess::ExitStatus) {
  80
  81         isProcExited = true;
  82
  83         if (parsing && guiUpdateTimer.isActive())
  84                 dbs("ASSERT in DataLoader: timer active while parsing");
  85
  86         if (parsing == guiUpdateTimer.isActive() && !canceling)
  87                 dbs("ASSERT in DataLoader: inconsistent timer");
  88
  89         if (guiUpdateTimer.isActive()) // no need to wait anymore
  90                 guiUpdateTimer.start(1);
  91 }
  92
  93 void DataLoader::on_timeout() {
  94
  95         if (canceling) {
  96                 deleteLater();
  97                 return; // we leave with guiUpdateTimer not active
  98         }
  99         parsing = true;
 100
 101         // process could exit while we are processing so save the flag now
 102         bool lastBuffer = isProcExited;
 103         loadedBytes += readNewData(lastBuffer);
 104         emit newDataReady(fh); // inserting in list view is about 3% of total time
 105
 106         if (lastBuffer) {
 107                 emit loaded(fh, loadedBytes, loadTime.elapsed(), true, "", "");
 108                 deleteLater();
 109
 110         } else if (isProcExited) { // exited while parsing
 111                 dbs("Exited while parsing!!!!");
 112                 guiUpdateTimer.start(1);
 113         } else
 114                 guiUpdateTimer.start(GUI_UPDATE_INTERVAL);
 115
 116         parsing = false;
 117 }
 118
 119 void DataLoader::parseSingleBuffer(const QByteArray& ba) {
 120
 121         if (ba.size() == 0 || canceling)
 122                 return;
 123
 124         int ofs = 0, newOfs, bz = ba.size();
 125
 126         /* Due to unknown reasons randomly first byte
 127          * of 'ba' is 0, this seems to happen only when
 128          * using QFile::read(), i.e. with temporary file
 129          * interface. Until we discover the real reason
 130          * workaround this skipping the bogus byte
 131          */
 132         if (ba.at(0) == 0 && bz > 1 && !halfChunk)
 133                 ofs++;
 134
 135         while (bz - ofs > 0) {
 136
 137                 if (!halfChunk) {
 138
 139                         newOfs = git->addChunk(fh, ba, ofs);
 140                         if (newOfs == -1)
 141                                 break; // half chunk detected
 142
 143                         ofs = newOfs;
 144
 145                 } else { // less then 1% of cases with READ_BLOCK_SIZE = 64KB
 146
 147                         int end = ba.indexOf('\0');
 148                         if (end == -1) // consecutives half chunks
 149                                 break;
 150
 151                         ofs = end + 1;
 152                         baAppend(&halfChunk, ba.constData(), ofs);
 153                         fh->rowData.append(halfChunk);
 154                         addSplittedChunks(halfChunk);
 155                         halfChunk = NULL;
 156                 }
 157         }
 158         // save any remaining half chunk
 159         if (bz - ofs > 0)
 160                 baAppend(&halfChunk, ba.constData() + ofs,  bz - ofs);
 161 }
 162
 163 void DataLoader::addSplittedChunks(const QByteArray* hc) {
 164
 165         if (hc->at(hc->size() - 1) != 0) {
 166                 dbs("ASSERT in DataLoader, bad half chunk");
 167                 return;
 168         }
 169         // do not assume we have only one chunk in hc
 170         int ofs = 0;
 171         while (ofs != -1 && ofs != (int)hc->size())
 172                 ofs = git->addChunk(fh, *hc, ofs);
 173 }
 174
 175 void DataLoader::baAppend(QByteArray** baPtr, const char* ascii, int len) {
 176
 177         if (*baPtr)
 178                 // we cannot use QByteArray::append(const char*)
 179                 // because 'ascii' is not '\0' terminating
 180                 (*baPtr)->append(QByteArray::fromRawData(ascii, len));
 181         else
 182                 *baPtr = new QByteArray(ascii, len);
 183 }
 184
 185 // *************** git interface facility dependant code *****************************
 186
 187 #ifdef USE_QPROCESS
 188
 189 ulong DataLoader::readNewData(bool lastBuffer) {
 190
 191         /*
 192            QByteArray copy c'tor uses shallow copy, but there is a deep copy in
 193            QProcess::readStdout(), from an internal buffers list to return value.
 194
 195            Qt uses a select() to detect new data is ready, copies immediately the
 196            data to the heap with a read() and stores the pointer to new data in a
 197            pointer list, from qprocess_unix.cpp:
 198
 199                 const int basize = 4096;
 200                 QByteArray *ba = new QByteArray(basize);
 201                 n = ::read(fd, ba->data(), basize);
 202                 buffer->append(ba); // added to a QPtrList<QByteArray> pointer list
 203
 204            When we call QProcess::readStdout() data from buffers pointed by the
 205            pointer list is memcpy() to the function return value, from qprocess.cpp:
 206
 207                 ....
 208                 return buf->readAll(); // memcpy() here
 209         */
 210         QByteArray* ba = new QByteArray(readAllStandardOutput());
 211         if (lastBuffer)
 212                 ba->append('\0'); // be sure stream is null terminated
 213
 214         if (ba->size() == 0) {
 215                 delete ba;
 216                 return 0;
 217         }
 218         fh->rowData.append(ba);
 219         parseSingleBuffer(*ba);
 220         return ba->size();
 221 }
 222
 223 bool DataLoader::createTemporaryFile() { return true; }
 224
 225 #else // temporary file as data exchange facility
 226
 227 ulong DataLoader::readNewData(bool lastBuffer) {
 228
 229         bool ok = dataFile &&
 230                  (dataFile->isOpen() || (dataFile->exists() && dataFile->unbufOpen()));
 231
 232         if (!ok)
 233                 return 0;
 234
 235         ulong cnt = 0;
 236         qint64 readPos = dataFile->pos();
 237
 238         while (true) {
 239                 // this is the ONLY deep copy involved in the whole loading
 240                 // QFile::read() calls standard C read() function when
 241                 // file is open with Unbuffered flag, or fread() otherwise
 242                 QByteArray* ba = new QByteArray();
 243                 ba->resize(READ_BLOCK_SIZE);
 244                 int len = dataFile->read(ba->data(), READ_BLOCK_SIZE);
 245
 246                 if (len <= 0) {
 247                         delete ba;
 248                         break;
 249
 250                 } else if (len < ba->size()) // unlikely
 251                         ba->resize(len);
 252
 253                 // current read position must be updated manually, it's
 254                 // not correctly incremented by read() if the producer
 255                 // process has already finished
 256                 readPos += len;
 257                 dataFile->seek(readPos);
 258
 259                 cnt += len;
 260                 fh->rowData.append(ba);
 261                 parseSingleBuffer(*ba);
 262
 263                 // avoid reading small chunks if data producer is still running
 264                 if (len < READ_BLOCK_SIZE && !lastBuffer)
 265                         break;
 266         }
 267         if (lastBuffer) { // be sure stream is null terminated
 268                 QByteArray* zb = new QByteArray(1, '\0');
 269                 fh->rowData.append(zb);
 270                 parseSingleBuffer(*zb);
 271         }
 272         return cnt;
 273 }
 274
 275 bool DataLoader::createTemporaryFile() {
 276
 277         // redirect 'git log' output to a temporary file
 278         dataFile = new UnbufferedTemporaryFile(this);
 279
 280 #ifndef Q_OS_WIN32
 281         /*
 282            For performance reasons we would like to use a tmpfs filesystem
 283            if available, this is normally mounted under '/tmp' in Linux.
 284
 285            According to Qt docs, a temporary file is placed in QDir::tempPath(),
 286            that should be system's temporary directory. On Unix/Linux systems this
 287            is usually /tmp; on Windows this is usually the path in the TEMP or TMP
 288            environment variable.
 289
 290            But due to a bug in Qt 4.2 QDir::tempPath() is instead set to $HOME/tmp
 291            under Unix/Linux, that is not a tmpfs filesystem.
 292
 293            So try to manually set the best directory for our temporary file.
 294         */
 295                 QDir dir("/tmp");
 296                 bool foundTmpDir = (dir.exists() && dir.isReadable());
 297                 if (foundTmpDir && dir.absolutePath() != QDir::tempPath()) {
 298
 299                         dataFile->setFileTemplate(dir.absolutePath() + "/qt_temp");
 300                         if (!dataFile->open()) { // test for write access
 301
 302                                 delete dataFile;
 303                                 dataFile = new UnbufferedTemporaryFile(this);
 304                                 dbs("WARNING: directory '/tmp' is not writable, "
 305                                     "fallback on Qt default one, there could "
 306                                     "be a performance penalty.");
 307                         } else
 308                                 dataFile->close();
 309                 }
 310 #endif
 311         if (!dataFile->open()) // to read the file name
 312                 return false;
 313
 314         setStandardOutputFile(dataFile->fileName());
 315         dataFile->close();
 316         return true;
 317 }
 318
 319 #endif // USE_QPROCESS