2 Description: async stream reader, used to load repository data on startup
4 Author: Marco Costalba (C) 2005-2007
6 Copyright: See COPYING file that comes with this distribution
10 #include <QTemporaryFile>
11 #include "FileHistory.h"
13 #include "dataloader.h"
15 #define GUI_UPDATE_INTERVAL 500
16 #define READ_BLOCK_SIZE 65535
18 class UnbufferedTemporaryFile
: public QTemporaryFile
{
20 explicit UnbufferedTemporaryFile(QObject
* p
) : QTemporaryFile(p
) {}
21 bool unbufOpen() { return open(QIODevice::ReadOnly
| QIODevice::Unbuffered
); }
24 DataLoader::DataLoader(Git
* g
, FileHistory
* f
) : QProcess(g
), git(g
), fh(f
) {
26 canceling
= parsing
= false;
31 guiUpdateTimer
.setSingleShot(true);
33 connect(git
, SIGNAL(cancelAllProcesses()), this, SLOT(on_cancel()));
34 connect(&guiUpdateTimer
, SIGNAL(timeout()), this, SLOT(on_timeout()));
37 DataLoader::~DataLoader() {
39 // avoid a Qt warning in case we are
40 // destroyed while still running
41 waitForFinished(1000);
44 void DataLoader::on_cancel(const FileHistory
* f
) {
50 void DataLoader::on_cancel() {
52 if (!canceling
) { // just once
54 kill(); // SIGKILL (Unix and Mac), TerminateProcess (Windows)
58 bool DataLoader::start(SCList args
, SCRef wd
, SCRef buf
) {
61 dbs("ASSERT in DataLoader::start(), called while processing");
65 setWorkingDirectory(wd
);
67 connect(this, SIGNAL(finished(int, QProcess::ExitStatus
)),
68 this, SLOT(on_finished(int, QProcess::ExitStatus
)));
70 if (!createTemporaryFile() || !QGit::startProcess(this, args
, buf
)) {
75 guiUpdateTimer
.start(GUI_UPDATE_INTERVAL
);
79 void DataLoader::on_finished(int, QProcess::ExitStatus
) {
83 if (parsing
&& guiUpdateTimer
.isActive())
84 dbs("ASSERT in DataLoader: timer active while parsing");
86 if (parsing
== guiUpdateTimer
.isActive() && !canceling
)
87 dbs("ASSERT in DataLoader: inconsistent timer");
89 if (guiUpdateTimer
.isActive()) // no need to wait anymore
90 guiUpdateTimer
.start(1);
93 void DataLoader::on_timeout() {
97 return; // we leave with guiUpdateTimer not active
101 // process could exit while we are processing so save the flag now
102 bool lastBuffer
= isProcExited
;
103 loadedBytes
+= readNewData(lastBuffer
);
104 emit
newDataReady(fh
); // inserting in list view is about 3% of total time
107 emit
loaded(fh
, loadedBytes
, loadTime
.elapsed(), true, "", "");
110 } else if (isProcExited
) { // exited while parsing
111 dbs("Exited while parsing!!!!");
112 guiUpdateTimer
.start(1);
114 guiUpdateTimer
.start(GUI_UPDATE_INTERVAL
);
119 void DataLoader::parseSingleBuffer(const QByteArray
& ba
) {
121 if (ba
.size() == 0 || canceling
)
124 int ofs
= 0, newOfs
, bz
= ba
.size();
126 /* Due to unknown reasons randomly first byte
127 * of 'ba' is 0, this seems to happen only when
128 * using QFile::read(), i.e. with temporary file
129 * interface. Until we discover the real reason
130 * workaround this skipping the bogus byte
132 if (ba
.at(0) == 0 && bz
> 1 && !halfChunk
)
135 while (bz
- ofs
> 0) {
139 newOfs
= git
->addChunk(fh
, ba
, ofs
);
141 break; // half chunk detected
145 } else { // less then 1% of cases with READ_BLOCK_SIZE = 64KB
147 int end
= ba
.indexOf('\0');
148 if (end
== -1) // consecutives half chunks
152 baAppend(&halfChunk
, ba
.constData(), ofs
);
153 fh
->rowData
.append(halfChunk
);
154 addSplittedChunks(halfChunk
);
158 // save any remaining half chunk
160 baAppend(&halfChunk
, ba
.constData() + ofs
, bz
- ofs
);
163 void DataLoader::addSplittedChunks(const QByteArray
* hc
) {
165 if (hc
->at(hc
->size() - 1) != 0) {
166 dbs("ASSERT in DataLoader, bad half chunk");
169 // do not assume we have only one chunk in hc
171 while (ofs
!= -1 && ofs
!= (int)hc
->size())
172 ofs
= git
->addChunk(fh
, *hc
, ofs
);
175 void DataLoader::baAppend(QByteArray
** baPtr
, const char* ascii
, int len
) {
178 // we cannot use QByteArray::append(const char*)
179 // because 'ascii' is not '\0' terminating
180 (*baPtr
)->append(QByteArray::fromRawData(ascii
, len
));
182 *baPtr
= new QByteArray(ascii
, len
);
185 // *************** git interface facility dependant code *****************************
189 ulong
DataLoader::readNewData(bool lastBuffer
) {
192 QByteArray copy c'tor uses shallow copy, but there is a deep copy in
193 QProcess::readStdout(), from an internal buffers list to return value.
195 Qt uses a select() to detect new data is ready, copies immediately the
196 data to the heap with a read() and stores the pointer to new data in a
197 pointer list, from qprocess_unix.cpp:
199 const int basize = 4096;
200 QByteArray *ba = new QByteArray(basize);
201 n = ::read(fd, ba->data(), basize);
202 buffer->append(ba); // added to a QPtrList<QByteArray> pointer list
204 When we call QProcess::readStdout() data from buffers pointed by the
205 pointer list is memcpy() to the function return value, from qprocess.cpp:
208 return buf->readAll(); // memcpy() here
210 QByteArray
* ba
= new QByteArray(readAllStandardOutput());
212 ba
->append('\0'); // be sure stream is null terminated
214 if (ba
->size() == 0) {
218 fh
->rowData
.append(ba
);
219 parseSingleBuffer(*ba
);
223 bool DataLoader::createTemporaryFile() { return true; }
225 #else // temporary file as data exchange facility
227 ulong
DataLoader::readNewData(bool lastBuffer
) {
229 bool ok
= dataFile
&&
230 (dataFile
->isOpen() || (dataFile
->exists() && dataFile
->unbufOpen()));
236 qint64 readPos
= dataFile
->pos();
239 // this is the ONLY deep copy involved in the whole loading
240 // QFile::read() calls standard C read() function when
241 // file is open with Unbuffered flag, or fread() otherwise
242 QByteArray
* ba
= new QByteArray();
243 ba
->resize(READ_BLOCK_SIZE
);
244 int len
= static_cast<int>(dataFile
->read(ba
->data(), READ_BLOCK_SIZE
));
250 } else if (len
< ba
->size()) // unlikely
253 // current read position must be updated manually, it's
254 // not correctly incremented by read() if the producer
255 // process has already finished
257 dataFile
->seek(readPos
);
260 fh
->rowData
.append(ba
);
261 parseSingleBuffer(*ba
);
263 // avoid reading small chunks if data producer is still running
264 if (len
< READ_BLOCK_SIZE
&& !lastBuffer
)
267 if (lastBuffer
) { // be sure stream is null terminated
268 QByteArray
* zb
= new QByteArray(1, '\0');
269 fh
->rowData
.append(zb
);
270 parseSingleBuffer(*zb
);
275 bool DataLoader::createTemporaryFile() {
277 // redirect 'git log' output to a temporary file
278 dataFile
= new UnbufferedTemporaryFile(this);
282 For performance reasons we would like to use a tmpfs filesystem
283 if available, this is normally mounted under '/tmp' in Linux.
285 According to Qt docs, a temporary file is placed in QDir::tempPath(),
286 that should be system's temporary directory. On Unix/Linux systems this
287 is usually /tmp; on Windows this is usually the path in the TEMP or TMP
288 environment variable.
290 But due to a bug in Qt 4.2 QDir::tempPath() is instead set to $HOME/tmp
291 under Unix/Linux, that is not a tmpfs filesystem.
293 So try to manually set the best directory for our temporary file.
296 bool foundTmpDir
= (dir
.exists() && dir
.isReadable());
297 if (foundTmpDir
&& dir
.absolutePath() != QDir::tempPath()) {
299 dataFile
->setFileTemplate(dir
.absolutePath() + "/qt_temp");
300 if (!dataFile
->open()) { // test for write access
303 dataFile
= new UnbufferedTemporaryFile(this);
304 dbs("WARNING: directory '/tmp' is not writable, "
305 "fallback on Qt default one, there could "
306 "be a performance penalty.");
311 if (!dataFile
->open()) // to read the file name
314 setStandardOutputFile(dataFile
->fileName());
319 #endif // USE_QPROCESS