tests/auto/qfile/largefile/tst_largefile.cpp

   1 /****************************************************************************
   2 **
   3 ** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
   4 ** All rights reserved.
   5 ** Contact: Nokia Corporation (qt-info@nokia.com)
   6 **
   7 ** This file is part of the test suite of the Qt Toolkit.
   8 **
   9 ** $QT_BEGIN_LICENSE:LGPL$
  10 ** No Commercial Usage
  11 ** This file contains pre-release code and may not be distributed.
  12 ** You may use this file in accordance with the terms and conditions
  13 ** contained in the Technology Preview License Agreement accompanying
  14 ** this package.
  15 **
  16 ** GNU Lesser General Public License Usage
  17 ** Alternatively, this file may be used under the terms of the GNU Lesser
  18 ** General Public License version 2.1 as published by the Free Software
  19 ** Foundation and appearing in the file LICENSE.LGPL included in the
  20 ** packaging of this file.  Please review the following information to
  21 ** ensure the GNU Lesser General Public License version 2.1 requirements
  22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
  23 **
  24 ** In addition, as a special exception, Nokia gives you certain additional
  25 ** rights.  These rights are described in the Nokia Qt LGPL Exception
  26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
  27 **
  28 ** If you have questions regarding the use of this file, please contact
  29 ** Nokia at qt-info@nokia.com.
  30 **
  31 **
  32 **
  33 **
  34 **
  35 **
  36 **
  37 **
  38 ** $QT_END_LICENSE$
  39 **
  40 ****************************************************************************/
  41
  42 #include <QTest>
  43
  44 #include <QtAlgorithms>
  45 #include <QFile>
  46 #include <QFileInfo>
  47 #include <qplatformdefs.h>
  48
  49 #include <QDebug>
  50
  51 #include <cstdlib>
  52 #include <cstdio>
  53
  54 #ifdef Q_OS_WIN
  55
  56 #include <windows.h>
  57 #include <io.h>
  58
  59 #ifndef FSCTL_SET_SPARSE
  60 // MinGW doesn't define this.
  61 #define FSCTL_SET_SPARSE (0x900C4)
  62 #endif
  63
  64 #endif // Q_OS_WIN
  65
  66 class tst_LargeFile
  67     : public QObject
  68 {
  69     Q_OBJECT
  70
  71 public:
  72     tst_LargeFile()
  73         : blockSize(1 << 12)
  74         , maxSizeBits()
  75         , fd_(-1)
  76         , stream_(0)
  77     {
  78     #if defined(QT_LARGEFILE_SUPPORT) && !defined(Q_OS_MAC)
  79         maxSizeBits = 36; // 64 GiB
  80     #elif defined(Q_OS_MAC)
  81         // HFS+ does not support sparse files, so we limit file size for the test
  82         // on Mac OS.
  83         maxSizeBits = 32; // 4 GiB
  84     #else
  85         maxSizeBits = 24; // 16 MiB
  86     #endif
  87     }
  88
  89 private:
  90     void sparseFileData();
  91     QByteArray const &getDataBlock(int index, qint64 position);
  92
  93 private slots:
  94     // The LargeFile test case was designed to be run in order as a single unit
  95
  96     void initTestCase();
  97     void cleanupTestCase();
  98
  99     void init();
 100     void cleanup();
 101
 102     // Create and fill large file
 103     void createSparseFile();
 104     void fillFileSparsely();
 105     void closeSparseFile();
 106
 107     // Verify file was created
 108     void fileCreated();
 109
 110     // Positioning in large files
 111     void filePositioning();
 112     void fdPositioning();
 113     void streamPositioning();
 114
 115     // Read data from file
 116     void openFileForReading();
 117     void readFile();
 118
 119     // Map/unmap large file
 120     void mapFile();
 121     void mapOffsetOverflow();
 122
 123     void closeFile() { largeFile.close(); }
 124
 125     // Test data
 126     void fillFileSparsely_data() { sparseFileData(); }
 127     void filePositioning_data() { sparseFileData(); }
 128     void fdPositioning_data() { sparseFileData(); }
 129     void streamPositioning_data() { sparseFileData(); }
 130     void readFile_data() { sparseFileData(); }
 131     void mapFile_data() { sparseFileData(); }
 132
 133 private:
 134     const int blockSize;
 135     int maxSizeBits;
 136
 137     QFile largeFile;
 138
 139     QVector<QByteArray> generatedBlocks;
 140
 141     int fd_;
 142     FILE *stream_;
 143 };
 144
 145 /*
 146     Convenience function to hide reinterpret_cast when copying a POD directly
 147     into a QByteArray.
 148  */
 149 template <class T>
 150 static inline void appendRaw(QByteArray &array, T data)
 151 {
 152     array.append(reinterpret_cast<char *>(&data), sizeof(T));
 153 }
 154
 155 /*
 156     Pad array with filler up to size. On return, array.size() returns size.
 157  */
 158 static inline void topUpWith(QByteArray &array, QByteArray filler, int size)
 159 {
 160     Q_ASSERT(filler.size() > 0);
 161
 162     for (int i = (size - array.size()) / filler.size(); i > 0; --i)
 163         array.append(filler);
 164
 165     if (array.size() < size) {
 166         Q_ASSERT(size - array.size() < filler.size());
 167         array.append(filler.left(size - array.size()));
 168     }
 169 }
 170
 171 /*
 172    Generate a unique data block containing identifiable data. Unaligned,
 173    overlapping and partial blocks should not compare equal.
 174  */
 175 static inline QByteArray generateDataBlock(int blockSize, QString text, qint64 userBits = -1)
 176 {
 177     QByteArray block;
 178     block.reserve(blockSize);
 179
 180     // Use of counter and randomBits means content of block will be dependent
 181     // on the generation order. For (file-)systems that do not support sparse
 182     // files, these can be removed so the test file can be reused and doesn't
 183     // have to be generated for every run.
 184
 185     static qint64 counter = 0;
 186
 187     qint64 randomBits = ((qint64)qrand() << 32)
 188             | ((qint64)qrand() & 0x00000000ffffffff);
 189
 190     appendRaw(block, randomBits);
 191     appendRaw(block, userBits);
 192     appendRaw(block, counter);
 193     appendRaw(block, (qint32)0xdeadbeef);
 194     appendRaw(block, blockSize);
 195
 196     QByteArray userContent = text.toUtf8();
 197     appendRaw(block, userContent.size());
 198     block.append(userContent);
 199     appendRaw(block, (qint64)0);
 200
 201     // size, so far
 202     appendRaw(block, block.size());
 203
 204     QByteArray filler("0123456789");
 205     block.append(filler.right(10 - block.size() % 10));
 206     topUpWith(block, filler, blockSize - 2 * sizeof(qint64));
 207
 208     appendRaw(block, counter);
 209     appendRaw(block, userBits);
 210     appendRaw(block, randomBits);
 211
 212     Q_ASSERT( block.size() >= blockSize );
 213     block.resize(blockSize);
 214
 215     ++counter;
 216     return block;
 217 }
 218
 219 /*
 220    Generates data blocks the first time they are requested. Keeps copies for reuse.
 221  */
 222 QByteArray const &tst_LargeFile::getDataBlock(int index, qint64 position)
 223 {
 224     if (index >= generatedBlocks.size())
 225         generatedBlocks.resize(index + 1);
 226
 227     if (generatedBlocks[index].isNull()) {
 228         QString text = QString("Current %1-byte block (index = %2) "
 229             "starts %3 bytes into the file '%4'.")
 230                 .arg(blockSize)
 231                 .arg(index)
 232                 .arg(position)
 233                 .arg("qt_largefile.tmp");
 234
 235         generatedBlocks[index] = generateDataBlock(blockSize, text, (qint64)1 << index);
 236     }
 237
 238     return generatedBlocks[index];
 239 }
 240
 241 void tst_LargeFile::initTestCase()
 242 {
 243     QFile file("qt_largefile.tmp");
 244     QVERIFY( !file.exists() || file.remove() );
 245 }
 246
 247 void tst_LargeFile::cleanupTestCase()
 248 {
 249     if (largeFile.isOpen())
 250         largeFile.close();
 251
 252     QFile file("qt_largefile.tmp");
 253     QVERIFY( !file.exists() || file.remove() );
 254 }
 255
 256 void tst_LargeFile::init()
 257 {
 258     fd_ = -1;
 259     stream_ = 0;
 260 }
 261
 262 void tst_LargeFile::cleanup()
 263 {
 264     if (-1 != fd_)
 265         QT_CLOSE(fd_);
 266     if (stream_)
 267         ::fclose(stream_);
 268 }
 269
 270 void tst_LargeFile::sparseFileData()
 271 {
 272     QTest::addColumn<int>("index");
 273     QTest::addColumn<qint64>("position");
 274     QTest::addColumn<QByteArray>("block");
 275
 276     QTest::newRow(QString("block[%1] @%2)")
 277             .arg(0).arg(0)
 278             .toLocal8Bit().constData())
 279         << 0 << (qint64)0 << getDataBlock(0, 0);
 280
 281     // While on Linux sparse files scale well, on Windows, testing at every
 282     // power of 2 leads to very large files. i += 4 gives us a good coverage
 283     // without taxing too much on resources.
 284     for (int index = 12; index <= maxSizeBits; index += 4) {
 285         qint64 position = (qint64)1 << index;
 286         QByteArray block = getDataBlock(index, position);
 287
 288         QTest::newRow(
 289             QString("block[%1] @%2)")
 290                 .arg(index).arg(position)
 291                 .toLocal8Bit().constData())
 292             << index << position << block;
 293     }
 294 }
 295
 296 void tst_LargeFile::createSparseFile()
 297 {
 298 #if defined(Q_OS_WIN)
 299     // On Windows platforms, we must explicitly set the file to be sparse,
 300     // so disk space is not allocated for the full file when writing to it.
 301     HANDLE handle = ::CreateFileA("qt_largefile.tmp",
 302         GENERIC_WRITE, 0, 0, CREATE_ALWAYS, 0, 0);
 303     QVERIFY( INVALID_HANDLE_VALUE != handle );
 304
 305     DWORD bytes;
 306     if (!::DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0,
 307             &bytes, NULL)) {
 308         QWARN("Unable to set test file as sparse. "
 309             "Limiting test file to 16MiB.");
 310         maxSizeBits = 24;
 311     }
 312
 313     int fd = ::_open_osfhandle((intptr_t)handle, 0);
 314     QVERIFY( -1 != fd );
 315     QVERIFY( largeFile.open(fd, QIODevice::WriteOnly | QIODevice::Unbuffered) );
 316 #else // !Q_OS_WIN
 317     largeFile.setFileName("qt_largefile.tmp");
 318     QVERIFY( largeFile.open(QIODevice::WriteOnly | QIODevice::Unbuffered) );
 319 #endif
 320 }
 321
 322 void tst_LargeFile::closeSparseFile()
 323 {
 324 #if defined(Q_OS_WIN)
 325     int fd = largeFile.handle();
 326 #endif
 327
 328     largeFile.close();
 329
 330 #if defined(Q_OS_WIN)
 331     if (-1 != fd)
 332         ::_close(fd);
 333 #endif
 334 }
 335
 336 void tst_LargeFile::fillFileSparsely()
 337 {
 338     QFETCH( qint64, position );
 339     QFETCH( QByteArray, block );
 340     QCOMPARE( block.size(), blockSize );
 341
 342     QVERIFY( largeFile.seek(position) );
 343     QCOMPARE( largeFile.pos(), position );
 344
 345     QCOMPARE( largeFile.write(block), (qint64)blockSize );
 346     QCOMPARE( largeFile.pos(), position + blockSize );
 347     QVERIFY( largeFile.flush() );
 348 }
 349
 350 void tst_LargeFile::fileCreated()
 351 {
 352     QFileInfo info("qt_largefile.tmp");
 353
 354     QVERIFY( info.exists() );
 355     QVERIFY( info.isFile() );
 356     QVERIFY( info.size() >= ((qint64)1 << maxSizeBits) + blockSize );
 357 }
 358
 359 void tst_LargeFile::filePositioning()
 360 {
 361     QFETCH( qint64, position );
 362
 363     QFile file("qt_largefile.tmp");
 364     QVERIFY( file.open(QIODevice::ReadOnly) );
 365
 366     QVERIFY( file.seek(position) );
 367     QCOMPARE( file.pos(), position );
 368 }
 369
 370 void tst_LargeFile::fdPositioning()
 371 {
 372     QFETCH( qint64, position );
 373
 374     fd_ = QT_OPEN("qt_largefile.tmp",
 375             QT_OPEN_RDONLY | QT_OPEN_LARGEFILE);
 376     QVERIFY( -1 != fd_ );
 377
 378     QFile file;
 379     QVERIFY( file.open(fd_, QIODevice::ReadOnly) );
 380     QCOMPARE( file.pos(), (qint64)0 );
 381     QVERIFY( file.seek(position) );
 382     QCOMPARE( file.pos(), position );
 383
 384     file.close();
 385
 386     QCOMPARE( QT_LSEEK(fd_, QT_OFF_T(0), SEEK_SET), QT_OFF_T(0) );
 387     QCOMPARE( QT_LSEEK(fd_, QT_OFF_T(position), SEEK_SET), QT_OFF_T(position) );
 388
 389     QVERIFY( file.open(fd_, QIODevice::ReadOnly) );
 390     QCOMPARE( QT_LSEEK(fd_, QT_OFF_T(0), SEEK_CUR), QT_OFF_T(position) );
 391     QCOMPARE( file.pos(), position );
 392     QVERIFY( file.seek(0) );
 393     QCOMPARE( file.pos(), (qint64)0 );
 394
 395     file.close();
 396
 397     QVERIFY( !QT_CLOSE(fd_) );
 398     fd_ = -1;
 399 }
 400
 401 void tst_LargeFile::streamPositioning()
 402 {
 403     QFETCH( qint64, position );
 404
 405 #if defined(QT_LARGEFILE_SUPPORT) && defined(Q_CC_MSVC) && _MSC_VER < 1400
 406     if (position >= (qint64)1 << 31)
 407         QSKIP("MSVC 2003 doesn't have 64 bit versions of fseek/ftell.", SkipSingle);
 408 #endif
 409
 410     stream_ = QT_FOPEN("qt_largefile.tmp", "rb");
 411     QVERIFY( 0 != stream_ );
 412
 413     QFile file;
 414     QVERIFY( file.open(stream_, QIODevice::ReadOnly) );
 415     QCOMPARE( file.pos(), (qint64)0 );
 416     QVERIFY( file.seek(position) );
 417     QCOMPARE( file.pos(), position );
 418
 419     file.close();
 420
 421     QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(0), SEEK_SET) );
 422     QCOMPARE( QT_FTELL(stream_), QT_OFF_T(0) );
 423     QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(position), SEEK_SET) );
 424     QCOMPARE( QT_FTELL(stream_), QT_OFF_T(position) );
 425
 426     QVERIFY( file.open(stream_, QIODevice::ReadOnly) );
 427     QCOMPARE( QT_FTELL(stream_), QT_OFF_T(position) );
 428     QCOMPARE( file.pos(), position );
 429     QVERIFY( file.seek(0) );
 430     QCOMPARE( file.pos(), (qint64)0 );
 431
 432     file.close();
 433
 434     QVERIFY( !::fclose(stream_) );
 435     stream_ = 0;
 436 }
 437
 438 void tst_LargeFile::openFileForReading()
 439 {
 440     largeFile.setFileName("qt_largefile.tmp");
 441     QVERIFY( largeFile.open(QIODevice::ReadOnly) );
 442 }
 443
 444 void tst_LargeFile::readFile()
 445 {
 446     QFETCH( qint64, position );
 447     QFETCH( QByteArray, block );
 448     QCOMPARE( block.size(), blockSize );
 449
 450     QVERIFY( largeFile.size() >= position + blockSize );
 451
 452     QVERIFY( largeFile.seek(position) );
 453     QCOMPARE( largeFile.pos(), position );
 454
 455     QCOMPARE( largeFile.read(blockSize), block );
 456     QCOMPARE( largeFile.pos(), position + blockSize );
 457 }
 458
 459 void tst_LargeFile::mapFile()
 460 {
 461     QFETCH( qint64, position );
 462     QFETCH( QByteArray, block );
 463     QCOMPARE( block.size(), blockSize );
 464
 465     // Keep full block mapped to facilitate OS and/or internal reuse by Qt.
 466     uchar *baseAddress = largeFile.map(position, blockSize);
 467     QVERIFY( baseAddress );
 468     QVERIFY( qEqual(block.begin(), block.end(), reinterpret_cast<char*>(baseAddress)) );
 469
 470     for (int offset = 1; offset <  blockSize; ++offset) {
 471         uchar *address = largeFile.map(position + offset, blockSize - offset);
 472
 473         QVERIFY( address );
 474         if ( !qEqual(block.begin() + offset, block.end(), reinterpret_cast<char*>(address)) ) {
 475             qDebug() << "Expected:" << block.toHex();
 476             qDebug() << "Actual  :" << QByteArray(reinterpret_cast<char*>(address), blockSize).toHex();
 477             QVERIFY(false);
 478         }
 479
 480         QVERIFY( largeFile.unmap( address ) );
 481     }
 482
 483     QVERIFY( largeFile.unmap( baseAddress ) );
 484 }
 485
 486 void tst_LargeFile::mapOffsetOverflow()
 487 {
 488     // Out-of-range mappings should fail, and not silently clip the offset
 489     for (int i = 50; i < 63; ++i) {
 490         uchar *address = 0;
 491
 492         address = largeFile.map(((qint64)1 << i), blockSize);
 493         QVERIFY( !address );
 494
 495         address = largeFile.map(((qint64)1 << i) + blockSize, blockSize);
 496         QVERIFY( !address );
 497     }
 498 }
 499
 500 QTEST_APPLESS_MAIN(tst_LargeFile)
 501 #include "tst_largefile.moc"
 502