lib/filestorage/tests/tgz_packer_test.php

   1 <?php
   2 // This file is part of Moodle - http://moodle.org/
   3 //
   4 // Moodle is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU General Public License as published by
   6 // the Free Software Foundation, either version 3 of the License, or
   7 // (at your option) any later version.
   8 //
   9 // Moodle is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU General Public License
  15 // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 /**
  18  * Unit tests for /lib/filestorage/tgz_packer.php and tgz_extractor.php.
  19  *
  20  * @package core_files
  21  * @copyright 2013 The Open University
  22  * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23  */
  24
  25 defined('MOODLE_INTERNAL') || die();
  26
  27 global $CFG;
  28 require_once($CFG->libdir . '/filestorage/file_progress.php');
  29
  30 class core_files_tgz_packer_testcase extends advanced_testcase implements file_progress {
  31     /**
  32      * @var array Progress information passed to the progress reporter
  33      */
  34     protected $progress;
  35
  36     /**
  37      * Puts contents with specified time.
  38      *
  39      * @param string $path File path
  40      * @param string $contents Contents of file
  41      * @param int $mtime Time modified
  42      */
  43     protected static function file_put_contents_at_time($path, $contents, $mtime) {
  44         file_put_contents($path, $contents);
  45         touch($path, $mtime);
  46     }
  47
  48     /**
  49      * Set up some files to be archived.
  50      *
  51      * @return array Array listing files of all types
  52      */
  53     protected function prepare_file_list() {
  54         global $CFG;
  55         $this->resetAfterTest(true);
  56
  57         // Make array listing files to archive.
  58         $filelist = array();
  59
  60         // Normal file.
  61         self::file_put_contents_at_time($CFG->tempdir . '/file1.txt', 'File 1', 1377993601);
  62         $filelist['out1.txt'] = $CFG->tempdir . '/file1.txt';
  63
  64         // Recursive directory w/ file and directory with file.
  65         check_dir_exists($CFG->tempdir . '/dir1/dir2');
  66         self::file_put_contents_at_time($CFG->tempdir . '/dir1/file2.txt', 'File 2', 1377993602);
  67         self::file_put_contents_at_time($CFG->tempdir . '/dir1/dir2/file3.txt', 'File 3', 1377993603);
  68         $filelist['out2'] = $CFG->tempdir . '/dir1';
  69
  70         // Moodle stored_file.
  71         $context = context_system::instance();
  72         $filerecord = array('contextid' => $context->id, 'component' => 'phpunit',
  73                 'filearea' => 'data', 'itemid' => 0, 'filepath' => '/',
  74                 'filename' => 'file4.txt', 'timemodified' => 1377993604);
  75         $fs = get_file_storage();
  76         $sf = $fs->create_file_from_string($filerecord, 'File 4');
  77         $filelist['out3.txt'] = $sf;
  78
  79          // Moodle stored_file directory.
  80         $filerecord['itemid'] = 1;
  81         $filerecord['filepath'] = '/dir1/';
  82         $filerecord['filename'] = 'file5.txt';
  83         $filerecord['timemodified'] = 1377993605;
  84         $fs->create_file_from_string($filerecord, 'File 5');
  85         $filerecord['filepath'] = '/dir1/dir2/';
  86         $filerecord['filename'] = 'file6.txt';
  87         $filerecord['timemodified'] = 1377993606;
  88         $fs->create_file_from_string($filerecord, 'File 6');
  89         $filerecord['filepath'] = '/';
  90         $filerecord['filename'] = 'excluded.txt';
  91         $fs->create_file_from_string($filerecord, 'Excluded');
  92         $filelist['out4'] = $fs->get_file($context->id, 'phpunit', 'data', 1, '/dir1/', '.');
  93
  94         // File stored as raw content.
  95         $filelist['out5.txt'] = array('File 7');
  96
  97         // File where there's just an empty directory.
  98         $filelist['out6'] = null;
  99
 100         return $filelist;
 101     }
 102
 103     /**
 104      * Tests getting the item.
 105      */
 106     public function test_get_packer() {
 107         $packer = get_file_packer('application/x-gzip');
 108         $this->assertInstanceOf('tgz_packer', $packer);
 109     }
 110
 111     /**
 112      * Tests basic archive and extract to file paths.
 113      */
 114     public function test_to_normal_files() {
 115         global $CFG;
 116         $packer = get_file_packer('application/x-gzip');
 117
 118         // Archive files.
 119         $files = $this->prepare_file_list();
 120         $archivefile = $CFG->tempdir . '/test.tar.gz';
 121         $packer->archive_to_pathname($files, $archivefile);
 122
 123         // Extract same files.
 124         $outdir = $CFG->tempdir . '/out';
 125         check_dir_exists($outdir);
 126         $result = $packer->extract_to_pathname($archivefile, $outdir);
 127
 128         // The result array should have file entries + directory entries for
 129         // all implicit directories + entry for the explicit directory.
 130         $expectedpaths = array('out1.txt', 'out2/', 'out2/dir2/', 'out2/dir2/file3.txt',
 131                 'out2/file2.txt', 'out3.txt', 'out4/', 'out4/dir2/', 'out4/file5.txt',
 132                 'out4/dir2/file6.txt', 'out5.txt', 'out6/');
 133         sort($expectedpaths);
 134         $actualpaths = array_keys($result);
 135         sort($actualpaths);
 136         $this->assertEquals($expectedpaths, $actualpaths);
 137         foreach ($result as $path => $booleantrue) {
 138             $this->assertTrue($booleantrue);
 139         }
 140
 141         // Check the files are as expected.
 142         $this->assertEquals('File 1', file_get_contents($outdir . '/out1.txt'));
 143         $this->assertEquals('File 2', file_get_contents($outdir . '/out2/file2.txt'));
 144         $this->assertEquals('File 3', file_get_contents($outdir . '/out2/dir2/file3.txt'));
 145         $this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
 146         $this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
 147         $this->assertEquals('File 6', file_get_contents($outdir . '/out4/dir2/file6.txt'));
 148         $this->assertEquals('File 7', file_get_contents($outdir . '/out5.txt'));
 149         $this->assertTrue(is_dir($outdir . '/out6'));
 150     }
 151
 152     /**
 153      * Tests archive and extract to Moodle file system.
 154      */
 155     public function test_to_stored_files() {
 156         global $CFG;
 157         $packer = get_file_packer('application/x-gzip');
 158
 159         // Archive files.
 160         $files = $this->prepare_file_list();
 161         $archivefile = $CFG->tempdir . '/test.tar.gz';
 162         $context = context_system::instance();
 163         $sf = $packer->archive_to_storage($files,
 164                 $context->id, 'phpunit', 'archive', 1, '/', 'archive.tar.gz');
 165         $this->assertInstanceOf('stored_file', $sf);
 166
 167         // Extract (from storage) to disk.
 168         $outdir = $CFG->tempdir . '/out';
 169         check_dir_exists($outdir);
 170         $packer->extract_to_pathname($sf, $outdir);
 171
 172         // Check the files are as expected.
 173         $this->assertEquals('File 1', file_get_contents($outdir . '/out1.txt'));
 174         $this->assertEquals('File 2', file_get_contents($outdir . '/out2/file2.txt'));
 175         $this->assertEquals('File 3', file_get_contents($outdir . '/out2/dir2/file3.txt'));
 176         $this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
 177         $this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
 178         $this->assertEquals('File 6', file_get_contents($outdir . '/out4/dir2/file6.txt'));
 179         $this->assertEquals('File 7', file_get_contents($outdir . '/out5.txt'));
 180         $this->assertTrue(is_dir($outdir . '/out6'));
 181
 182         // Extract to Moodle storage.
 183         $packer->extract_to_storage($sf, $context->id, 'phpunit', 'data', 2, '/out/');
 184         $fs = get_file_storage();
 185         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out1.txt');
 186         $this->assertNotEmpty($out);
 187         $this->assertEquals('File 1', $out->get_content());
 188         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out2/', 'file2.txt');
 189         $this->assertNotEmpty($out);
 190         $this->assertEquals('File 2', $out->get_content());
 191         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out2/dir2/', 'file3.txt');
 192         $this->assertNotEmpty($out);
 193         $this->assertEquals('File 3', $out->get_content());
 194         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out3.txt');
 195         $this->assertNotEmpty($out);
 196         $this->assertEquals('File 4', $out->get_content());
 197         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out4/', 'file5.txt');
 198         $this->assertNotEmpty($out);
 199         $this->assertEquals('File 5', $out->get_content());
 200         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out4/dir2/', 'file6.txt');
 201         $this->assertNotEmpty($out);
 202         $this->assertEquals('File 6', $out->get_content());
 203         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/', 'out5.txt');
 204         $this->assertNotEmpty($out);
 205         $this->assertEquals('File 7', $out->get_content());
 206         $out = $fs->get_file($context->id, 'phpunit', 'data', 2, '/out/out6/', '.');
 207         $this->assertNotEmpty($out);
 208         $this->assertTrue($out->is_directory());
 209
 210         // These functions are supposed to overwrite existing files; test they
 211         // don't give errors when run twice.
 212         $sf = $packer->archive_to_storage($files,
 213                 $context->id, 'phpunit', 'archive', 1, '/', 'archive.tar.gz');
 214         $this->assertInstanceOf('stored_file', $sf);
 215         $packer->extract_to_storage($sf, $context->id, 'phpunit', 'data', 2, '/out/');
 216     }
 217
 218     /**
 219      * Tests extracting with a list of specified files.
 220      */
 221     public function test_only_specified_files() {
 222         global $CFG;
 223         $packer = get_file_packer('application/x-gzip');
 224
 225         // Archive files.
 226         $files = $this->prepare_file_list();
 227         $archivefile = $CFG->tempdir . '/test.tar.gz';
 228         $packer->archive_to_pathname($files, $archivefile);
 229
 230         // Extract same files.
 231         $outdir = $CFG->tempdir . '/out';
 232         check_dir_exists($outdir);
 233         $result = $packer->extract_to_pathname($archivefile, $outdir,
 234                 array('out3.txt', 'out6/', 'out4/file5.txt'));
 235
 236         // Check result reporting only includes specified files.
 237         $expectedpaths = array('out3.txt', 'out4/file5.txt', 'out6/');
 238         sort($expectedpaths);
 239         $actualpaths = array_keys($result);
 240         sort($actualpaths);
 241         $this->assertEquals($expectedpaths, $actualpaths);
 242
 243         // Check the files are as expected.
 244         $this->assertFalse(file_exists($outdir . '/out1.txt'));
 245         $this->assertEquals('File 4', file_get_contents($outdir . '/out3.txt'));
 246         $this->assertEquals('File 5', file_get_contents($outdir . '/out4/file5.txt'));
 247         $this->assertTrue(is_dir($outdir . '/out6'));
 248     }
 249
 250     /**
 251      * Tests extracting files returning only a boolean state with success.
 252      */
 253     public function test_extract_to_pathname_returnvalue_successful() {
 254         $packer = get_file_packer('application/x-gzip');
 255
 256         // Prepare files.
 257         $files = $this->prepare_file_list();
 258         $archivefile = make_request_directory() . '/test.tgz';
 259         $packer->archive_to_pathname($files, $archivefile);
 260
 261         // Extract same files.
 262         $outdir = make_request_directory();
 263         $result = $packer->extract_to_pathname($archivefile, $outdir, null, null, true);
 264
 265         $this->assertTrue($result);
 266     }
 267
 268     /**
 269      * Tests extracting files returning only a boolean state with failure.
 270      */
 271     public function test_extract_to_pathname_returnvalue_failure() {
 272         $packer = get_file_packer('application/x-gzip');
 273
 274         // Create sample files.
 275         $archivefile = make_request_directory() . '/test.tgz';
 276         file_put_contents($archivefile, '');
 277
 278         // Extract same files.
 279         $outdir = make_request_directory();
 280
 281         $result = $packer->extract_to_pathname($archivefile, $outdir, null, null, true);
 282
 283         $this->assertFalse($result);
 284     }
 285
 286     /**
 287      * Tests the progress reporting.
 288      */
 289     public function test_file_progress() {
 290         global $CFG;
 291
 292         // Set up.
 293         $filelist = $this->prepare_file_list();
 294         $packer = get_file_packer('application/x-gzip');
 295         $archive = "$CFG->tempdir/archive.tgz";
 296         $context = context_system::instance();
 297
 298         // Archive to pathname.
 299         $this->progress = array();
 300         $result = $packer->archive_to_pathname($filelist, $archive, true, $this);
 301         $this->assertTrue($result);
 302         // Should send progress at least once per file.
 303         $this->assertTrue(count($this->progress) >= count($filelist));
 304         // Progress should obey some restrictions.
 305         $this->check_progress_toward_max();
 306
 307         // Archive to storage.
 308         $this->progress = array();
 309         $archivefile = $packer->archive_to_storage($filelist, $context->id,
 310                 'phpunit', 'test', 0, '/', 'archive.tgz', null, true, $this);
 311         $this->assertInstanceOf('stored_file', $archivefile);
 312         $this->assertTrue(count($this->progress) >= count($filelist));
 313         $this->check_progress_toward_max();
 314
 315         // Extract to pathname.
 316         $this->progress = array();
 317         $target = "$CFG->tempdir/test/";
 318         check_dir_exists($target);
 319         $result = $packer->extract_to_pathname($archive, $target, null, $this);
 320         remove_dir($target);
 321         // We only output progress once per block, and this is kind of a small file.
 322         $this->assertTrue(count($this->progress) >= 1);
 323         $this->check_progress_toward_max();
 324
 325         // Extract to storage (from storage).
 326         $this->progress = array();
 327         $result = $packer->extract_to_storage($archivefile, $context->id,
 328                 'phpunit', 'target', 0, '/', null, $this);
 329         $this->assertTrue(count($this->progress) >= 1);
 330         $this->check_progress_toward_max();
 331
 332         // Extract to storage (from path).
 333         $this->progress = array();
 334         $result = $packer->extract_to_storage($archive, $context->id,
 335                 'phpunit', 'target', 0, '/', null, $this);
 336         $this->assertTrue(count($this->progress) >= 1);
 337         $this->check_progress_toward_max();
 338
 339         // Wipe created disk file.
 340         unlink($archive);
 341     }
 342
 343     /**
 344      * Tests the list_files function with and without an index file.
 345      */
 346     public function test_list_files() {
 347         global $CFG;
 348
 349         // Set up.
 350         $filelist = $this->prepare_file_list();
 351         $packer = get_file_packer('application/x-gzip');
 352         $archive = "$CFG->tempdir/archive.tgz";
 353
 354         // Archive with an index (default).
 355         $packer = get_file_packer('application/x-gzip');
 356         $result = $packer->archive_to_pathname($filelist, $archive, true, $this);
 357         $this->assertTrue($result);
 358         $hashwith = file_storage::hash_from_path($archive);
 359
 360         // List files.
 361         $files = $packer->list_files($archive);
 362
 363         // Check they match expected.
 364         $expectedinfo = array(
 365             array('out1.txt', 1377993601, false, 6),
 366             array('out2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
 367             array('out2/dir2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
 368             array('out2/dir2/file3.txt', 1377993603, false, 6),
 369             array('out2/file2.txt', 1377993602, false, 6),
 370             array('out3.txt', 1377993604, false, 6),
 371             array('out4/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
 372             array('out4/dir2/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
 373             array('out4/dir2/file6.txt', 1377993606, false, 6),
 374             array('out4/file5.txt', 1377993605, false, 6),
 375             array('out5.txt', tgz_packer::DEFAULT_TIMESTAMP, false, 6),
 376             array('out6/', tgz_packer::DEFAULT_TIMESTAMP, true, 0),
 377         );
 378         $this->assertEquals($expectedinfo, self::convert_info_for_assert($files));
 379
 380         // Archive with no index. Should have same result.
 381         $this->progress = array();
 382         $packer->set_include_index(false);
 383         $result = $packer->archive_to_pathname($filelist, $archive, true, $this);
 384         $this->assertTrue($result);
 385         $hashwithout = file_storage::hash_from_path($archive);
 386         $files = $packer->list_files($archive);
 387         $this->assertEquals($expectedinfo, self::convert_info_for_assert($files));
 388
 389         // Check it actually is different (does have index in)!
 390         $this->assertNotEquals($hashwith, $hashwithout);
 391
 392         // Put the index back on in case of future tests.
 393         $packer->set_include_index(true);
 394     }
 395
 396     /**
 397      * Utility function to convert the file info array into a simpler format
 398      * for making comparisons.
 399      *
 400      * @param array $files Array from list_files result
 401      */
 402     protected static function convert_info_for_assert(array $files) {
 403         $actualinfo = array();
 404         foreach ($files as $file) {
 405             $actualinfo[] = array($file->pathname, $file->mtime, $file->is_directory, $file->size);
 406         }
 407         usort($actualinfo, function($a, $b) {
 408             return strcmp($a[0], $b[0]);
 409         });
 410         return $actualinfo;
 411     }
 412
 413     public function test_is_tgz_file() {
 414         global $CFG;
 415
 416         // Set up.
 417         $filelist = $this->prepare_file_list();
 418         $packer1 = get_file_packer('application/x-gzip');
 419         $packer2 = get_file_packer('application/zip');
 420         $archive2 = "$CFG->tempdir/archive.zip";
 421
 422         // Archive in tgz and zip format.
 423         $context = context_system::instance();
 424         $archive1 = $packer1->archive_to_storage($filelist, $context->id,
 425                 'phpunit', 'test', 0, '/', 'archive.tgz', null, true, $this);
 426         $this->assertInstanceOf('stored_file', $archive1);
 427         $result = $packer2->archive_to_pathname($filelist, $archive2);
 428         $this->assertTrue($result);
 429
 430         // Use is_tgz_file to detect which is which. First check is from storage,
 431         // second check is from filesystem.
 432         $this->assertTrue(tgz_packer::is_tgz_file($archive1));
 433         $this->assertFalse(tgz_packer::is_tgz_file($archive2));
 434     }
 435
 436     /**
 437      * Checks that progress reported is numeric rather than indeterminate,
 438      * and follows the progress reporting rules.
 439      */
 440     protected function check_progress_toward_max() {
 441         $lastvalue = -1; $lastmax = -1;
 442         foreach ($this->progress as $progressitem) {
 443             list($value, $max) = $progressitem;
 444             if ($lastmax != -1) {
 445                 $this->assertEquals($max, $lastmax);
 446             } else {
 447                 $lastmax = $max;
 448             }
 449             $this->assertTrue(is_integer($value));
 450             $this->assertTrue(is_integer($max));
 451             $this->assertNotEquals(file_progress::INDETERMINATE, $max);
 452             $this->assertTrue($value <= $max);
 453             $this->assertTrue($value >= $lastvalue);
 454             $lastvalue = $value;
 455         }
 456     }
 457
 458     /**
 459      * Handles file_progress interface.
 460      *
 461      * @param int $progress
 462      * @param int $max
 463      */
 464     public function progress($progress = file_progress::INDETERMINATE, $max = file_progress::INDETERMINATE) {
 465         $this->progress[] = array($progress, $max);
 466     }
 467 }