mod/data/tests/entries_import_test.php

   1 <?php
   2 // This file is part of Moodle - http://moodle.org/
   3 //
   4 // Moodle is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU General Public License as published by
   6 // the Free Software Foundation, either version 3 of the License, or
   7 // (at your option) any later version.
   8 //
   9 // Moodle is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU General Public License
  15 // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 namespace mod_data;
  18
  19 use coding_exception;
  20 use dml_exception;
  21 use mod_data\local\importer\csv_entries_importer;
  22 use moodle_exception;
  23 use zip_archive;
  24
  25 /**
  26  * Unit tests for import.php.
  27  *
  28  * @package    mod_data
  29  * @category   test
  30  * @copyright  2019 Tobias Reischmann
  31  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  32  */
  33 class entries_import_test extends \advanced_testcase {
  34
  35     /**
  36      * Set up function.
  37      */
  38     protected function setUp(): void {
  39         parent::setUp();
  40
  41         global $CFG;
  42         require_once($CFG->dirroot . '/mod/data/lib.php');
  43         require_once($CFG->dirroot . '/lib/datalib.php');
  44         require_once($CFG->dirroot . '/lib/csvlib.class.php');
  45         require_once($CFG->dirroot . '/search/tests/fixtures/testable_core_search.php');
  46         require_once($CFG->dirroot . '/mod/data/tests/generator/lib.php');
  47     }
  48
  49     /**
  50      * Get the test data.
  51      * In this instance we are setting up database records to be used in the unit tests.
  52      *
  53      * @return array
  54      */
  55     protected function get_test_data(): array {
  56         $this->resetAfterTest(true);
  57
  58         $generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
  59         $course = $this->getDataGenerator()->create_course();
  60         $teacher = $this->getDataGenerator()->create_and_enrol($course, 'teacher');
  61         $this->setUser($teacher);
  62         $student = $this->getDataGenerator()->create_and_enrol($course, 'student', array('username' => 'student'));
  63
  64         $data = $generator->create_instance(array('course' => $course->id));
  65         $cm = get_coursemodule_from_instance('data', $data->id);
  66
  67         // Add fields.
  68         $fieldrecord = new \stdClass();
  69         $fieldrecord->name = 'ID'; // Identifier of the records for testing.
  70         $fieldrecord->type = 'number';
  71         $generator->create_field($fieldrecord, $data);
  72
  73         $fieldrecord->name = 'Param2';
  74         $fieldrecord->type = 'text';
  75         $generator->create_field($fieldrecord, $data);
  76
  77         $fieldrecord->name = 'filefield';
  78         $fieldrecord->type = 'file';
  79         $generator->create_field($fieldrecord, $data);
  80
  81         $fieldrecord->name = 'picturefield';
  82         $fieldrecord->type = 'picture';
  83         $generator->create_field($fieldrecord, $data);
  84
  85         return [
  86             'teacher' => $teacher,
  87             'student' => $student,
  88             'data' => $data,
  89             'cm' => $cm,
  90         ];
  91     }
  92
  93     /**
  94      * Test uploading entries for a data instance without userdata.
  95      *
  96      * @throws dml_exception
  97      */
  98     public function test_import(): void {
  99         [
 100             'data' => $data,
 101             'cm' => $cm,
 102             'teacher' => $teacher,
 103         ] = $this->get_test_data();
 104
 105         $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import.csv',
 106             'test_data_import.csv');
 107         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 108
 109         // No userdata is present in the file: Fallback is to assign the uploading user as author.
 110         $expecteduserids = array();
 111         $expecteduserids[1] = $teacher->id;
 112         $expecteduserids[2] = $teacher->id;
 113
 114         $records = $this->get_data_records($data->id);
 115         $this->assertCount(2, $records);
 116         foreach ($records as $record) {
 117             $identifier = $record->items['ID']->content;
 118             $this->assertEquals($expecteduserids[$identifier], $record->userid);
 119         }
 120     }
 121
 122     /**
 123      * Test uploading entries for a data instance with userdata.
 124      *
 125      * At least one entry has an identifiable user, which is assigned as author.
 126      *
 127      * @throws dml_exception
 128      */
 129     public function test_import_with_userdata(): void {
 130         [
 131             'data' => $data,
 132             'cm' => $cm,
 133             'teacher' => $teacher,
 134             'student' => $student,
 135         ] = $this->get_test_data();
 136
 137         $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
 138             'test_data_import_with_userdata.csv');
 139         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 140
 141         $expecteduserids = array();
 142         $expecteduserids[1] = $student->id; // User student exists and is assigned as author.
 143         $expecteduserids[2] = $teacher->id; // User student2 does not exist. Fallback is the uploading user.
 144
 145         $records = $this->get_data_records($data->id);
 146         $this->assertCount(2, $records);
 147         foreach ($records as $record) {
 148             $identifier = $record->items['ID']->content;
 149             $this->assertEquals($expecteduserids[$identifier], $record->userid);
 150         }
 151     }
 152
 153     /**
 154      * Test uploading entries for a data instance with userdata and a defined field 'Username'.
 155      *
 156      * This should test the corner case, in which a user has defined a data fields, which has the same name
 157      * as the current lang string for username. In that case, the first Username entry is used for the field.
 158      * The second one is used to identify the author.
 159      *
 160      * @throws coding_exception
 161      * @throws dml_exception
 162      */
 163     public function test_import_with_field_username(): void {
 164         [
 165             'data' => $data,
 166             'cm' => $cm,
 167             'teacher' => $teacher,
 168             'student' => $student,
 169         ] = $this->get_test_data();
 170         $generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
 171
 172         // Add username field.
 173         $fieldrecord = new \stdClass();
 174         $fieldrecord->name = 'Username';
 175         $fieldrecord->type = 'text';
 176         $generator->create_field($fieldrecord, $data);
 177
 178         $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_field_username.csv',
 179             'test_data_import_with_field_username.csv');
 180         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 181
 182         $expecteduserids = array();
 183         $expecteduserids[1] = $student->id; // User student exists and is assigned as author.
 184         $expecteduserids[2] = $teacher->id; // User student2 does not exist. Fallback is the uploading user.
 185         $expecteduserids[3] = $student->id; // User student exists and is assigned as author.
 186
 187         $expectedcontent = array();
 188         $expectedcontent[1] = array(
 189             'Username' => 'otherusername1',
 190             'Param2' => 'My first entry',
 191         );
 192         $expectedcontent[2] = array(
 193             'Username' => 'otherusername2',
 194             'Param2' => 'My second entry',
 195         );
 196         $expectedcontent[3] = array(
 197             'Username' => 'otherusername3',
 198             'Param2' => 'My third entry',
 199         );
 200
 201         $records = $this->get_data_records($data->id);
 202         $this->assertCount(3, $records);
 203         foreach ($records as $record) {
 204             $identifier = $record->items['ID']->content;
 205             $this->assertEquals($expecteduserids[$identifier], $record->userid);
 206
 207             foreach ($expectedcontent[$identifier] as $field => $value) {
 208                 $this->assertEquals($value, $record->items[$field]->content,
 209                     "The value of field \"$field\" for the record at position \"$identifier\" " .
 210                     "which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
 211             }
 212         }
 213     }
 214
 215     /**
 216      * Test uploading entries for a data instance with a field 'Username' but only one occurrence in the csv file.
 217      *
 218      * This should test the corner case, in which a user has defined a data fields, which has the same name
 219      * as the current lang string for username. In that case, the only Username entry is used for the field.
 220      * The author should not be set.
 221      *
 222      * @throws coding_exception
 223      * @throws dml_exception
 224      */
 225     public function test_import_with_field_username_without_userdata(): void {
 226         [
 227             'data' => $data,
 228             'cm' => $cm,
 229             'teacher' => $teacher,
 230             'student' => $student,
 231         ] = $this->get_test_data();
 232         $generator = $this->getDataGenerator()->get_plugin_generator('mod_data');
 233
 234         // Add username field.
 235         $fieldrecord = new \stdClass();
 236         $fieldrecord->name = 'Username';
 237         $fieldrecord->type = 'text';
 238         $generator->create_field($fieldrecord, $data);
 239
 240         $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_userdata.csv',
 241             'test_data_import_with_userdata.csv');
 242         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 243
 244         // No userdata is present in the file: Fallback is to assign the uploading user as author.
 245         $expecteduserids = array();
 246         $expecteduserids[1] = $teacher->id;
 247         $expecteduserids[2] = $teacher->id;
 248
 249         $expectedcontent = array();
 250         $expectedcontent[1] = array(
 251             'Username' => 'student',
 252             'Param2' => 'My first entry',
 253         );
 254         $expectedcontent[2] = array(
 255             'Username' => 'student2',
 256             'Param2' => 'My second entry',
 257         );
 258
 259         $records = $this->get_data_records($data->id);
 260         $this->assertCount(2, $records);
 261         foreach ($records as $record) {
 262             $identifier = $record->items['ID']->content;
 263             $this->assertEquals($expecteduserids[$identifier], $record->userid);
 264
 265             foreach ($expectedcontent[$identifier] as $field => $value) {
 266                 $this->assertEquals($value, $record->items[$field]->content,
 267                     "The value of field \"$field\" for the record at position \"$identifier\" " .
 268                     "which is \"{$record->items[$field]->content}\" does not match the expected value \"$value\".");
 269             }
 270         }
 271     }
 272
 273     /**
 274      * Tests the import including files from a zip archive.
 275      *
 276      * @covers \mod_data\local\importer\entries_importer
 277      * @covers \mod_data\local\importer\csv_entries_importer
 278      * @return void
 279      */
 280     public function test_import_with_files(): void {
 281         [
 282             'data' => $data,
 283             'cm' => $cm,
 284         ] = $this->get_test_data();
 285
 286         $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files.zip',
 287             'test_data_import_with_files.zip');
 288         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 289
 290         $records = $this->get_data_records($data->id);
 291         $ziparchive = new zip_archive();
 292         $ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files.zip');
 293
 294         $importedcontent = array_values($records)[0]->items;
 295         $this->assertEquals(17, $importedcontent['ID']->content);
 296         $this->assertEquals('samplefile.png', $importedcontent['filefield']->content);
 297         $this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
 298
 299         // We now check if content of imported file from zip content is identical to the content of the file
 300         // stored in the mod_data record in the field 'filefield'.
 301         $fileindex = array_values(array_map(fn($file) => $file->index,
 302             array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplefile.png')))[0];
 303         $filestream = $ziparchive->get_stream($fileindex);
 304         $filefield = data_get_field_from_name('filefield', $data);
 305         $filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
 306         $this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
 307             $filefieldfilecontent);
 308         fclose($filestream);
 309
 310         // We now check if content of imported picture from zip content is identical to the content of the picture file
 311         // stored in the mod_data record in the field 'picturefield'.
 312         $fileindex = array_values(array_map(fn($file) => $file->index,
 313             array_filter($ziparchive->list_files(), fn($file) => $file->pathname === 'files/samplepicture.png')))[0];
 314         $filestream = $ziparchive->get_stream($fileindex);
 315         $filefield = data_get_field_from_name('picturefield', $data);
 316         $filefieldfilecontent = fread($filestream, $ziparchive->get_info($fileindex)->size);
 317         $this->assertEquals($filefield->get_file(array_keys($records)[0])->get_content(),
 318             $filefieldfilecontent);
 319         fclose($filestream);
 320         $this->assertCount(1, $importer->get_added_records_messages());
 321         $ziparchive->close();
 322     }
 323
 324     /**
 325      * Tests the import including files from a zip archive.
 326      *
 327      * @covers \mod_data\local\importer\entries_importer
 328      * @covers \mod_data\local\importer\csv_entries_importer
 329      * @return void
 330      */
 331     public function test_import_with_files_missing_file(): void {
 332         [
 333             'data' => $data,
 334             'cm' => $cm,
 335         ] = $this->get_test_data();
 336
 337         $importer = new csv_entries_importer(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip',
 338             'test_data_import_with_files_missing_file.zip');
 339         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 340
 341         $records = $this->get_data_records($data->id);
 342         $ziparchive = new zip_archive();
 343         $ziparchive->open(__DIR__ . '/fixtures/test_data_import_with_files_missing_file.zip');
 344
 345         $importedcontent = array_values($records)[0]->items;
 346         $this->assertEquals(17, $importedcontent['ID']->content);
 347         $this->assertFalse(isset($importedcontent['filefield']));
 348         $this->assertEquals('samplepicture.png', $importedcontent['picturefield']->content);
 349         $this->assertCount(1, $importer->get_added_records_messages());
 350         $ziparchive->close();
 351     }
 352
 353     /**
 354      * Returns the records of the data instance.
 355      *
 356      * Each records has an item entry, which contains all fields associated with this item.
 357      * Each fields has the parameters name, type and content.
 358      *
 359      * @param int $dataid Id of the data instance.
 360      * @return array The records of the data instance.
 361      * @throws dml_exception
 362      */
 363     private function get_data_records(int $dataid): array {
 364         global $DB;
 365
 366         $records = $DB->get_records('data_records', ['dataid' => $dataid]);
 367         foreach ($records as $record) {
 368             $sql = 'SELECT f.name, f.type, con.content FROM
 369                 {data_content} con JOIN {data_fields} f ON con.fieldid = f.id
 370                 WHERE con.recordid = :recordid';
 371             $items = $DB->get_records_sql($sql, array('recordid' => $record->id));
 372             $record->items = $items;
 373         }
 374         return $records;
 375     }
 376
 377     /**
 378      * Tests if the amount of imported records is counted properly.
 379      *
 380      * @covers \mod_data\local\importer\csv_entries_importer::import_csv
 381      * @covers \mod_data\local\importer\csv_entries_importer::get_added_records_messages
 382      * @dataProvider get_added_record_messages_provider
 383      * @param string $datafilecontent the content of the datafile to test as string
 384      * @param int $expectedcount the expected count of messages depending on the datafile content
 385      */
 386     public function test_get_added_record_messages(string $datafilecontent, int $expectedcount): void {
 387         [
 388             'data' => $data,
 389             'cm' => $cm,
 390         ] = $this->get_test_data();
 391
 392         // First we need to create the zip file from the provided data.
 393         $tmpdir = make_request_directory();
 394         $datafile = $tmpdir . '/entries_import_test_datafile_tmp_' . time() . '.csv';
 395         file_put_contents($datafile, $datafilecontent);
 396
 397         $importer = new csv_entries_importer($datafile, 'testdatafile.csv');
 398         $importer->import_csv($cm, $data, 'UTF-8', 'comma');
 399         $this->assertEquals($expectedcount, count($importer->get_added_records_messages()));
 400     }
 401
 402     /**
 403      * Data provider method for self::test_get_added_record_messages.
 404      *
 405      * @return array data for testing
 406      */
 407     public function get_added_record_messages_provider(): array {
 408         return [
 409             'only header' => [
 410                 'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL,
 411                 'expectedcount' => 0 // One line is being assumed to be the header.
 412             ],
 413             'one record' => [
 414                 'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
 415                     . '5,"some short text",testfilename.pdf,testpicture.png',
 416                 'expectedcount' => 1
 417             ],
 418             'two records' => [
 419                 'datafilecontent' => 'ID,Param2,filefield,picturefield' . PHP_EOL
 420                     . '5,"some short text",testfilename.pdf,testpicture.png' . PHP_EOL
 421                     . '3,"other text",testfilename2.pdf,testpicture2.png',
 422                 'expectedcount' => 2
 423             ],
 424         ];
 425     }
 426 }