lib/csvlib.class.php

   1 <?php
   2 // This file is part of Moodle - http://moodle.org/
   3 //
   4 // Moodle is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU General Public License as published by
   6 // the Free Software Foundation, either version 3 of the License, or
   7 // (at your option) any later version.
   8 //
   9 // Moodle is distributed in the hope that it will be useful,
  10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 // GNU General Public License for more details.
  13 //
  14 // You should have received a copy of the GNU General Public License
  15 // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 /**
  18  * This is a one-line short description of the file
  19  *
  20  * You can have a rather longer description of the file as well,
  21  * if you like, and it can span multiple lines.
  22  *
  23  * @package    core
  24  * @subpackage lib
  25  * @copyright  Petr Skoda
  26  * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  27  */
  28
  29 defined('MOODLE_INTERNAL') || die();
  30
  31 /**
  32  * Utitily class for importing of CSV files.
  33  * @copyright Petr Skoda
  34  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  35  * @package   moodlecore
  36  */
  37 class csv_import_reader {
  38
  39     /**
  40      * @var int import identifier
  41      */
  42     private $_iid;
  43
  44     /**
  45      * @var string which script imports?
  46      */
  47     private $_type;
  48
  49     /**
  50      * @var string|null Null if ok, error msg otherwise
  51      */
  52     private $_error;
  53
  54     /**
  55      * @var array cached columns
  56      */
  57     private $_columns;
  58
  59     /**
  60      * @var object file handle used during import
  61      */
  62     private $_fp;
  63
  64     /**
  65      * Contructor
  66      *
  67      * @param int $iid import identifier
  68      * @param string $type which script imports?
  69      */
  70     public function __construct($iid, $type) {
  71         $this->_iid  = $iid;
  72         $this->_type = $type;
  73     }
  74
  75     /**
  76      * Make sure the file is closed when this object is discarded.
  77      */
  78     public function __destruct() {
  79         $this->close();
  80     }
  81
  82     /**
  83      * Parse this content
  84      *
  85      * @param string $content the content to parse.
  86      * @param string $encoding content encoding
  87      * @param string $delimiter_name separator (comma, semicolon, colon, cfg)
  88      * @param string $column_validation name of function for columns validation, must have one param $columns
  89      * @param string $enclosure field wrapper. One character only.
  90      * @return bool false if error, count of data lines if ok; use get_error() to get error string
  91      */
  92     public function load_csv_content($content, $encoding, $delimiter_name, $column_validation=null, $enclosure='"') {
  93         global $USER, $CFG;
  94
  95         $this->close();
  96         $this->_error = null;
  97
  98         $content = core_text::convert($content, $encoding, 'utf-8');
  99         // remove Unicode BOM from first line
 100         $content = core_text::trim_utf8_bom($content);
 101         // Fix mac/dos newlines
 102         $content = preg_replace('!\r\n?!', "\n", $content);
 103         // Remove any spaces or new lines at the end of the file.
 104         if ($delimiter_name == 'tab') {
 105             // trim() by default removes tabs from the end of content which is undesirable in a tab separated file.
 106             $content = trim($content, chr(0x20) . chr(0x0A) . chr(0x0D) . chr(0x00) . chr(0x0B));
 107         } else {
 108             $content = trim($content);
 109         }
 110
 111         $csv_delimiter = csv_import_reader::get_delimiter($delimiter_name);
 112         // $csv_encode    = csv_import_reader::get_encoded_delimiter($delimiter_name);
 113
 114         // Create a temporary file and store the csv file there,
 115         // do not try using fgetcsv() because there is nothing
 116         // to split rows properly - fgetcsv() itself can not do it.
 117         $tempfile = tempnam(make_temp_directory('/csvimport'), 'tmp');
 118         if (!$fp = fopen($tempfile, 'w+b')) {
 119             $this->_error = get_string('cannotsavedata', 'error');
 120             @unlink($tempfile);
 121             return false;
 122         }
 123         fwrite($fp, $content);
 124         fseek($fp, 0);
 125         // Create an array to store the imported data for error checking.
 126         $columns = array();
 127         // str_getcsv doesn't iterate through the csv data properly. It has
 128         // problems with line returns.
 129         while ($fgetdata = fgetcsv($fp, 0, $csv_delimiter, $enclosure)) {
 130             // Check to see if we have an empty line.
 131             if (count($fgetdata) == 1) {
 132                 if ($fgetdata[0] !== null) {
 133                     // The element has data. Add it to the array.
 134                     $columns[] = $fgetdata;
 135                 }
 136             } else {
 137                 $columns[] = $fgetdata;
 138             }
 139         }
 140         $col_count = 0;
 141
 142         // process header - list of columns
 143         if (!isset($columns[0])) {
 144             $this->_error = get_string('csvemptyfile', 'error');
 145             fclose($fp);
 146             unlink($tempfile);
 147             return false;
 148         } else {
 149             $col_count = count($columns[0]);
 150         }
 151
 152         // Column validation.
 153         if ($column_validation) {
 154             $result = $column_validation($columns[0]);
 155             if ($result !== true) {
 156                 $this->_error = $result;
 157                 fclose($fp);
 158                 unlink($tempfile);
 159                 return false;
 160             }
 161         }
 162
 163         $this->_columns = $columns[0]; // cached columns
 164         // check to make sure that the data columns match up with the headers.
 165         foreach ($columns as $rowdata) {
 166             if (count($rowdata) !== $col_count) {
 167                 $this->_error = get_string('csvweirdcolumns', 'error');
 168                 fclose($fp);
 169                 unlink($tempfile);
 170                 $this->cleanup();
 171                 return false;
 172             }
 173         }
 174
 175         $filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid;
 176         $filepointer = fopen($filename, "w");
 177         // The information has been stored in csv format, as serialized data has issues
 178         // with special characters and line returns.
 179         $storedata = csv_export_writer::print_array($columns, ',', '"', true);
 180         fwrite($filepointer, $storedata);
 181
 182         fclose($fp);
 183         unlink($tempfile);
 184         fclose($filepointer);
 185
 186         $datacount = count($columns);
 187         return $datacount;
 188     }
 189
 190     /**
 191      * Returns list of columns
 192      *
 193      * @return array
 194      */
 195     public function get_columns() {
 196         if (isset($this->_columns)) {
 197             return $this->_columns;
 198         }
 199
 200         global $USER, $CFG;
 201
 202         $filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid;
 203         if (!file_exists($filename)) {
 204             return false;
 205         }
 206         $fp = fopen($filename, "r");
 207         $line = fgetcsv($fp);
 208         fclose($fp);
 209         if ($line === false) {
 210             return false;
 211         }
 212         $this->_columns = $line;
 213         return $this->_columns;
 214     }
 215
 216     /**
 217      * Init iterator.
 218      *
 219      * @global object
 220      * @global object
 221      * @return bool Success
 222      */
 223     public function init() {
 224         global $CFG, $USER;
 225
 226         if (!empty($this->_fp)) {
 227             $this->close();
 228         }
 229         $filename = $CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid;
 230         if (!file_exists($filename)) {
 231             return false;
 232         }
 233         if (!$this->_fp = fopen($filename, "r")) {
 234             return false;
 235         }
 236         //skip header
 237         return (fgetcsv($this->_fp) !== false);
 238     }
 239
 240     /**
 241      * Get next line
 242      *
 243      * @return mixed false, or an array of values
 244      */
 245     public function next() {
 246         if (empty($this->_fp) or feof($this->_fp)) {
 247             return false;
 248         }
 249         if ($ser = fgetcsv($this->_fp)) {
 250             return $ser;
 251         } else {
 252             return false;
 253         }
 254     }
 255
 256     /**
 257      * Release iteration related resources
 258      *
 259      * @return void
 260      */
 261     public function close() {
 262         if (!empty($this->_fp)) {
 263             fclose($this->_fp);
 264             $this->_fp = null;
 265         }
 266     }
 267
 268     /**
 269      * Get last error
 270      *
 271      * @return string error text of null if none
 272      */
 273     public function get_error() {
 274         return $this->_error;
 275     }
 276
 277     /**
 278      * Cleanup temporary data
 279      *
 280      * @global object
 281      * @global object
 282      * @param boolean $full true means do a full cleanup - all sessions for current user, false only the active iid
 283      */
 284     public function cleanup($full=false) {
 285         global $USER, $CFG;
 286
 287         if ($full) {
 288             @remove_dir($CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id);
 289         } else {
 290             @unlink($CFG->tempdir.'/csvimport/'.$this->_type.'/'.$USER->id.'/'.$this->_iid);
 291         }
 292     }
 293
 294     /**
 295      * Get list of cvs delimiters
 296      *
 297      * @return array suitable for selection box
 298      */
 299     public static function get_delimiter_list() {
 300         global $CFG;
 301         $delimiters = array('comma'=>',', 'semicolon'=>';', 'colon'=>':', 'tab'=>'\\t');
 302         if (isset($CFG->CSV_DELIMITER) and strlen($CFG->CSV_DELIMITER) === 1 and !in_array($CFG->CSV_DELIMITER, $delimiters)) {
 303             $delimiters['cfg'] = $CFG->CSV_DELIMITER;
 304         }
 305         return $delimiters;
 306     }
 307
 308     /**
 309      * Get delimiter character
 310      *
 311      * @param string separator name
 312      * @return string delimiter char
 313      */
 314     public static function get_delimiter($delimiter_name) {
 315         global $CFG;
 316         switch ($delimiter_name) {
 317             case 'colon':     return ':';
 318             case 'semicolon': return ';';
 319             case 'tab':       return "\t";
 320             case 'cfg':       if (isset($CFG->CSV_DELIMITER)) { return $CFG->CSV_DELIMITER; } // no break; fall back to comma
 321             case 'comma':     return ',';
 322             default :         return ',';  // If anything else comes in, default to comma.
 323         }
 324     }
 325
 326     /**
 327      * Get encoded delimiter character
 328      *
 329      * @global object
 330      * @param string separator name
 331      * @return string encoded delimiter char
 332      */
 333     public static function get_encoded_delimiter($delimiter_name) {
 334         global $CFG;
 335         if ($delimiter_name == 'cfg' and isset($CFG->CSV_ENCODE)) {
 336             return $CFG->CSV_ENCODE;
 337         }
 338         $delimiter = csv_import_reader::get_delimiter($delimiter_name);
 339         return '&#'.ord($delimiter);
 340     }
 341
 342     /**
 343      * Create new import id
 344      *
 345      * @global object
 346      * @param string who imports?
 347      * @return int iid
 348      */
 349     public static function get_new_iid($type) {
 350         global $USER;
 351
 352         $filename = make_temp_directory('csvimport/'.$type.'/'.$USER->id);
 353
 354         // use current (non-conflicting) time stamp
 355         $iiid = time();
 356         while (file_exists($filename.'/'.$iiid)) {
 357             $iiid--;
 358         }
 359
 360         return $iiid;
 361     }
 362 }
 363
 364
 365 /**
 366  * Utitily class for exporting of CSV files.
 367  * @copyright 2012 Adrian Greeve
 368  * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
 369  * @package   core
 370  * @category  csv
 371  */
 372 class csv_export_writer {
 373     /**
 374      * @var string $delimiter  The name of the delimiter. Supported types(comma, tab, semicolon, colon, cfg)
 375      */
 376     var $delimiter;
 377     /**
 378      * @var string $csvenclosure  How fields with spaces and commas are enclosed.
 379      */
 380     var $csvenclosure;
 381     /**
 382      * @var string $mimetype  Mimetype of the file we are exporting.
 383      */
 384     var $mimetype;
 385     /**
 386      * @var string $filename  The filename for the csv file to be downloaded.
 387      */
 388     var $filename;
 389     /**
 390      * @var string $path  The directory path for storing the temporary csv file.
 391      */
 392     var $path;
 393     /**
 394      * @var resource $fp  File pointer for the csv file.
 395      */
 396     protected $fp;
 397
 398     /**
 399      * Constructor for the csv export reader
 400      *
 401      * @param string $delimiter      The name of the character used to seperate fields. Supported types(comma, tab, semicolon, colon, cfg)
 402      * @param string $enclosure      The character used for determining the enclosures.
 403      * @param string $mimetype       Mime type of the file that we are exporting.
 404      */
 405     public function __construct($delimiter = 'comma', $enclosure = '"', $mimetype = 'application/download') {
 406         $this->delimiter = $delimiter;
 407         // Check that the enclosure is a single character.
 408         if (strlen($enclosure) == 1) {
 409             $this->csvenclosure = $enclosure;
 410         } else {
 411             $this->csvenclosure = '"';
 412         }
 413         $this->filename = "Moodle-data-export.csv";
 414         $this->mimetype = $mimetype;
 415     }
 416
 417     /**
 418      * Set the file path to the temporary file.
 419      */
 420     protected function set_temp_file_path() {
 421         global $USER, $CFG;
 422         make_temp_directory('csvimport/' . $USER->id);
 423         $path = $CFG->tempdir . '/csvimport/' . $USER->id. '/' . $this->filename;
 424         // Check to see if the file exists, if so delete it.
 425         if (file_exists($path)) {
 426             unlink($path);
 427         }
 428         $this->path = $path;
 429     }
 430
 431     /**
 432      * Add data to the temporary file in csv format
 433      *
 434      * @param array $row  An array of values.
 435      */
 436     public function add_data($row) {
 437         if(!isset($this->path)) {
 438             $this->set_temp_file_path();
 439             $this->fp = fopen($this->path, 'w+');
 440         }
 441         $delimiter = csv_import_reader::get_delimiter($this->delimiter);
 442         fputcsv($this->fp, $row, $delimiter, $this->csvenclosure);
 443     }
 444
 445     /**
 446      * Echos or returns a csv data line by line for displaying.
 447      *
 448      * @param bool $return  Set to true to return a string with the csv data.
 449      * @return string       csv data.
 450      */
 451     public function print_csv_data($return = false) {
 452         fseek($this->fp, 0);
 453         $returnstring = '';
 454         while (($content = fgets($this->fp)) !== false) {
 455             if (!$return){
 456                 echo $content;
 457             } else {
 458                 $returnstring .= $content;
 459             }
 460         }
 461         if ($return) {
 462             return $returnstring;
 463         }
 464     }
 465
 466     /**
 467      * Set the filename for the uploaded csv file
 468      *
 469      * @param string $dataname    The name of the module.
 470      * @param string $extenstion  File extension for the file.
 471      */
 472     public function set_filename($dataname, $extension = '.csv') {
 473         $filename = clean_filename($dataname);
 474         $filename .= clean_filename('-' . gmdate("Ymd_Hi"));
 475         $filename .= clean_filename("-{$this->delimiter}_separated");
 476         $filename .= $extension;
 477         $this->filename = $filename;
 478     }
 479
 480     /**
 481      * Output file headers to initialise the download of the file.
 482      */
 483     protected function send_header() {
 484         global $CFG;
 485
 486         if (defined('BEHAT_SITE_RUNNING')) {
 487             // For text based formats - we cannot test the output with behat if we force a file download.
 488             return;
 489         }
 490         if (is_https()) { // HTTPS sites - watch out for IE! KB812935 and KB316431.
 491             header('Cache-Control: max-age=10');
 492             header('Pragma: ');
 493         } else { //normal http - prevent caching at all cost
 494             header('Cache-Control: private, must-revalidate, pre-check=0, post-check=0, max-age=0');
 495             header('Pragma: no-cache');
 496         }
 497         header('Expires: '. gmdate('D, d M Y H:i:s', 0) .' GMT');
 498         header("Content-Type: $this->mimetype\n");
 499         header("Content-Disposition: attachment; filename=\"$this->filename\"");
 500     }
 501
 502     /**
 503      * Download the csv file.
 504      */
 505     public function download_file() {
 506         $this->send_header();
 507         $this->print_csv_data();
 508         exit;
 509     }
 510
 511     /**
 512      * Creates a file for downloading an array into a deliminated format.
 513      * This function is useful if you are happy with the defaults and all of your
 514      * information is in one array.
 515      *
 516      * @param string $filename    The filename of the file being created.
 517      * @param array $records      An array of information to be converted.
 518      * @param string $delimiter   The name of the delimiter. Supported types(comma, tab, semicolon, colon, cfg)
 519      * @param string $enclosure   How speical fields are enclosed.
 520      */
 521     public static function download_array($filename, array &$records, $delimiter = 'comma', $enclosure='"') {
 522         $csvdata = new csv_export_writer($delimiter, $enclosure);
 523         $csvdata->set_filename($filename);
 524         foreach ($records as $row) {
 525             $csvdata->add_data($row);
 526         }
 527         $csvdata->download_file();
 528     }
 529
 530     /**
 531      * This will convert an array of values into a deliminated string.
 532      * Like the above function, this is for convenience.
 533      *
 534      * @param array $records     An array of information to be converted.
 535      * @param string $delimiter  The name of the delimiter. Supported types(comma, tab, semicolon, colon, cfg)
 536      * @param string $enclosure  How speical fields are enclosed.
 537      * @param bool $return       If true will return a string with the csv data.
 538      * @return string            csv data.
 539      */
 540     public static function print_array(array &$records, $delimiter = 'comma', $enclosure = '"', $return = false) {
 541         $csvdata = new csv_export_writer($delimiter, $enclosure);
 542         foreach ($records as $row) {
 543             $csvdata->add_data($row);
 544         }
 545         $data = $csvdata->print_csv_data($return);
 546         if ($return) {
 547             return $data;
 548         }
 549     }
 550
 551     /**
 552      * Make sure that everything is closed when we are finished.
 553      */
 554     public function __destruct() {
 555         fclose($this->fp);
 556         unlink($this->path);
 557     }
 558 }