library/edihistory/ibr_archive.php

   1 <?php
   2 /*************  ibr_archive.php
   3  * Author:  Kevin McCormick   Longview Texas
   4  *
   5  * Copyright 2012 Kevin McCormick    Longview, Texas
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  20  * MA 02110-1301, USA.
  21  *
  22  *
  23  * Purpose: to archive old entries in the csv files and old files
  24  *
  25  * @author Kevin McCormick
  26  * @link: http://www.open-emr.org
  27  * @package OpenEMR
  28  * @subpackage ediHistory
  29  */
  30
  31 // a security measure to prevent direct web access to this file
  32 // must be accessed through the main calling script ibr_history.php
  33 // from admin at rune-city dot com;  found in php manual
  34 //if (!defined('SITE_IN')) die('Direct access not allowed!');
  35
  36 /**
  37  * @param string $date -- expect format mm/dd/yyyy
  38  * @return int $days days prior to today
  39  */
  40 function csv_days_prior($date) {
  41         //  deal with CCYY/MM/DD or MM/DD/CCYY
  42         $d1 = preg_split('/\D/', $date);
  43         if (!is_array($d1) || count($d1) != 3) {return FALSE;}
  44         //
  45         $isOK = checkdate($d1[0], $d1[1], $d1[2]) ? TRUE : checkdate($d1[1], $d1[2], $d1[0]);
  46         if (!$isOK) { return FALSE; }
  47         //
  48         $date1 = new DateTime(date("m/d/Y", $now=time()));
  49         $date2 = new DateTime($date);
  50         $dys = ($date1->format('U') - $date2->format('U')) / 86400;
  51         //
  52         return abs(floor($dys));
  53 }
  54
  55 /**
  56  * Creates a zip archive of the files in the $filename_ar array and
  57  * returns the path/name of the archive or FALSE on error
  58  *
  59  * @param array $parameters array for file type from csv_parameters
  60  * @param array $filename_ar filenames to be archived
  61  * @param string $archive_date  date of archive to be incorporated in archive file name
  62  * @return mixed   either the name of the zip archive or FALSE in case or error
  63  */
  64 function csv_zip_dir($parameters, $filename_ar, $archive_date) {
  65         // we deal with possible maximum files issues by chunking the $fn_ar array
  66         //$ulim = array();
  67         //exec("ulimit -a | grep 'open files'", $ulim);  open files  (-n) 1024
  68         //
  69         $f_max = 200;
  70         $fn_ar2 = array();
  71         if ( count($filename_ar) > $f_max) {
  72                 $fn_ar2 = array_chunk($filename_ar, $f_max);
  73         } else {
  74                 $fn_ar2[] = $filename_ar;
  75         }
  76         //
  77         $zip_name = IBR_UPLOAD_DIR.DIRECTORY_SEPARATOR.$type."_$archive_date.zip";
  78         $ftmpn = IBR_UPLOAD_DIR.DIRECTORY_SEPARATOR;
  79         $fdir = dirname(__FILE__).$parameters['directory'].DIRECTORY_SEPARATOR;
  80         $type = $parameters['type'];
  81         //
  82         $zip_obj = new ZipArchive();
  83         //
  84         foreach($fn_ar2 as $fnz) {
  85                 // reopen the zip archive on each loop so the open file count is controlled
  86                 if (is_file($zip_name) ) {
  87                         $isOK = $zip_obj->open($zip_name, ZIPARCHIVE::CHECKCONS);
  88                         $isNew = FALSE;
  89                 } else {
  90                         $isOK = $zip_obj->open($zip_name, ZIPARCHIVE::CREATE);
  91                         $isNew = $isOK;
  92                 }
  93                 //
  94                 if ($isOK && $isNew) {
  95                         $zip_obj->addEmptyDir($type);
  96                         $zip_obj->setArchiveComment("archive " . $fdir . "prior to $archive_date");
  97                 }
  98                 //
  99                 if ($isOK) {
 100                         // we are working with the open archive
 101                         // now add the files to the archive
 102                         foreach($fnz as $fz) {
 103                                 if (is_file($fdir.$fz) ) {
 104                                         $iscp = copy($fdir.$fz, $ftmpn.$fz);
 105                                         $isOK = $zip_obj->addFile($ftmpn.$fz, $type.DIRECTORY_SEPARATOR.$fz);
 106                                 } else {
 107                                         csv_edihist_log("csv_zip_dir: in record, but not in directory $fz ");
 108                                         // possible that file is in csv table, but not in directory?
 109                                 }
 110                                 //
 111                                 if ($isOK && $iscp) {
 112                                         // if we have added the file to the archive, remove it from the storage directory
 113                                         // but keep the /tmp file copy for now
 114                                         unlink($fdir.$fz);
 115                                 } else {
 116                                         $msg = $zip_obj->getStatusString();
 117                                         csv_edihist_log("csv_zip_dir: $type ZipArchive failed for $fz  $msg");
 118                                 }
 119                         } // end foreach
 120                 } else {
 121                         // ZipArchive open() failed -- try to get the error message and return false
 122                         $msg = $zip_obj->getStatusString();
 123                         csv_edihist_log("csv_zip_dir: $type ZipArchive open() failed  $msg");
 124                         return $isOK;
 125                 }
 126                 //
 127                 // errors on close would be non-existing file added or something else
 128                 $isOK = $zip_obj->close($zip_name);
 129                 if (!$isOK) {
 130                         $msg = $zip_obj->getStatusString();
 131                         csv_edihist_log("csv_zip_dir: $type ZipArchive close() error for $fz  $msg");
 132                         //
 133                         return $isOK;
 134                 }
 135         }  // end foreach($fn_ar2 as $fnz)
 136         //
 137         return ($isOK) ? $zip_name : $isOK;
 138 }
 139
 140
 141 /**
 142  * restores files from a zip archive or the tmp dir if the archive process needs to be aborted
 143  *
 144  * @param string $csv_type     either 'file' or 'claim'
 145  * @param array $parameters    the paramaters array for the particular type
 146  * @param array $filename_ar   array of file names that may have been deleted
 147  */
 148 function csv_restore_files($csv_type, $parameters, $filename_ar) {
 149         //
 150         if (!is_dir($old_dir) ) { return FALSE; }
 151         if (!is_array($filename_ar) || count($filename_ar) == 0) { return FALSE; }
 152         //
 153         $csv_p = ($csv_type == 'file') ? $parameters['files_csv'] : $parameters['claims_csv'];
 154         $csv_dir = dirname(__FILE__).dirname($csv_p).DIRECTORY_SEPARATOR;
 155         $csv_file = basename($csv_p);
 156         //
 157         $fdir = dirname(__FILE__) . $parameters['directory'] . DIRECTORY_SEPARATOR;
 158         //
 159         $fileslost = array();
 160         //
 161         csv_edihist_log("csv_move_old_stuff: add file to archive failed for $fz");
 162         // we are in a jam -- archive is messed up
 163         $ntmpname = IBR_UPLOAD_DIR.DIRECTORY_SEPARATOR;
 164         //
 165         foreach($filename_ar as $fnz) {
 166                 foreach($fnz as $fz) {
 167                         //
 168                         if (is_file($fdir.$fz) ) {
 169                                 // file is still in our type directory
 170                                 continue;
 171                         } else {
 172                                 $iscp = copy($ntmpname.$fz, $fdir.$fz);
 173                                 if (!$iscp) {
 174                                         csv_edihist_log("csv_move_old_stuff: $type archive unwind failed for $fz");
 175                                         $fileslost[] = $fz;
 176                                 }
 177                         }
 178                 }
 179         }
 180         // put the csv file back
 181         $ntmpcsv = IBR_UPLOAD_DIR.DIRECTORY_SEPARATOR.$csv_file;
 182         $iscp = copy($ntmpcsv, $csv_dir.$csv_file);
 183         if (!$iscp) { $fileslost[] = $csv_file; }
 184         //
 185         return $fileslost;
 186 }
 187
 188
 189 /**
 190  * After the archive is created, the csv record needs to be re-written so the archived
 191  * files are not in the csv file and hence, not searched for
 192  *
 193  * @param string $csv_path   the tmp csv file path is expected
 194  * @param array $heading_ar  the column heading for the csv file
 195  * @param array $row_array   the data rows to be written
 196  * @return integer           count the characters written as returned by fputcsv()
 197  */
 198 function csv_rewrite_record($csv_path, $heading_ar, $row_array) {
 199         // @param string $csv_path -- the tmp csv file path is expected
 200         // @param array $heading_ar -- the column heading for the csv file
 201         // @param array $row_array -- the data rows to be written
 202         //
 203         // count characters written -- returned by fputcsv
 204         $ocwct = 0;
 205         $fh3 = fopen($csv_path, 'w');
 206
 207         // if we fail to open the file, return the result, expect FALSE
 208         if (!$fh3) {
 209                 csv_edihist_log("csv_rewrite_record: failed to open $csv_path");
 210         } else {
 211                 // it is a an empty file, so write the heading row
 212                 if (count($ar_h) ) {
 213                         $ocwct += fputcsv ( $fh3, $heading_ar );
 214                         // wrote heading, now add rows
 215                         foreach($row_array as $row) {
 216                                 $ocwct += fputcsv ($fh3, $row );
 217                         }
 218                         fclose($fh3);
 219                 } else {
 220                         csv_edihist_log("csv_rewrite_record: empty records array passed for $csv_path");
 221                 }
 222         }
 223         csv_edihist_log("csv_rewrite_record: wrote $ocwct characters " . count($row_array) . " rows to $csv_path");
 224         return $ocwct;
 225 }
 226
 227
 228 /**
 229  * Reads the current csv file and divides it into two arrays 'arch_csv' and 'curr_csv'
 230  * the 'arch_csv' contains the rows that will be archived (prior to archive_date)
 231  * and the 'curr_csv' contains the rows that will be retained
 232  * Also, if the csv_type is 'file' an array of file names is stored under 'files' key
 233  *
 234  * @param string $csv_type
 235  * @param string $csv_path
 236  * @param integer $date_col
 237  * @param integer $fn_column
 238  * @param string $archive_date
 239  * @return array                $arch_ar keys ['arch_csv'] ['curr_csv'] ['files']
 240  */
 241 function csv_archive_array($csv_type, $csv_path, $date_col, $fn_column, $archive_date) {
 242         //
 243         $fh = fopen($csv_path, 'r');
 244         $idx = 0;
 245         //
 246         $arch_ar = array();
 247         //
 248         $dt = strpos($archive_date, '/') ? str_replace('/', '', $archive_date) : $archive_date;
 249         //
 250         if ($fh !== FALSE) {
 251                 while (($data = fgetcsv($fh)) !== FALSE) {
 252                         //
 253                         if ($idx == 0) {
 254                                 $arch_csv[] = $data;
 255                         } else {
 256                                 $isok = (substr($data[$date_col], 0, 8) < $dt) ? TRUE : FALSE;
 257                                 if ($isok) {
 258                                         $arch_ar['arch_csv'][] = $data;
 259                                         if ($csv_type == 'file') { $arch_ar['files'][] = $data[$fn_column]; }
 260                                 } else {
 261                                         // retained csv rows go here
 262                                         $arch_ar['curr_csv'][] = $data;
 263                                 }
 264                         }
 265                         $idx++;
 266                 }
 267                 flock($fh2, LOCK_UN);
 268                 fclose($fh);
 269         } else {
 270                 csv_edihist_log("csv_archive_array: failed to open " . basename($csv_path));
 271                 return FALSE;
 272         }
 273
 274         return $arch_ar;
 275 }
 276
 277 /**
 278  * The main function in this ibr_archive.php script.  This function gets the parameters array
 279  * from csv_parameters() and calls the archiving functions on each type of file
 280  * in the parameters array.
 281  *
 282  * @param string $archive_date    yyyy/mm/dd date prior to which is archived
 283  * @return string                 descriptive message in html format
 284  */
 285 function csv__archive_old($archive_date) {
 286         //
 287         //
 288         if (!is_dir(IBR_HISTORY_DIR.DIRECTORY_SEPARATOR.'archive') ) {
 289                 // should have been created at setup
 290                 mkdir (IBR_HISTORY_DIR.DIRECTORY_SEPARATOR.'archive', 0755);
 291         }
 292         //
 293         $days = csv_days_prior($archive_date);
 294         if (!$days || $days < 60 ) {
 295                 $out_html = "Archive date $archive_date invalid or less than 60 days prior <br />" .PHP_EOL;
 296                 return $out_html;
 297         }
 298         //
 299         $out_html = "Archiving records prior to $archive_date <br />" .PHP_EOL;
 300         //
 301         $dt = str_replace('/', '', $archive_date);
 302         //
 303         $isarchived = FALSE;
 304         $haserr = FALSE;
 305         $params = csv_parameters();
 306         //
 307         $f_max = 200;
 308         //
 309         foreach($params as $k=>$p) {
 310                 $type = $p['type'];
 311                 $fdir = dirname(__FILE__) . $p['directory'] . DIRECTORY_SEPARATOR;
 312                 //
 313                 $fn_ar = array();
 314                 $arch_csv = array();
 315                 $curr_csvd = array();
 316                 //
 317                 $archive_ar = array();
 318                 //
 319                 $csv_dir = dirname(__FILE__).IBR_HISTORY_DIR.DIRECTORY_SEPARATOR;
 320                 $archive_dir = dirname(__FILE__).IBR_HISTORY_DIR.DIRECTORY_SEPARATOR.'archive';
 321                 $tmp_dir = IBR_UPLOAD_DIR.DIRECTORY_SEPARATOR;
 322                 //
 323                 // type dpr has only a claim csv type
 324                 $head_ar = ($type == 'dpr') ?  csv_files_header($type, 'claim') : csv_files_header($type, 'file');
 325                 //
 326                 $fncol = $p['fncolumn'];
 327                 $datecol = $p['datecolumn'];
 328                 //
 329                 // files csv temporary names
 330                 $file_csv = dirname(__FILE__).$p['files_csv'];
 331                 $file_csv_copy = $tmp_dir.basename($file_csv);
 332                 $tmp_fold_csv = $tmp_dir.$type.'_old_'.basename($file_csv);
 333                 $tmp_fnew_csv = $tmp_dir.$type.'_new_'.basename($file_csv);
 334                 $iscpf = copy ($file_csv, $file_csv_copy);
 335                 //
 336                 // claims csv temporary names
 337                 $claim_csv = dirname(__FILE__).$p['claims_csv'];
 338                 $claim_csv_copy = $tmp_dir.basename($claim_csv);
 339                 $tmp_cold_csv = $tmp_dir.$type.'_old_'.basename($claim_csv);
 340                 $tmp_cnew_csv = $tmp_dir.$type.'_new_'.basename($claim_csv);
 341                 $iscpc = copy ($claim_csv, $claim_csv_copy);
 342                 //
 343                 if (!$iscpf || !$iscpc) {
 344                         csv_edihist_log("csv_move_old_stuff: copy to tmp dir failed for csv file $type");
 345                         $out_html = "Archive temporary files operation failed ... aborting <br />" .PHP_EOL;
 346                         return $out_html;
 347                 }
 348                 //
 349                 // lock the original files
 350                 $fh1 = fopen($file_csv, 'r');
 351                 $islk1 = flock($fh1, LOCK_EX);
 352                 if (!$islk1) { fclose($fh1) ; } // assume we are on a system that does not support locks
 353                 $fh2 = fopen($claim_csv, 'r');
 354                 $islk2 = flock($fh2, LOCK_EX);
 355                 if (!$islk2) { fclose($fh2) ; } // assume we are on a system that does not support locks
 356                 //
 357                 // do the archive for the files_type.csv
 358                 $archive_ar = csv_archive_array('file', $file_csv_copy, $datecol, $fncol, $dt);
 359                 if (!$archive_ar) {
 360                         csv_edihist_log("csv_move_old_stuff: creating archive information failed for " . basename($file_csv_copy));
 361                         continue;
 362                 }
 363                 $och = csv_rewrite_record($tmp_old_csv, $headfile, $archive_ar['arch_csv']);
 364                 $nch = csv_rewrite_record($tmp_new_csv, $headfile, $archive_ar['curr_csv']);
 365                 $zarch = csv_zip_dir($params, $archive_ar['files'], $archive_date);
 366                 // now move the reconfigured files
 367                 // unlink the present csv file, since it is possible for a rename error if it exists
 368                 $islk1 = ($islk1) ? flock($fh1, LOCK_UN) : $islk1;
 369                 if ($islk1) { fclose($fh1); }
 370                 $isunl = unlink($file_csv);
 371                 $ismvz = rename($zarch, $archive_dir.DIRECTORY_SEPARATOR.basename($zarch) );
 372                 $ismvo = rename($tmp_fold_csv, $archive_dir.DIRECTORY_SEPARATOR.$dt.basename($tmp_fold_csv) );
 373                 $ismvn = rename($tmp_fnew_csv, $file_csv );
 374                 //
 375                 if ($ismvz && $ismvo && $ismvn) {
 376                         // everything is working - clear out the files we put in tmp_dir
 377                         // the tmp dir should be empty, but there might have been something else created there
 378                         $isclr = csv_clear_tmpdir();
 379                         $out_html .= "Archived: type $type <br />" .PHP_EOL;
 380                         $out_html .= "&nbsp; archived " .count($archive_ar['files']) . " files   <br />" .PHP_EOL;
 381                         $out_html .= "&nbsp; archived " .count($archive_ar['arch_csv']) . " rows from " .basename($file_csv) ." <br />" .PHP_EOL;
 382                         $out_html .= "&nbsp; there are now " .count($archive_ar['curr_csv']) . " rows in " .basename($file_csv) ." <br />" .PHP_EOL;
 383                 } else {
 384                         // in case or error, try to restore everything
 385                         $fl_ar = csv_restore_files('file', $p, $archive_ar['files']);
 386                         if (is_array($fl_ar) && count($fl_ar) > 0) {
 387                                 foreach ($fl_ar as $f) {
 388                                         csv_edihist_log("csv_move_old_stuff lost file: $f");
 389                                 }
 390                         } elseif (is_array($fl_ar) && count($fl_ar) == 0) {
 391                                 csv_edihist_log("csv_move_old_stuff archiving failed, and files restored");
 392                         } else {
 393                                 csv_edihist_log("csv_move_old_stuff archive failed and files were lost");
 394                         }
 395                         // give a message and quit
 396                         $out_html .= "Archiving error: type $type archive errors ... aborting <br />" .PHP_EOL;
 397                         return $out_html;
 398                 }
 399                 //
 400                 // now we do the claims table
 401                 //
 402                 // dpr type has only claim table, treated as a file table above
 403                 if ($type == 'dpr') { continue; }
 404                 //
 405                 $head_ar = csv_files_header($type, 'claim');
 406                 //
 407                 $archive_ar = csv_archive_array('claim', $claim_csv_copy, $datecol, $fncol, $dt);
 408                 if (!$archive_ar) {
 409                         csv_edihist_log("csv_move_old_stuff: creating archive information failed for " . basename($file_csv_copy));
 410                         continue;
 411                 }
 412                 //
 413                 $och = csv_rewrite_record($tmp_cold_csv, $head_ar, $archive_ar['arch_csv']);
 414                 $nch = csv_rewrite_record($tmp_cnew_csv, $head_ar, $archive_ar['curr_csv']);
 415                 //
 416                 $islk2 = ($islk2) ? flock($fh2, LOCK_UN) : $islk2;
 417                 if ($islk2) { fclose($fh2); }
 418                 $isunl = unlink($claim_csv);
 419                 $ismvo = rename($tmp_cold_csv, $archive_dir.DIRECTORY_SEPARATOR.$dt.basename($tmp_cold_csv) );
 420                 $ismvn = rename($tmp_cnew_csv, $claim_csv );
 421                 //
 422                 if ($ismvo && $ismvn) {
 423                         // everything is working - clear out the files we put in tmp_dir
 424                         // the tmp dir should be empty, but there might have been something else created there
 425                         $isclr = csv_clear_tmpdir();
 426                         $out_html .= "&nbsp; archived " .count($archive_ar['arch_csv']) . " rows from " . basename($claim_csv) ." <br />" .PHP_EOL;
 427                         $out_html .= "&nbsp; there are now " .count($archive_ar['curr_csv']) . " rows in " . basename($claim_csv) ." <br />" .PHP_EOL;
 428                 } else {
 429                         $fl_ar = csv_restore_files('claim', $p, $archive_ar['files']);
 430                         if (is_array($fl_ar) && count($fl_ar) > 0) {
 431                                 foreach ($fl_ar as $f) {
 432                                         csv_edihist_log("csv_move_old_stuff lost file: $f");
 433                                 }
 434                         } elseif (is_array($fl_ar) && count($fl_ar) == 0) {
 435                                 csv_edihist_log("csv_move_old_stuff archiving failed, and files restored");
 436                         } else {
 437                                 csv_edihist_log("csv_move_old_stuff archive failed and " . count($fl_ar) . " files were lost");
 438                         }
 439                         $out_html .= "Archiving error: type $type archive errors ... aborting <br />" .PHP_EOL;
 440                         return $out_html;
 441                 }
 442         } // end foreach($params as $k=>$p)
 443         //
 444         return $out_html;
 445 }
 446
 447
 448
 449
 450
 451 ?>