library/edihistory/ibr_archive.php

   1 <?php
   2 /*************  ibr_archive.php
   3  * Author:  Kevin McCormick   Longview Texas
   4  *
   5  * Copyright 2012 Kevin McCormick    Longview, Texas
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  20  * MA 02110-1301, USA.
  21  *
  22  *
  23  * Purpose: to archive old entries in the csv files and old files
  24  *
  25  * @author Kevin McCormick
  26  * @link: http://www.open-emr.org
  27  * @package OpenEMR
  28  * @subpackage ediHistory
  29  */
  30
  31 // a security measure to prevent direct web access to this file
  32 // must be accessed through the main calling script ibr_history.php
  33 // from admin at rune-city dot com;  found in php manual
  34 //if (!defined('SITE_IN')) die('Direct access not allowed!');
  35
  36 /**
  37  * @param string $date -- expect format mm/dd/yyyy
  38  * @return int $days days prior to today
  39  */
  40 function csv_days_prior($date) {
  41         //  deal with CCYY/MM/DD or MM/DD/CCYY
  42         $d1 = preg_split('/\D/', $date);
  43         if (!is_array($d1) || count($d1) != 3) {return FALSE;}
  44         //
  45         $isOK = checkdate($d1[0], $d1[1], $d1[2]) ? TRUE : checkdate($d1[1], $d1[2], $d1[0]);
  46         if (!$isOK) { return FALSE; }
  47         //
  48         $date1 = new DateTime(date("m/d/Y", $now=time()));
  49         $date2 = new DateTime($date);
  50         $dys = ($date1->format('U') - $date2->format('U')) / 86400;
  51         //
  52         return abs(floor($dys));
  53 }
  54
  55 /**
  56  * Creates a zip archive of the files in the $filename_ar array and
  57  * returns the path/name of the archive or FALSE on error
  58  *
  59  * @param array $parameters array for file type from csv_parameters
  60  * @param array $filename_ar filenames to be archived
  61  * @param string $archive_date  date of archive to be incorporated in archive file name
  62  * @return mixed   either the name of the zip archive or FALSE in case or error
  63  */
  64 function csv_zip_dir($parameters, $filename_ar, $archive_date) {
  65         // we deal with possible maximum files issues by chunking the $fn_ar array
  66         //$ulim = array();
  67         //exec("ulimit -a | grep 'open files'", $ulim);  open files  (-n) 1024
  68         //
  69         $f_max = 200;
  70         $fn_ar2 = array();
  71         if ( count($filename_ar) > $f_max) {
  72                 $fn_ar2 = array_chunk($filename_ar, $f_max);
  73         } else {
  74                 $fn_ar2[] = $filename_ar;
  75         }
  76         //
  77     $zpath = csv_edih_basedir();
  78     $ztemp = csv_edih_tmpdir();
  79         $zip_name = $zpath.DIRECTORY_SEPARATOR."archive".DIRECTORY_SEPARATOR.$type."_$archive_date.zip";
  80         $ftmpn = $ztemp.DIRECTORY_SEPARATOR;
  81         $fdir = $parameters['directory'].DIRECTORY_SEPARATOR;
  82         $type = $parameters['type'];
  83         //
  84         $zip_obj = new ZipArchive();
  85         //
  86         foreach($fn_ar2 as $fnz) {
  87                 // reopen the zip archive on each loop so the open file count is controlled
  88                 if (is_file($zip_name) ) {
  89                         $isOK = $zip_obj->open($zip_name, ZIPARCHIVE::CHECKCONS);
  90                         $isNew = FALSE;
  91                 } else {
  92                         $isOK = $zip_obj->open($zip_name, ZIPARCHIVE::CREATE);
  93                         $isNew = $isOK;
  94                 }
  95                 //
  96                 if ($isOK && $isNew) {
  97                         $zip_obj->addEmptyDir($type);
  98                         $zip_obj->setArchiveComment("archive " . $fdir . "prior to $archive_date");
  99                 }
 100                 //
 101                 if ($isOK) {
 102                         // we are working with the open archive
 103                         // now add the files to the archive
 104                         foreach($fnz as $fz) {
 105                                 if (is_file($fdir.$fz) ) {
 106                                         $iscp = copy($fdir.$fz, $ftmpn.$fz);
 107                                         $isOK = $zip_obj->addFile($ftmpn.$fz, $type.DIRECTORY_SEPARATOR.$fz);
 108                                 } else {
 109                                         csv_edihist_log("csv_zip_dir: in record, but not in directory $fz ");
 110                                         // possible that file is in csv table, but not in directory?
 111                                 }
 112                                 //
 113                                 if ($isOK && $iscp) {
 114                                         // if we have added the file to the archive, remove it from the storage directory
 115                                         // but keep the /tmp file copy for now
 116                                         unlink($fdir.$fz);
 117                                 } else {
 118                                         $msg = $zip_obj->getStatusString();
 119                                         csv_edihist_log("csv_zip_dir: $type ZipArchive failed for $fz  $msg");
 120                                 }
 121                         } // end foreach
 122                 } else {
 123                         // ZipArchive open() failed -- try to get the error message and return false
 124                         $msg = $zip_obj->getStatusString();
 125                         csv_edihist_log("csv_zip_dir: $type ZipArchive open() failed  $msg");
 126                         return $isOK;
 127                 }
 128                 //
 129                 // errors on close would be non-existing file added or something else
 130                 $isOK = $zip_obj->close($zip_name);
 131                 if (!$isOK) {
 132                         $msg = $zip_obj->getStatusString();
 133                         csv_edihist_log("csv_zip_dir: $type ZipArchive close() error for $fz  $msg");
 134                         //
 135                         return $isOK;
 136                 }
 137         }  // end foreach($fn_ar2 as $fnz)
 138         //
 139         return ($isOK) ? $zip_name : $isOK;
 140 }
 141
 142
 143 /**
 144  * restores files from a zip archive or the tmp dir if the archive process needs to be aborted
 145  *
 146  * @param string $csv_type     either 'file' or 'claim'
 147  * @param array $parameters    the parameters array for the particular type
 148  * @param array $filename_ar   array of file names that may have been deleted
 149  */
 150 function csv_restore_files($csv_type, $parameters, $filename_ar) {
 151         //
 152         if (!is_array($filename_ar) || !count($filename_ar)) { return FALSE; }
 153         //
 154         $csv_p = ($csv_type == 'file') ? $parameters['files_csv'] : $parameters['claims_csv'];
 155         $csv_dir .= dirname($csv_p).DIRECTORY_SEPARATOR;
 156         $csv_file = basename($csv_p);
 157         //
 158         $fdir = $parameters['directory'] . DIRECTORY_SEPARATOR;
 159     if (!is_dir($fdir) ) {
 160         csv_edihist_log("csv_restore_files: missing directory $fdir");
 161         return FALSE;
 162     }
 163         //
 164         $fileslost = array();
 165         //
 166         // we are in a jam -- archive is messed up
 167     $ntmpname = csv_edih_tmpdir();
 168         $ntmpname .= $ntmpname.DIRECTORY_SEPARATOR;
 169         //
 170         foreach($filename_ar as $fnz) {
 171                 foreach($fnz as $fz) {
 172                         //
 173                         if (is_file($fdir.$fz) ) {
 174                                 // file is still in our type directory
 175                                 continue;
 176                         } else {
 177                                 $iscp = copy($ntmpname.$fz, $fdir.$fz);
 178                                 if (!$iscp) {
 179                                         csv_edihist_log("csv_restore_files: $type archive restore failed for $fz");
 180                                         $fileslost[] = $fz;
 181                                 }
 182                         }
 183                 }
 184         }
 185         // put the csv file back
 186         $ntmpcsv = $ntmpname.DIRECTORY_SEPARATOR.$csv_file;
 187         $iscp = copy($ntmpcsv, $csv_dir.$csv_file);
 188         if (!$iscp) {
 189         csv_edihist_log("csv_restore_files: archive restore may have lost $csv_file");
 190         $fileslost[] = $csv_file;
 191     }
 192         //
 193         return $fileslost;
 194 }
 195
 196
 197 /**
 198  * After the archive is created, the csv record needs to be re-written so the archived
 199  * files are not in the csv file and hence, not searched for
 200  *
 201  * @param string $csv_path   the tmp csv file path is expected
 202  * @param array $heading_ar  the column heading for the csv file
 203  * @param array $row_array   the data rows to be written to the file
 204  * @return integer           count the characters written as returned by fputcsv()
 205  */
 206 function csv_rewrite_record($csv_path, $heading_ar, $row_array) {
 207         //
 208         // count characters written -- returned by fputcsv
 209         $ocwct = 0;
 210         $fh3 = fopen($csv_path, 'w');
 211
 212         // if we fail to open the file, return the result, expect FALSE
 213         if (!$fh3) {
 214                 csv_edihist_log("csv_rewrite_record: failed to open $csv_path");
 215         } else {
 216                 // it is a an empty file, so write the heading row
 217                 if (count($row_array) ) {
 218                         $ocwct += fputcsv ( $fh3, $heading_ar );
 219                         // wrote heading, now add rows
 220                         foreach($row_array as $row) {
 221                                 $ocwct += fputcsv ($fh3, $row );
 222                         }
 223                         fclose($fh3);
 224                 } else {
 225                         csv_edihist_log("csv_rewrite_record: empty records array passed for $csv_path");
 226                 }
 227         }
 228         csv_edihist_log("csv_rewrite_record: wrote $ocwct characters " . count($row_array) . " rows to $csv_path");
 229         return $ocwct;
 230 }
 231
 232
 233 /**
 234  * Reads the current csv file and divides it into two arrays 'arch_csv' and 'curr_csv'
 235  * the 'arch_csv' contains the rows that will be archived (prior to archive_date)
 236  * and the 'curr_csv' contains the rows that will be retained
 237  * Also, if the csv_type is 'file' an array of file names is stored under 'files' key
 238  *
 239  * @param string $csv_type
 240  * @param string $csv_path
 241  * @param integer $date_col
 242  * @param integer $fn_column
 243  * @param string $archive_date
 244  * @return array                $arch_ar keys ['arch_csv'] ['curr_csv'] ['files']
 245  */
 246 function csv_archive_array($csv_type, $csv_path, $date_col, $fn_column, $archive_date) {
 247         //
 248         $fh = fopen($csv_path, 'r');
 249         $idx = 0;
 250         //
 251         $arch_ar = array();
 252         //
 253         $dt = strpos($archive_date, '/') ? str_replace('/', '', $archive_date) : $archive_date;
 254         //
 255         if ($fh !== FALSE) {
 256                 while (($data = fgetcsv($fh)) !== FALSE) {
 257                         //
 258                         if ($idx == 0) {
 259                                 $arch_csv[] = $data;
 260                         } else {
 261                                 $isok = (substr($data[$date_col], 0, 8) < $dt) ? TRUE : FALSE;
 262                                 if ($isok) {
 263                                         $arch_ar['arch_csv'][] = $data;
 264                                         if ($csv_type == 'file') { $arch_ar['files'][] = $data[$fn_column]; }
 265                                 } else {
 266                                         // retained csv rows go here
 267                                         $arch_ar['curr_csv'][] = $data;
 268                                 }
 269                         }
 270                         $idx++;
 271                 }
 272                 flock($fh2, LOCK_UN);
 273                 fclose($fh);
 274         } else {
 275                 csv_edihist_log("csv_archive_array: failed to open " . basename($csv_path));
 276                 return FALSE;
 277         }
 278         return $arch_ar;
 279 }
 280
 281 /**
 282  * The main function in this ibr_archive.php script.  This function gets the parameters array
 283  * from csv_parameters() and calls the archiving functions on each type of file
 284  * in the parameters array.
 285  *
 286  * @param string $archive_date    yyyy/mm/dd date prior to which is archived
 287  * @return string                 descriptive message in html format
 288  */
 289 function csv_archive_old($archive_date) {
 290         //
 291         // paths
 292     $edih_dir = csv_edih_basedir();
 293     $archive_dir = $edih_dir.DIRECTORY_SEPARATOR.'archive';
 294     $csv_dir = $edih_dir.DIRECTORY_SEPARATOR.'csv';
 295     $tmp_dir = csv_edih_tmpdir();
 296     $tmp_dir .= $tmp_dir.DIRECTORY_SEPARATOR;
 297     //
 298         if (!is_dir($edih_dir.DIRECTORY_SEPARATOR.'archive') ) {
 299                 // should have been created at setup
 300                 mkdir ($edih_dir.DIRECTORY_SEPARATOR.'archive', 0755);
 301         }
 302         //
 303         $days = csv_days_prior($archive_date);
 304         if (!$days || $days < 90 ) {
 305                 $out_html = "Archive date $archive_date invalid or less than 90 days prior <br />" .PHP_EOL;
 306                 return $out_html;
 307         }
 308         //
 309         $out_html = "Archiving records prior to $archive_date <br />" .PHP_EOL;
 310         //
 311         $dt = str_replace('/', '', $archive_date);
 312         //
 313         $isarchived = FALSE;
 314         $haserr = FALSE;
 315         $params = csv_parameters();
 316         //
 317         $f_max = 200;
 318         //
 319         foreach($params as $k=>$p) {
 320                 $type = $p['type'];
 321                 $fdir = $p['directory'] . DIRECTORY_SEPARATOR;
 322                 //
 323                 $fn_ar = array();
 324                 $arch_csv = array();
 325                 $curr_csvd = array();
 326                 //
 327                 $archive_ar = array();
 328                 //
 329                 // type dpr has only a claim csv type
 330                 $head_ar = ($type == 'dpr') ?  csv_files_header($type, 'claim') : csv_files_header($type, 'file');
 331                 //
 332                 $fncol = $p['fncolumn'];
 333                 $datecol = $p['datecolumn'];
 334                 //
 335                 // files csv temporary names
 336                 $file_csv = $p['files_csv'];
 337                 $file_csv_copy = $tmp_dir.basename($file_csv);
 338                 $tmp_fold_csv = $tmp_dir.$type.'_old_'.basename($file_csv);
 339                 $tmp_fnew_csv = $tmp_dir.$type.'_new_'.basename($file_csv);
 340                 $iscpf = copy ($file_csv, $file_csv_copy);
 341                 //
 342                 // claims csv temporary names
 343                 $claim_csv = $p['claims_csv'];
 344                 $claim_csv_copy = $tmp_dir.basename($claim_csv);
 345                 $tmp_cold_csv = $tmp_dir.$type.'_old_'.basename($claim_csv);
 346                 $tmp_cnew_csv = $tmp_dir.$type.'_new_'.basename($claim_csv);
 347                 $iscpc = copy ($claim_csv, $claim_csv_copy);
 348                 //
 349                 if (!$iscpf || !$iscpc) {
 350                         csv_edihist_log("csv_archive_old: copy to tmp dir failed for csv file $type");
 351                         $out_html = "Archive temporary files operation failed ... aborting <br />" .PHP_EOL;
 352                         return $out_html;
 353                 }
 354                 //
 355                 // lock the original files
 356                 $fh1 = fopen($file_csv, 'r');
 357                 $islk1 = flock($fh1, LOCK_EX);
 358                 if (!$islk1) { fclose($fh1) ; } // assume we are on a system that does not support locks
 359                 $fh2 = fopen($claim_csv, 'r');
 360                 $islk2 = flock($fh2, LOCK_EX);
 361                 if (!$islk2) { fclose($fh2) ; } // assume we are on a system that does not support locks
 362                 //
 363                 // do the archive for the files_type.csv
 364                 $archive_ar = csv_archive_array('file', $file_csv_copy, $datecol, $fncol, $dt);
 365                 if (!$archive_ar) {
 366                         csv_edihist_log("csv_archive_old: creating archive information failed for " . basename($file_csv_copy));
 367                         continue;
 368                 }
 369                 $och = csv_rewrite_record($tmp_old_csv, $head_ar, $archive_ar['arch_csv']);
 370                 $nch = csv_rewrite_record($tmp_new_csv, $head_ar, $archive_ar['curr_csv']);
 371                 $zarch = csv_zip_dir($params, $archive_ar['files'], $archive_date);
 372                 // now move the reconfigured files
 373                 // unlink the present csv file, since it is possible for a rename error if it exists
 374                 $islk1 = ($islk1) ? flock($fh1, LOCK_UN) : $islk1;
 375                 if ($islk1) { fclose($fh1); }
 376                 $isunl = unlink($file_csv);
 377         if ($zarch) {
 378             // we got back the zip archive name from csv_zip_dir()
 379             $ismvz = rename($zarch, $archive_dir.DIRECTORY_SEPARATOR.basename($zarch) );
 380             $ismvo = rename($tmp_fold_csv, $archive_dir.DIRECTORY_SEPARATOR.$dt.basename($tmp_fold_csv) );
 381             $ismvn = rename($tmp_fnew_csv, $file_csv );
 382             //
 383             if ($ismvz && $ismvo && $ismvn) {
 384                 // everything is working - clear out the files we put in tmp_dir
 385                 // the tmp dir should be empty, but there might have been something else created there
 386                 $isclr = csv_clear_tmpdir();
 387                 $out_html .= "Archived: type $type <br />" .PHP_EOL;
 388                 $out_html .= "&nbsp; archived " .count($archive_ar['files']) . " files   <br />" .PHP_EOL;
 389                 $out_html .= "&nbsp; archived " .count($archive_ar['arch_csv']) . " rows from " .basename($file_csv) ." <br />" .PHP_EOL;
 390                 $out_html .= "&nbsp; there are now " .count($archive_ar['curr_csv']) . " rows in " .basename($file_csv) ." <br />" .PHP_EOL;
 391             } else {
 392                 // in case or error, try to restore everything
 393                 $fl_ar = csv_restore_files('file', $p, $archive_ar['files']);
 394                 if (is_array($fl_ar) && count($fl_ar) > 0) {
 395                     foreach ($fl_ar as $f) {
 396                         csv_edihist_log("csv_archive_old: lost file $f");
 397                     }
 398                 } elseif (is_array($fl_ar) && count($fl_ar) == 0) {
 399                     csv_edihist_log("csv_archive_old archiving failed, and files restored");
 400                 } else {
 401                     csv_edihist_log("csv_archive_old archive failed and files were lost");
 402                 }
 403                 // give a message and quit
 404                 $out_html .= "Archiving error: type $type archive errors ... aborting <br />" .PHP_EOL;
 405                 return $out_html;
 406             }
 407         } else {
 408             // zip create error
 409             csv_edihist_log("csv_archive_old: creating zip archive failed for " . basename($file_csv));
 410             $fl_ar = csv_restore_files('file', $p, $archive_ar['files']);
 411             if (is_array($fl_ar) && count($fl_ar) > 0) {
 412                 foreach ($fl_ar as $f) {
 413                     csv_edihist_log("csv_archive_old: lost file $f");
 414                 }
 415             }
 416             $out_html .= "Archiving error: type $type archive errors ... aborting <br />" .PHP_EOL;
 417             return $out_html;
 418         }
 419                 //
 420                 // now we do the claims table
 421                 //$cldate = date_create($archive_date);
 422         //date_sub($cldate, date_interval_create_from_date_string('1 month'));
 423         //$cldt = date_format($cldate, 'Ymd');
 424         //
 425                 // dpr type has only claim table, treated as a file table above
 426                 if ($type == 'dpr') { continue; }
 427                 //
 428                 $head_ar = csv_files_header($type, 'claim');
 429                 //
 430                 $archive_ar = csv_archive_array('claim', $claim_csv_copy, $datecol, $fncol, $dt);
 431                 if (!$archive_ar) {
 432                         csv_edihist_log("csv_archive_old: creating archive information failed for " . basename($file_csv_copy));
 433                         continue;
 434                 }
 435                 //
 436                 $och = csv_rewrite_record($tmp_cold_csv, $head_ar, $archive_ar['arch_csv']);
 437                 $nch = csv_rewrite_record($tmp_cnew_csv, $head_ar, $archive_ar['curr_csv']);
 438                 //
 439                 $islk2 = ($islk2) ? flock($fh2, LOCK_UN) : $islk2;
 440                 if ($islk2) { fclose($fh2); }
 441                 $isunl = unlink($claim_csv);
 442                 $ismvo = rename($tmp_cold_csv, $archive_dir.DIRECTORY_SEPARATOR.$dt.basename($tmp_cold_csv) );
 443                 $ismvn = rename($tmp_cnew_csv, $claim_csv );
 444                 //
 445                 if ($ismvo && $ismvn) {
 446                         // everything is working - clear out the files we put in tmp_dir
 447                         // the tmp dir should be empty, but there might have been something else created there
 448                         $isclr = csv_clear_tmpdir();
 449                         $out_html .= "&nbsp; archived " .count($archive_ar['arch_csv']) . " rows from " . basename($claim_csv) ." <br />" .PHP_EOL;
 450                         $out_html .= "&nbsp; there are now " .count($archive_ar['curr_csv']) . " rows in " . basename($claim_csv) ." <br />" .PHP_EOL;
 451                 } else {
 452                         $fl_ar = csv_restore_files('claim', $p, $archive_ar['files']);
 453                         if (is_array($fl_ar) && count($fl_ar) > 0) {
 454                                 foreach ($fl_ar as $f) {
 455                                         csv_edihist_log("csv_archive_old: lost file $f");
 456                                 }
 457                         } elseif (is_array($fl_ar) && count($fl_ar) == 0) {
 458                                 csv_edihist_log("csv_archive_old: archiving failed, and files restored");
 459                         } else {
 460                                 csv_edihist_log("csv_archive_old: archive failed and " . count($fl_ar) . " files were lost");
 461                         }
 462                         $out_html .= "Archiving error: type $type archive errors ... aborting <br />" .PHP_EOL;
 463                         return $out_html;
 464                 }
 465         } // end foreach($params as $k=>$p)
 466         //
 467         return $out_html;
 468 }
 469
 470
 471
 472
 473
 474 ?>