tools/export.pl

   1 #!/usr/bin/perl
   2
   3 #
   4 # This file is part of Koha.
   5 #
   6 # Koha is free software; you can redistribute it and/or modify it under the
   7 # terms of the GNU General Public License as published by the Free Software
   8 # Foundation; either version 2 of the License, or (at your option) any later
   9 # version.
  10 #
  11 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
  12 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13 # A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
  17 # Suite 330, Boston, MA  02111-1307 USA
  18
  19 use Modern::Perl;
  20 use List::MoreUtils qw(uniq);
  21 use Getopt::Long;
  22 use CGI;
  23 use C4::Auth;
  24 use C4::AuthoritiesMarc;    # GetAuthority
  25 use C4::Biblio;             # GetMarcBiblio GetXmlBiblio
  26 use C4::Branch;             # GetBranches
  27 use C4::Csv;
  28 use C4::Koha;               # GetItemTypes
  29 use C4::Output;
  30 use C4::Record;
  31
  32 my $query = new CGI;
  33
  34 my $clean;
  35 my $output_format;
  36 my $dont_export_items;
  37 my $deleted_barcodes;
  38 my $timestamp;
  39 my $record_type;
  40 my $help;
  41 my $op       = $query->param("op")       || '';
  42 my $filename = $query->param("filename") || 'koha.mrc';
  43 my $dbh      = C4::Context->dbh;
  44 my $marcflavour = C4::Context->preference("marcflavour");
  45 my $format = $query->param("format") || 'iso2709';
  46
  47 # Checks if the script is called from commandline
  48 my $commandline = not defined $ENV{GATEWAY_INTERFACE};
  49
  50 if ( $commandline ) {
  51
  52     # Getting parameters
  53     $op = 'export';
  54     GetOptions(
  55         'format=s'          => \$output_format,
  56         'date=s'            => \$timestamp,
  57         'dont_export_items' => \$dont_export_items,
  58         'deleted_barcodes'  => \$deleted_barcodes,
  59         'clean'             => \$clean,
  60         'filename=s'        => \$filename,
  61         'record-type=s'     => \$record_type,
  62         'help|?'            => \$help
  63     );
  64
  65     if ($help) {
  66         print <<_USAGE_;
  67 export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] --filename=outputfile
  68
  69
  70  --format=FORMAT        FORMAT is either 'xml' or 'marc' (default)
  71
  72  --date=DATE            DATE should be entered as the 'dateformat' syspref is
  73                         set (dd/mm/yyyy for metric, yyyy-mm-dd for iso,
  74                         mm/dd/yyyy for us) records exported are the ones that
  75                         have been modified since DATE
  76
  77  --record-type=TYPE     TYPE is 'bibs' or 'auths'
  78
  79  --deleted_barcodes     If used, a list of barcodes of items deleted since DATE
  80                         is produced (or from all deleted items if no date is
  81                         specified). Used only if TYPE is 'bibs'
  82
  83  --clean                removes NSE/NSB
  84 _USAGE_
  85         exit;
  86     }
  87
  88     # Default parameters values :
  89     $output_format     ||= 'marc';
  90     $timestamp         ||= '';
  91     $dont_export_items ||= 0;
  92     $deleted_barcodes  ||= 0;
  93     $clean             ||= 0;
  94     $record_type       ||= "bibs";
  95
  96     # Redirect stdout
  97     open STDOUT, '>', $filename if $filename;
  98
  99 }
 100 else {
 101
 102     $op       = $query->param("op")       || '';
 103     $filename = $query->param("filename") || 'koha.mrc';
 104     $filename =~ s/(\r|\n)//;
 105
 106 }
 107
 108 my ( $template, $loggedinuser, $cookie, $flags ) = get_template_and_user(
 109     {
 110         template_name   => "tools/export.tmpl",
 111         query           => $query,
 112         type            => "intranet",
 113         authnotrequired => $commandline,
 114         flagsrequired   => { tools => 'export_catalog' },
 115         debug           => 1,
 116     }
 117 );
 118
 119 my $limit_ind_branch =
 120   (      C4::Context->preference('IndependantBranches')
 121       && C4::Context->userenv
 122       && !( C4::Context->userenv->{flags} & 1 )
 123       && C4::Context->userenv->{branch} ) ? 1 : 0;
 124
 125 my $branch = $query->param("branch") || '';
 126 if (   C4::Context->preference("IndependantBranches")
 127     && C4::Context->userenv
 128     && !( C4::Context->userenv->{flags} & 1 ) )
 129 {
 130     $branch = C4::Context->userenv->{'branch'};
 131 }
 132
 133 my $backupdir = C4::Context->config('backupdir');
 134
 135 if ( $op eq "export" ) {
 136     if ( $format eq "iso2709" or $format eq "xml" ) {
 137         my $charset  = 'utf-8';
 138         my $mimetype = 'application/octet-stream';
 139         binmode STDOUT, ':encoding(UTF-8)';
 140         if ( $filename =~ m/\.gz$/ ) {
 141             $mimetype = 'application/x-gzip';
 142             $charset  = '';
 143             binmode STDOUT;
 144         }
 145         elsif ( $filename =~ m/\.bz2$/ ) {
 146             $mimetype = 'application/x-bzip2';
 147             binmode STDOUT;
 148             $charset = '';
 149         }
 150         print $query->header(
 151             -type       => $mimetype,
 152             -charset    => $charset,
 153             -attachment => $filename
 154         ) unless ($commandline);
 155
 156         $record_type = $query->param("record_type") unless ($commandline);
 157         $output_format = $query->param("output_format") || 'marc'
 158           unless ($commandline);
 159         my $export_remove_fields = $query->param("export_remove_fields");
 160         my @biblionumbers      = $query->param("biblionumbers");
 161         my @itemnumbers        = $query->param("itemnumbers");
 162         my @sql_params;
 163         my $sql_query;
 164         my @recordids;
 165
 166         my $StartingBiblionumber = $query->param("StartingBiblionumber");
 167         my $EndingBiblionumber   = $query->param("EndingBiblionumber");
 168         my $itemtype             = $query->param("itemtype");
 169         my $start_callnumber     = $query->param("start_callnumber");
 170         my $end_callnumber       = $query->param("end_callnumber");
 171         $timestamp = ($timestamp) ? C4::Dates->new($timestamp) : ''
 172           if ($commandline);
 173         my $start_accession =
 174           ( $query->param("start_accession") )
 175           ? C4::Dates->new( $query->param("start_accession") )
 176           : '';
 177         my $end_accession =
 178           ( $query->param("end_accession") )
 179           ? C4::Dates->new( $query->param("end_accession") )
 180           : '';
 181         $dont_export_items = $query->param("dont_export_item")
 182           unless ($commandline);
 183
 184         my $strip_nonlocal_items = $query->param("strip_nonlocal_items");
 185
 186         my $biblioitemstable =
 187           ( $commandline and $deleted_barcodes )
 188           ? 'deletedbiblioitems'
 189           : 'biblioitems';
 190         my $itemstable =
 191           ( $commandline and $deleted_barcodes )
 192           ? 'deleteditems'
 193           : 'items';
 194
 195         my $starting_authid = $query->param('starting_authid');
 196         my $ending_authid   = $query->param('ending_authid');
 197         my $authtype        = $query->param('authtype');
 198
 199         if ( $record_type eq 'bibs' and not @biblionumbers ) {
 200             if ($timestamp) {
 201
 202             # Specific query when timestamp is used
 203             # Actually it's used only with CLI and so all previous filters
 204             # are not used.
 205             # If one day timestamp is used via the web interface, this part will
 206             # certainly have to be rewrited
 207                 my ( $query, $params ) = construct_query(
 208                     {
 209                         recordtype       => $record_type,
 210                         timestamp        => $timestamp,
 211                         biblioitemstable => $biblioitemstable,
 212                     }
 213                 );
 214                 $sql_query  = $query;
 215                 @sql_params = @$params;
 216
 217             }
 218             else {
 219                 my ( $query, $params ) = construct_query(
 220                     {
 221                         recordtype           => $record_type,
 222                         biblioitemstable     => $biblioitemstable,
 223                         itemstable           => $itemstable,
 224                         StartingBiblionumber => $StartingBiblionumber,
 225                         EndingBiblionumber   => $EndingBiblionumber,
 226                         branch               => $branch,
 227                         start_callnumber     => $start_callnumber,
 228                         end_callnumber       => $end_callnumber,
 229                         start_accession      => $start_accession,
 230                         end_accession        => $end_accession,
 231                         itemtype             => $itemtype,
 232                     }
 233                 );
 234                 $sql_query  = $query;
 235                 @sql_params = @$params;
 236             }
 237         }
 238         elsif ( $record_type eq 'auths' ) {
 239             my ( $query, $params ) = construct_query(
 240                 {
 241                     recordtype      => $record_type,
 242                     starting_authid => $starting_authid,
 243                     ending_authid   => $ending_authid,
 244                     authtype        => $authtype,
 245                 }
 246             );
 247             $sql_query  = $query;
 248             @sql_params = @$params;
 249
 250         }
 251         elsif ( $record_type eq 'db' ) {
 252             my $successful_export;
 253             if ( $flags->{superlibrarian}
 254                 && C4::Context->config('backup_db_via_tools') )
 255             {
 256                 $successful_export = download_backup(
 257                     {
 258                         directory => "$backupdir",
 259                         extension => 'sql',
 260                         filename  => "$filename"
 261                     }
 262                 );
 263             }
 264             unless ($successful_export) {
 265                 my $remotehost = $query->remote_host();
 266                 $remotehost =~ s/(\n|\r)//;
 267                 warn
 268 "A suspicious attempt was made to download the db at '$filename' by someone at "
 269                   . $remotehost . "\n";
 270             }
 271             exit;
 272         }
 273         elsif ( $record_type eq 'conf' ) {
 274             my $successful_export;
 275             if ( $flags->{superlibrarian}
 276                 && C4::Context->config('backup_conf_via_tools') )
 277             {
 278                 $successful_export = download_backup(
 279                     {
 280                         directory => "$backupdir",
 281                         extension => 'tar',
 282                         filename  => "$filename"
 283                     }
 284                 );
 285             }
 286             unless ($successful_export) {
 287                 my $remotehost = $query->remote_host();
 288                 $remotehost =~ s/(\n|\r)//;
 289                 warn
 290 "A suspicious attempt was made to download the configuration at '$filename' by someone at "
 291                   . $remotehost . "\n";
 292             }
 293             exit;
 294         }
 295         elsif (@biblionumbers) {
 296             push @recordids, (@biblionumbers);
 297         }
 298         else {
 299
 300             # Someone is trying to mess us up
 301             exit;
 302         }
 303
 304         unless (@biblionumbers) {
 305             my $sth = $dbh->prepare($sql_query);
 306             $sth->execute(@sql_params);
 307             push @recordids, map {
 308                 map { $$_[0] } $_
 309             } @{ $sth->fetchall_arrayref };
 310         }
 311
 312         for my $recordid ( uniq @recordids ) {
 313             if ($deleted_barcodes) {
 314                 my $q = "
 315                     SELECT DISTINCT barcode
 316                     FROM deleteditems
 317                     WHERE deleteditems.biblionumber = ?
 318                 ";
 319                 my $sth = $dbh->prepare($q);
 320                 $sth->execute($recordid);
 321                 while ( my $row = $sth->fetchrow_array ) {
 322                     print "$row\n";
 323                 }
 324             }
 325             else {
 326                 my $record;
 327                 if ( $record_type eq 'bibs' ) {
 328                     $record = eval { GetMarcBiblio($recordid); };
 329
 330                     next if $@;
 331                     next if not defined $record;
 332                     C4::Biblio::EmbedItemsInMarcBiblio( $record, $recordid,
 333                         \@itemnumbers )
 334                       unless $dont_export_items;
 335                     if (   $strip_nonlocal_items
 336                         || $limit_ind_branch
 337                         || $dont_export_items )
 338                     {
 339                         my ( $homebranchfield, $homebranchsubfield ) =
 340                           GetMarcFromKohaField( 'items.homebranch', '' );
 341                         for my $itemfield ( $record->field($homebranchfield) ) {
 342
 343 # if stripping nonlocal items, use loggedinuser's branch if they didn't select one
 344                             $branch = C4::Context->userenv->{'branch'}
 345                               unless $branch;
 346                             $record->delete_field($itemfield)
 347                               if ( $dont_export_items
 348                                 || $itemfield->subfield($homebranchsubfield) ne
 349                                 $branch );
 350                         }
 351                     }
 352                 }
 353                 elsif ( $record_type eq 'auths' ) {
 354                     $record = C4::AuthoritiesMarc::GetAuthority($recordid);
 355                     next if not defined $record;
 356                 }
 357
 358                 if ($export_remove_fields) {
 359                     my @fields = split " ", $export_remove_fields;
 360                     foreach (@fields) {
 361                         /^(\d*)(\w)?$/;
 362                         my $field    = $1;
 363                         my $subfield = $2;
 364
 365                         # skip if this record doesn't have this field
 366                         next if not defined $record->field($field);
 367                         if ($subfield) {
 368                             $record->field($field)->delete_subfields($subfield);
 369                         }
 370                         else {
 371                             $record->delete_field( $record->field($field) );
 372                         }
 373                     }
 374                 }
 375                 RemoveAllNsb($record) if ($clean);
 376                 if ( $output_format eq "xml" ) {
 377                     if ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' )
 378                     {
 379                         print $record->as_xml_record('UNIMARCAUTH');
 380                     }
 381                     else {
 382                         print $record->as_xml_record($marcflavour);
 383                     }
 384                 }
 385                 else {
 386                     print $record->as_usmarc();
 387                 }
 388             }
 389         }
 390         exit;
 391     }
 392     elsif ( $format eq "csv" ) {
 393         my @biblionumbers = uniq $query->param("biblionumbers");
 394         my @itemnumbers   = $query->param("itemnumbers");
 395         my $output =
 396           marc2csv( \@biblionumbers,
 397             GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') ),
 398             \@itemnumbers, );
 399         print $query->header(
 400             -type                        => 'application/octet-stream',
 401             -'Content-Transfer-Encoding' => 'binary',
 402             -attachment                  => "export.csv"
 403         );
 404         print $output;
 405         exit;
 406     }
 407 }    # if export
 408
 409 else {
 410
 411     my $itemtypes = GetItemTypes;
 412     my @itemtypesloop;
 413     foreach my $thisitemtype ( sort keys %$itemtypes ) {
 414         my %row = (
 415             value       => $thisitemtype,
 416             description => $itemtypes->{$thisitemtype}->{'description'},
 417         );
 418         push @itemtypesloop, \%row;
 419     }
 420     my $branches = GetBranches($limit_ind_branch);
 421     my @branchloop;
 422     for my $thisbranch (
 423         sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} }
 424         keys %{$branches}
 425       )
 426     {
 427         push @branchloop,
 428           {
 429             value      => $thisbranch,
 430             selected   => $thisbranch eq $branch,
 431             branchname => $branches->{$thisbranch}->{'branchname'},
 432           };
 433     }
 434
 435     my $authtypes = getauthtypes;
 436     my @authtypesloop;
 437     foreach my $thisauthtype ( sort keys %$authtypes ) {
 438         next unless $thisauthtype;
 439         my %row = (
 440             value       => $thisauthtype,
 441             description => $authtypes->{$thisauthtype}->{'authtypetext'},
 442         );
 443         push @authtypesloop, \%row;
 444     }
 445
 446     if (   $flags->{superlibrarian}
 447         && C4::Context->config('backup_db_via_tools')
 448         && $backupdir
 449         && -d $backupdir )
 450     {
 451         $template->{VARS}->{'allow_db_export'} = 1;
 452         $template->{VARS}->{'dbfiles'}         = getbackupfilelist(
 453             { directory => "$backupdir", extension => 'sql' } );
 454     }
 455
 456     if (   $flags->{superlibrarian}
 457         && C4::Context->config('backup_conf_via_tools')
 458         && $backupdir
 459         && -d $backupdir )
 460     {
 461         $template->{VARS}->{'allow_conf_export'} = 1;
 462         $template->{VARS}->{'conffiles'}         = getbackupfilelist(
 463             { directory => "$backupdir", extension => 'tar' } );
 464     }
 465
 466     $template->param(
 467         branchloop               => \@branchloop,
 468         itemtypeloop             => \@itemtypesloop,
 469         DHTMLcalendar_dateformat => C4::Dates->DHTMLcalendar(),
 470         authtypeloop             => \@authtypesloop,
 471         export_remove_fields     => C4::Context->preference("ExportRemoveFields"),
 472     );
 473
 474     output_html_with_http_headers $query, $cookie, $template->output;
 475 }
 476
 477 sub construct_query {
 478     my ($params) = @_;
 479
 480     my ( $sql_query, @sql_params );
 481
 482     if ( $params->{recordtype} eq "bibs" ) {
 483         if ( $params->{timestamp} ) {
 484             my $biblioitemstable = $params->{biblioitemstable};
 485             $sql_query = " (
 486                 SELECT biblionumber
 487                 FROM $biblioitemstable
 488                   LEFT JOIN items USING(biblionumber)
 489                 WHERE $biblioitemstable.timestamp >= ?
 490                   OR items.timestamp >= ?
 491             ) UNION (
 492                 SELECT biblionumber
 493                 FROM $biblioitemstable
 494                   LEFT JOIN deleteditems USING(biblionumber)
 495                 WHERE $biblioitemstable.timestamp >= ?
 496                   OR deleteditems.timestamp >= ?
 497             ) ";
 498             my $ts = $timestamp->output('iso');
 499             @sql_params = ( $ts, $ts, $ts, $ts );
 500         }
 501         else {
 502             my $biblioitemstable     = $params->{biblioitemstable};
 503             my $itemstable           = $params->{itemstable};
 504             my $StartingBiblionumber = $params->{StartingBiblionumber};
 505             my $EndingBiblionumber   = $params->{EndingBiblionumber};
 506             my $branch               = $params->{branch};
 507             my $start_callnumber     = $params->{start_callnumber};
 508             my $end_callnumber       = $params->{end_callnumber};
 509             my $start_accession      = $params->{star_accession};
 510             my $end_accession        = $params->{end_accession};
 511             my $itemtype             = $params->{itemtype};
 512             my $items_filter =
 513                  $branch
 514               || $start_callnumber
 515               || $end_callnumber
 516               || $start_accession
 517               || $end_accession
 518               || ( $itemtype && C4::Context->preference('item-level_itypes') );
 519             $sql_query = $items_filter
 520               ? "SELECT DISTINCT $biblioitemstable.biblionumber
 521                 FROM $biblioitemstable JOIN $itemstable
 522                 USING (biblionumber) WHERE 1"
 523               : "SELECT $biblioitemstable.biblionumber FROM $biblioitemstable WHERE biblionumber >0 ";
 524
 525             if ($StartingBiblionumber) {
 526                 $sql_query .= " AND $biblioitemstable.biblionumber >= ? ";
 527                 push @sql_params, $StartingBiblionumber;
 528             }
 529
 530             if ($EndingBiblionumber) {
 531                 $sql_query .= " AND $biblioitemstable.biblionumber <= ? ";
 532                 push @sql_params, $EndingBiblionumber;
 533             }
 534
 535             if ($branch) {
 536                 $sql_query .= " AND homebranch = ? ";
 537                 push @sql_params, $branch;
 538             }
 539
 540             if ($start_callnumber) {
 541                 $sql_query .= " AND itemcallnumber >= ? ";
 542                 push @sql_params, $start_callnumber;
 543             }
 544
 545             if ($end_callnumber) {
 546                 $sql_query .= " AND itemcallnumber <= ? ";
 547                 push @sql_params, $end_callnumber;
 548             }
 549             if ($start_accession) {
 550                 $sql_query .= " AND dateaccessioned >= ? ";
 551                 push @sql_params, $start_accession->output('iso');
 552             }
 553
 554             if ($end_accession) {
 555                 $sql_query .= " AND dateaccessioned <= ? ";
 556                 push @sql_params, $end_accession->output('iso');
 557             }
 558
 559             if ($itemtype) {
 560                 $sql_query .=
 561                   ( C4::Context->preference('item-level_itypes') )
 562                   ? " AND items.itype = ? "
 563                   : " AND biblioitems.itemtype = ?";
 564                 push @sql_params, $itemtype;
 565             }
 566         }
 567     }
 568     elsif ( $params->{recordtype} eq "auths" ) {
 569         if ( $params->{timestamp} ) {
 570
 571             #TODO
 572         }
 573         else {
 574             my $starting_authid = $params->{starting_authid};
 575             my $ending_authid   = $params->{ending_authid};
 576             my $authtype        = $params->{authtype};
 577             $sql_query =
 578               "SELECT DISTINCT auth_header.authid FROM auth_header WHERE 1";
 579
 580             if ($starting_authid) {
 581                 $sql_query .= " AND auth_header.authid >= ? ";
 582                 push @sql_params, $starting_authid;
 583             }
 584
 585             if ($ending_authid) {
 586                 $sql_query .= " AND auth_header.authid <= ? ";
 587                 push @sql_params, $ending_authid;
 588             }
 589
 590             if ($authtype) {
 591                 $sql_query .= " AND auth_header.authtypecode = ? ";
 592                 push @sql_params, $authtype;
 593             }
 594         }
 595     }
 596     return ( $sql_query, \@sql_params );
 597 }
 598
 599 sub getbackupfilelist {
 600     my $args      = shift;
 601     my $directory = $args->{directory};
 602     my $extension = $args->{extension};
 603     my @files;
 604
 605     if ( opendir( my $dir, $directory ) ) {
 606         while ( my $file = readdir($dir) ) {
 607             next unless ( $file =~ m/\.$extension(\.(gz|bz2|xz))?/ );
 608             push @files, $file
 609               if ( -f "$directory/$file" && -r "$directory/$file" );
 610         }
 611         closedir($dir);
 612     }
 613     return \@files;
 614 }
 615
 616 sub download_backup {
 617     my $args      = shift;
 618     my $directory = $args->{directory};
 619     my $extension = $args->{extension};
 620     my $filename  = $args->{filename};
 621
 622     return unless ( $directory && -d $directory );
 623     return unless ( $filename =~ m/\.$extension(\.(gz|bz2|xz))?$/ );
 624     return if ( $filename =~ m#/# );
 625     $filename = "$directory/$filename";
 626     return unless ( -f $filename && -r $filename );
 627     return unless ( open( my $dump, '<', $filename ) );
 628     binmode $dump;
 629
 630     while ( read( $dump, my $data, 64 * 1024 ) ) {
 631         print $data;
 632     }
 633     close($dump);
 634     return 1;
 635 }