C4/ImportBatch.pm

   1 package C4::ImportBatch;
   2
   3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
   4 #
   5 # This file is part of Koha.
   6 #
   7 # Koha is free software; you can redistribute it and/or modify it under the
   8 # terms of the GNU General Public License as published by the Free Software
   9 # Foundation; either version 2 of the License, or (at your option) any later
  10 # version.
  11 #
  12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
  13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  14 # A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License along
  17 # with Koha; if not, write to the Free Software Foundation, Inc.,
  18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19
  20 use strict;
  21 use warnings;
  22
  23 use C4::Context;
  24 use C4::Koha;
  25 use C4::Biblio;
  26 use C4::Items;
  27 use C4::Charset;
  28 use C4::AuthoritiesMarc;
  29 use C4::MarcModificationTemplates;
  30
  31 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
  32
  33 BEGIN {
  34         # set the version for version checking
  35     $VERSION = 3.07.00.049;
  36         require Exporter;
  37         @ISA    = qw(Exporter);
  38         @EXPORT = qw(
  39     GetZ3950BatchId
  40     GetWebserviceBatchId
  41     GetImportRecordMarc
  42     GetImportRecordMarcXML
  43     AddImportBatch
  44     GetImportBatch
  45     AddAuthToBatch
  46     AddBiblioToBatch
  47     AddItemsToImportBiblio
  48     ModAuthorityInBatch
  49     ModBiblioInBatch
  50
  51     BatchStageMarcRecords
  52     BatchFindDuplicates
  53     BatchCommitRecords
  54     BatchRevertRecords
  55     CleanBatch
  56
  57     GetAllImportBatches
  58     GetStagedWebserviceBatches
  59     GetImportBatchRangeDesc
  60     GetNumberOfNonZ3950ImportBatches
  61     GetImportBiblios
  62     GetImportRecordsRange
  63         GetItemNumbersFromImportBatch
  64
  65     GetImportBatchStatus
  66     SetImportBatchStatus
  67     GetImportBatchOverlayAction
  68     SetImportBatchOverlayAction
  69     GetImportBatchNoMatchAction
  70     SetImportBatchNoMatchAction
  71     GetImportBatchItemAction
  72     SetImportBatchItemAction
  73     GetImportBatchMatcher
  74     SetImportBatchMatcher
  75     GetImportRecordOverlayStatus
  76     SetImportRecordOverlayStatus
  77     GetImportRecordStatus
  78     SetImportRecordStatus
  79     GetImportRecordMatches
  80     SetImportRecordMatches
  81         );
  82 }
  83
  84 =head1 NAME
  85
  86 C4::ImportBatch - manage batches of imported MARC records
  87
  88 =head1 SYNOPSIS
  89
  90 use C4::ImportBatch;
  91
  92 =head1 FUNCTIONS
  93
  94 =head2 GetZ3950BatchId
  95
  96   my $batchid = GetZ3950BatchId($z3950server);
  97
  98 Retrieves the ID of the import batch for the Z39.50
  99 reservoir for the given target.  If necessary,
 100 creates the import batch.
 101
 102 =cut
 103
 104 sub GetZ3950BatchId {
 105     my ($z3950server) = @_;
 106
 107     my $dbh = C4::Context->dbh;
 108     my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
 109                              WHERE  batch_type = 'z3950'
 110                              AND    file_name = ?");
 111     $sth->execute($z3950server);
 112     my $rowref = $sth->fetchrow_arrayref();
 113     $sth->finish();
 114     if (defined $rowref) {
 115         return $rowref->[0];
 116     } else {
 117         my $batch_id = AddImportBatch( {
 118                 overlay_action => 'create_new',
 119                 import_status => 'staged',
 120                 batch_type => 'z3950',
 121                 file_name => $z3950server,
 122             } );
 123         return $batch_id;
 124     }
 125
 126 }
 127
 128 =head2 GetWebserviceBatchId
 129
 130   my $batchid = GetWebserviceBatchId();
 131
 132 Retrieves the ID of the import batch for webservice.
 133 If necessary, creates the import batch.
 134
 135 =cut
 136
 137 my $WEBSERVICE_BASE_QRY = <<EOQ;
 138 SELECT import_batch_id FROM import_batches
 139 WHERE  batch_type = 'webservice'
 140 AND    import_status = 'staged'
 141 EOQ
 142 sub GetWebserviceBatchId {
 143     my ($params) = @_;
 144
 145     my $dbh = C4::Context->dbh;
 146     my $sql = $WEBSERVICE_BASE_QRY;
 147     my @args;
 148     foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
 149         if (my $val = $params->{$field}) {
 150             $sql .= " AND $field = ?";
 151             push @args, $val;
 152         }
 153     }
 154     my $id = $dbh->selectrow_array($sql, undef, @args);
 155     return $id if $id;
 156
 157     $params->{batch_type} = 'webservice';
 158     $params->{import_status} = 'staged';
 159     return AddImportBatch($params);
 160 }
 161
 162 =head2 GetImportRecordMarc
 163
 164   my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
 165
 166 =cut
 167
 168 sub GetImportRecordMarc {
 169     my ($import_record_id) = @_;
 170
 171     my $dbh = C4::Context->dbh;
 172     my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?");
 173     $sth->execute($import_record_id);
 174     my ($marc, $encoding) = $sth->fetchrow();
 175     $sth->finish();
 176     return $marc, $encoding;
 177
 178 }
 179
 180 =head2 GetImportRecordMarcXML
 181
 182   my $marcxml = GetImportRecordMarcXML($import_record_id);
 183
 184 =cut
 185
 186 sub GetImportRecordMarcXML {
 187     my ($import_record_id) = @_;
 188
 189     my $dbh = C4::Context->dbh;
 190     my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?");
 191     $sth->execute($import_record_id);
 192     my ($marcxml) = $sth->fetchrow();
 193     $sth->finish();
 194     return $marcxml;
 195
 196 }
 197
 198 =head2 AddImportBatch
 199
 200   my $batch_id = AddImportBatch($params_hash);
 201
 202 =cut
 203
 204 sub AddImportBatch {
 205     my ($params) = @_;
 206
 207     my (@fields, @vals);
 208     foreach (qw( matcher_id template_id branchcode
 209                  overlay_action nomatch_action item_action
 210                  import_status batch_type file_name comments record_type )) {
 211         if (exists $params->{$_}) {
 212             push @fields, $_;
 213             push @vals, $params->{$_};
 214         }
 215     }
 216     my $dbh = C4::Context->dbh;
 217     $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
 218                                   VALUES (".join( ',', map '?', @fields).")",
 219              undef,
 220              @vals);
 221     return $dbh->{'mysql_insertid'};
 222 }
 223
 224 =head2 GetImportBatch
 225
 226   my $row = GetImportBatch($batch_id);
 227
 228 Retrieve a hashref of an import_batches row.
 229
 230 =cut
 231
 232 sub GetImportBatch {
 233     my ($batch_id) = @_;
 234
 235     my $dbh = C4::Context->dbh;
 236     my $sth = $dbh->prepare_cached("SELECT * FROM import_batches WHERE import_batch_id = ?");
 237     $sth->bind_param(1, $batch_id);
 238     $sth->execute();
 239     my $result = $sth->fetchrow_hashref;
 240     $sth->finish();
 241     return $result;
 242
 243 }
 244
 245 =head2 AddBiblioToBatch
 246
 247   my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
 248                 $marc_record, $encoding, $z3950random, $update_counts);
 249
 250 =cut
 251
 252 sub AddBiblioToBatch {
 253     my $batch_id = shift;
 254     my $record_sequence = shift;
 255     my $marc_record = shift;
 256     my $encoding = shift;
 257     my $z3950random = shift;
 258     my $update_counts = @_ ? shift : 1;
 259
 260     my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour'));
 261     _add_biblio_fields($import_record_id, $marc_record);
 262     _update_batch_record_counts($batch_id) if $update_counts;
 263     return $import_record_id;
 264 }
 265
 266 =head2 ModBiblioInBatch
 267
 268   ModBiblioInBatch($import_record_id, $marc_record);
 269
 270 =cut
 271
 272 sub ModBiblioInBatch {
 273     my ($import_record_id, $marc_record) = @_;
 274
 275     _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
 276     _update_biblio_fields($import_record_id, $marc_record);
 277
 278 }
 279
 280 =head2 AddAuthToBatch
 281
 282   my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
 283                 $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]);
 284
 285 =cut
 286
 287 sub AddAuthToBatch {
 288     my $batch_id = shift;
 289     my $record_sequence = shift;
 290     my $marc_record = shift;
 291     my $encoding = shift;
 292     my $z3950random = shift;
 293     my $update_counts = @_ ? shift : 1;
 294     my $marc_type = shift || C4::Context->preference('marcflavour');
 295
 296     $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC';
 297
 298     my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type);
 299     _add_auth_fields($import_record_id, $marc_record);
 300     _update_batch_record_counts($batch_id) if $update_counts;
 301     return $import_record_id;
 302 }
 303
 304 =head2 ModAuthInBatch
 305
 306   ModAuthInBatch($import_record_id, $marc_record);
 307
 308 =cut
 309
 310 sub ModAuthInBatch {
 311     my ($import_record_id, $marc_record) = @_;
 312
 313     my $marcflavour = C4::Context->preference('marcflavour');
 314     _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC');
 315
 316 }
 317
 318 =head2 BatchStageMarcRecords
 319
 320   ($batch_id, $num_records, $num_items, @invalid_records) =
 321     BatchStageMarcRecords($encoding, $marc_records, $file_name, $marc_modification_template,
 322                           $comments, $branch_code, $parse_items,
 323                           $leave_as_staging,
 324                           $progress_interval, $progress_callback);
 325
 326 =cut
 327
 328 sub BatchStageMarcRecords {
 329     my $record_type = shift;
 330     my $encoding = shift;
 331     my $marc_records = shift;
 332     my $file_name = shift;
 333     my $marc_modification_template = shift;
 334     my $comments = shift;
 335     my $branch_code = shift;
 336     my $parse_items = shift;
 337     my $leave_as_staging = shift;
 338
 339     # optional callback to monitor status
 340     # of job
 341     my $progress_interval = 0;
 342     my $progress_callback = undef;
 343     if ($#_ == 1) {
 344         $progress_interval = shift;
 345         $progress_callback = shift;
 346         $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
 347         $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
 348     }
 349
 350     my $batch_id = AddImportBatch( {
 351             overlay_action => 'create_new',
 352             import_status => 'staging',
 353             batch_type => 'batch',
 354             file_name => $file_name,
 355             comments => $comments,
 356             record_type => $record_type,
 357         } );
 358     if ($parse_items) {
 359         SetImportBatchItemAction($batch_id, 'always_add');
 360     } else {
 361         SetImportBatchItemAction($batch_id, 'ignore');
 362     }
 363
 364     my $marc_type = C4::Context->preference('marcflavour');
 365     $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
 366     my @invalid_records = ();
 367     my $num_valid = 0;
 368     my $num_items = 0;
 369     # FIXME - for now, we're dealing only with bibs
 370     my $rec_num = 0;
 371     foreach my $marc_blob (split(/\x1D/, $marc_records)) {
 372         $marc_blob =~ s/^\s+//g;
 373         $marc_blob =~ s/\s+$//g;
 374         next unless $marc_blob;
 375         $rec_num++;
 376         if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
 377             &$progress_callback($rec_num);
 378         }
 379         my ($marc_record, $charset_guessed, $char_errors) =
 380             MarcToUTF8Record($marc_blob, $marc_type, $encoding);
 381
 382         $encoding = $charset_guessed unless $encoding;
 383
 384         ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
 385
 386         my $import_record_id;
 387         if (scalar($marc_record->fields()) == 0) {
 388             push @invalid_records, $marc_blob;
 389         } else {
 390
 391             # Normalize the record so it doesn't have separated diacritics
 392             SetUTF8Flag($marc_record);
 393
 394             $num_valid++;
 395             if ($record_type eq 'biblio') {
 396                 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
 397                 if ($parse_items) {
 398                     my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
 399                     $num_items += scalar(@import_items_ids);
 400                 }
 401             } elsif ($record_type eq 'auth') {
 402                 $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type);
 403             }
 404         }
 405     }
 406     unless ($leave_as_staging) {
 407         SetImportBatchStatus($batch_id, 'staged');
 408     }
 409     # FIXME branch_code, number of bibs, number of items
 410     _update_batch_record_counts($batch_id);
 411     return ($batch_id, $num_valid, $num_items, @invalid_records);
 412 }
 413
 414 =head2 AddItemsToImportBiblio
 415
 416   my @import_items_ids = AddItemsToImportBiblio($batch_id,
 417                 $import_record_id, $marc_record, $update_counts);
 418
 419 =cut
 420
 421 sub AddItemsToImportBiblio {
 422     my $batch_id = shift;
 423     my $import_record_id = shift;
 424     my $marc_record = shift;
 425     my $update_counts = @_ ? shift : 0;
 426
 427     my @import_items_ids = ();
 428
 429     my $dbh = C4::Context->dbh;
 430     my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
 431     foreach my $item_field ($marc_record->field($item_tag)) {
 432         my $item_marc = MARC::Record->new();
 433         $item_marc->leader("00000    a              "); # must set Leader/09 to 'a'
 434         $item_marc->append_fields($item_field);
 435         $marc_record->delete_field($item_field);
 436         my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
 437                                         VALUES (?, ?, ?)");
 438         $sth->bind_param(1, $import_record_id);
 439         $sth->bind_param(2, 'staged');
 440         $sth->bind_param(3, $item_marc->as_xml());
 441         $sth->execute();
 442         push @import_items_ids, $dbh->{'mysql_insertid'};
 443         $sth->finish();
 444     }
 445
 446     if ($#import_items_ids > -1) {
 447         _update_batch_record_counts($batch_id) if $update_counts;
 448         _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
 449     }
 450     return @import_items_ids;
 451 }
 452
 453 =head2 BatchFindDuplicates
 454
 455   my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
 456              $max_matches, $progress_interval, $progress_callback);
 457
 458 Goes through the records loaded in the batch and attempts to
 459 find duplicates for each one.  Sets the matching status
 460 of each record to "no_match" or "auto_match" as appropriate.
 461
 462 The $max_matches parameter is optional; if it is not supplied,
 463 it defaults to 10.
 464
 465 The $progress_interval and $progress_callback parameters are
 466 optional; if both are supplied, the sub referred to by
 467 $progress_callback will be invoked every $progress_interval
 468 records using the number of records processed as the
 469 singular argument.
 470
 471 =cut
 472
 473 sub BatchFindDuplicates {
 474     my $batch_id = shift;
 475     my $matcher = shift;
 476     my $max_matches = @_ ? shift : 10;
 477
 478     # optional callback to monitor status
 479     # of job
 480     my $progress_interval = 0;
 481     my $progress_callback = undef;
 482     if ($#_ == 1) {
 483         $progress_interval = shift;
 484         $progress_callback = shift;
 485         $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
 486         $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
 487     }
 488
 489     my $dbh = C4::Context->dbh;
 490
 491     my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc
 492                              FROM import_records
 493                              WHERE import_batch_id = ?");
 494     $sth->execute($batch_id);
 495     my $num_with_matches = 0;
 496     my $rec_num = 0;
 497     while (my $rowref = $sth->fetchrow_hashref) {
 498         $rec_num++;
 499         if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
 500             &$progress_callback($rec_num);
 501         }
 502         my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
 503         my @matches = ();
 504         if (defined $matcher) {
 505             @matches = $matcher->get_matches($marc_record, $max_matches);
 506         }
 507         if (scalar(@matches) > 0) {
 508             $num_with_matches++;
 509             SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
 510             SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
 511         } else {
 512             SetImportRecordMatches($rowref->{'import_record_id'}, ());
 513             SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
 514         }
 515     }
 516     $sth->finish();
 517     return $num_with_matches;
 518 }
 519
 520 =head2 BatchCommitRecords
 521
 522   my ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored) =
 523         BatchCommitRecords($batch_id, $framework,
 524         $progress_interval, $progress_callback);
 525
 526 =cut
 527
 528 sub BatchCommitRecords {
 529     my $batch_id = shift;
 530     my $framework = shift;
 531
 532     # optional callback to monitor status
 533     # of job
 534     my $progress_interval = 0;
 535     my $progress_callback = undef;
 536     if ($#_ == 1) {
 537         $progress_interval = shift;
 538         $progress_callback = shift;
 539         $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
 540         $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
 541     }
 542
 543     my $record_type;
 544     my $num_added = 0;
 545     my $num_updated = 0;
 546     my $num_items_added = 0;
 547     my $num_items_replaced = 0;
 548     my $num_items_errored = 0;
 549     my $num_ignored = 0;
 550     # commit (i.e., save, all records in the batch)
 551     SetImportBatchStatus('importing');
 552     my $overlay_action = GetImportBatchOverlayAction($batch_id);
 553     my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
 554     my $item_action = GetImportBatchItemAction($batch_id);
 555     my $item_tag;
 556     my $item_subfield;
 557     my $dbh = C4::Context->dbh;
 558     my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
 559                              FROM import_records
 560                              LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
 561                              LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
 562                              WHERE import_batch_id = ?");
 563     $sth->execute($batch_id);
 564     my $marcflavour = C4::Context->preference('marcflavour');
 565     my $rec_num = 0;
 566     while (my $rowref = $sth->fetchrow_hashref) {
 567         $record_type = $rowref->{'record_type'};
 568         $rec_num++;
 569         if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
 570             &$progress_callback($rec_num);
 571         }
 572         if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
 573             $num_ignored++;
 574             next;
 575         }
 576
 577         my $marc_type;
 578         if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
 579             $marc_type = 'UNIMARCAUTH';
 580         } elsif ($marcflavour eq 'UNIMARC') {
 581             $marc_type = 'UNIMARC';
 582         } else {
 583             $marc_type = 'USMARC';
 584         }
 585         my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
 586
 587         if ($record_type eq 'biblio') {
 588             # remove any item tags - rely on BatchCommitItems
 589             ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
 590             foreach my $item_field ($marc_record->field($item_tag)) {
 591                 $marc_record->delete_field($item_field);
 592             }
 593         }
 594
 595         my ($record_result, $item_result, $record_match) =
 596             _get_commit_action($overlay_action, $nomatch_action, $item_action,
 597                                $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
 598
 599         my $recordid;
 600         my $query;
 601         if ($record_result eq 'create_new') {
 602             $num_added++;
 603             if ($record_type eq 'biblio') {
 604                 my $biblioitemnumber;
 605                 ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework);
 606                 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
 607                 if ($item_result eq 'create_new' || $item_result eq 'replace') {
 608                     my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
 609                     $num_items_added += $bib_items_added;
 610                     $num_items_replaced += $bib_items_replaced;
 611                     $num_items_errored += $bib_items_errored;
 612                 }
 613             } else {
 614                 $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
 615                 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
 616             }
 617             my $sth = $dbh->prepare_cached($query);
 618             $sth->execute($recordid, $rowref->{'import_record_id'});
 619             $sth->finish();
 620             SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
 621         } elsif ($record_result eq 'replace') {
 622             $num_updated++;
 623             $recordid = $record_match;
 624             my $oldxml;
 625             if ($record_type eq 'biblio') {
 626                 my ($count, $oldbiblio) = GetBiblio($recordid);
 627                 $oldxml = GetXmlBiblio($recordid);
 628
 629                 # remove item fields so that they don't get
 630                 # added again if record is reverted
 631                 # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore.
 632                 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type);
 633                 foreach my $item_field ($old_marc->field($item_tag)) {
 634                     $old_marc->delete_field($item_field);
 635                 }
 636                 $oldxml = $old_marc->as_xml($marc_type);
 637
 638                 ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'});
 639                 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
 640
 641                 if ($item_result eq 'create_new' || $item_result eq 'replace') {
 642                     my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
 643                     $num_items_added += $bib_items_added;
 644                     $num_items_replaced += $bib_items_replaced;
 645                     $num_items_errored += $bib_items_errored;
 646                 }
 647             } else {
 648                 $oldxml = GetAuthorityXML($recordid);
 649
 650                 ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
 651                 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
 652             }
 653             my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
 654             $sth->execute($oldxml, $rowref->{'import_record_id'});
 655             $sth->finish();
 656             my $sth2 = $dbh->prepare_cached($query);
 657             $sth2->execute($recordid, $rowref->{'import_record_id'});
 658             $sth2->finish();
 659             SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
 660             SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
 661         } elsif ($record_result eq 'ignore') {
 662             $recordid = $record_match;
 663             $num_ignored++;
 664             $recordid = $record_match;
 665             if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) {
 666                 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
 667                 $num_items_added += $bib_items_added;
 668          $num_items_replaced += $bib_items_replaced;
 669                 $num_items_errored += $bib_items_errored;
 670                 # still need to record the matched biblionumber so that the
 671                 # items can be reverted
 672                 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
 673                 $sth2->execute($recordid, $rowref->{'import_record_id'});
 674                 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
 675             }
 676             SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
 677         }
 678     }
 679     $sth->finish();
 680     SetImportBatchStatus($batch_id, 'imported');
 681     return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored);
 682 }
 683
 684 =head2 BatchCommitItems
 685
 686   ($num_items_added, $num_items_errored) =
 687          BatchCommitItems($import_record_id, $biblionumber);
 688
 689 =cut
 690
 691 sub BatchCommitItems {
 692     my ( $import_record_id, $biblionumber, $action ) = @_;
 693
 694     my $dbh = C4::Context->dbh;
 695
 696     my $num_items_added = 0;
 697     my $num_items_errored = 0;
 698     my $num_items_replaced = 0;
 699
 700     my $sth = $dbh->prepare( "
 701         SELECT import_items_id, import_items.marcxml, encoding
 702         FROM import_items
 703         JOIN import_records USING (import_record_id)
 704         WHERE import_record_id = ?
 705         ORDER BY import_items_id
 706     " );
 707     $sth->bind_param( 1, $import_record_id );
 708     $sth->execute();
 709
 710     while ( my $row = $sth->fetchrow_hashref() ) {
 711         my $item_marc = MARC::Record->new_from_xml( StripNonXmlChars( $row->{'marcxml'} ), 'UTF-8', $row->{'encoding'} );
 712
 713         # Delete date_due subfield as to not accidentally delete item checkout due dates
 714         my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan', GetFrameworkCode($biblionumber) );
 715         $item_marc->field($MARCfield)->delete_subfield( code => $MARCsubfield );
 716
 717         my $item = TransformMarcToKoha( $dbh, $item_marc );
 718
 719         my $duplicate_barcode = exists( $item->{'barcode'} ) && GetItemnumberFromBarcode( $item->{'barcode'} );
 720         my $duplicate_itemnumber = exists( $item->{'itemnumber'} );
 721
 722         my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?");
 723         if ( $action eq "replace" && $duplicate_itemnumber ) {
 724             # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying
 725             ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber} );
 726             $updsth->bind_param( 1, 'imported' );
 727             $updsth->bind_param( 2, $item->{itemnumber} );
 728             $updsth->bind_param( 3, $row->{'import_items_id'} );
 729             $updsth->execute();
 730             $updsth->finish();
 731             $num_items_replaced++;
 732         } elsif ( $action eq "replace" && $duplicate_barcode ) {
 733             my $itemnumber = GetItemnumberFromBarcode( $item->{'barcode'} );
 734             ModItemFromMarc( $item_marc, $biblionumber, $itemnumber );
 735             $updsth->bind_param( 1, 'imported' );
 736             $updsth->bind_param( 2, $item->{itemnumber} );
 737             $updsth->bind_param( 3, $row->{'import_items_id'} );
 738             $updsth->execute();
 739             $updsth->finish();
 740             $num_items_replaced++;
 741         } elsif ($duplicate_barcode) {
 742             $updsth->bind_param( 1, 'error' );
 743             $updsth->bind_param( 2, 'duplicate item barcode' );
 744             $updsth->bind_param( 3, $row->{'import_items_id'} );
 745             $updsth->execute();
 746             $num_items_errored++;
 747         } else {
 748             my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber );
 749             $updsth->bind_param( 1, 'imported' );
 750             $updsth->bind_param( 2, $itemnumber );
 751             $updsth->bind_param( 3, $row->{'import_items_id'} );
 752             $updsth->execute();
 753             $updsth->finish();
 754             $num_items_added++;
 755         }
 756     }
 757
 758     return ( $num_items_added, $num_items_replaced, $num_items_errored );
 759 }
 760
 761 =head2 BatchRevertRecords
 762
 763   my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
 764       $num_ignored) = BatchRevertRecords($batch_id);
 765
 766 =cut
 767
 768 sub BatchRevertRecords {
 769     my $batch_id = shift;
 770
 771     my $record_type;
 772     my $num_deleted = 0;
 773     my $num_errors = 0;
 774     my $num_reverted = 0;
 775     my $num_ignored = 0;
 776     my $num_items_deleted = 0;
 777     # commit (i.e., save, all records in the batch)
 778     SetImportBatchStatus('reverting');
 779     my $overlay_action = GetImportBatchOverlayAction($batch_id);
 780     my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
 781     my $dbh = C4::Context->dbh;
 782     my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
 783                              FROM import_records
 784                              LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
 785                              LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
 786                              WHERE import_batch_id = ?");
 787     $sth->execute($batch_id);
 788     my $marc_type;
 789     my $marcflavour = C4::Context->preference('marcflavour');
 790     while (my $rowref = $sth->fetchrow_hashref) {
 791         $record_type = $rowref->{'record_type'};
 792         if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
 793             $num_ignored++;
 794             next;
 795         }
 796         if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
 797             $marc_type = 'UNIMARCAUTH';
 798         } elsif ($marcflavour eq 'UNIMARC') {
 799             $marc_type = 'UNIMARC';
 800         } else {
 801             $marc_type = 'USMARC';
 802         }
 803
 804         my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
 805
 806         if ($record_result eq 'delete') {
 807             my $error = undef;
 808             if  ($record_type eq 'biblio') {
 809                 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
 810                 $error = DelBiblio($rowref->{'matched_biblionumber'});
 811             } else {
 812                 my $deletedauthid = DelAuthority($rowref->{'matched_authid'});
 813             }
 814             if (defined $error) {
 815                 $num_errors++;
 816             } else {
 817                 $num_deleted++;
 818                 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
 819             }
 820         } elsif ($record_result eq 'restore') {
 821             $num_reverted++;
 822             my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type);
 823             if ($record_type eq 'biblio') {
 824                 my $biblionumber = $rowref->{'matched_biblionumber'};
 825                 my ($count, $oldbiblio) = GetBiblio($biblionumber);
 826                 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
 827                 ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
 828             } else {
 829                 my $authid = $rowref->{'matched_authid'};
 830                 ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
 831             }
 832             SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
 833         } elsif ($record_result eq 'ignore') {
 834             if ($record_type eq 'biblio') {
 835                 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
 836             }
 837             SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
 838         }
 839         my $query;
 840         if ($record_type eq 'biblio') {
 841             # remove matched_biblionumber only if there is no 'imported' item left
 842             $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?";
 843             $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?  AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
 844         } else {
 845             $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
 846         }
 847         my $sth2 = $dbh->prepare_cached($query);
 848         $sth2->execute($rowref->{'import_record_id'});
 849     }
 850
 851     $sth->finish();
 852     SetImportBatchStatus($batch_id, 'reverted');
 853     return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
 854 }
 855
 856 =head2 BatchRevertItems
 857
 858   my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
 859
 860 =cut
 861
 862 sub BatchRevertItems {
 863     my ($import_record_id, $biblionumber) = @_;
 864
 865     my $dbh = C4::Context->dbh;
 866     my $num_items_deleted = 0;
 867
 868     my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
 869                                    FROM import_items
 870                                    JOIN items USING (itemnumber)
 871                                    WHERE import_record_id = ?");
 872     $sth->bind_param(1, $import_record_id);
 873     $sth->execute();
 874     while (my $row = $sth->fetchrow_hashref()) {
 875         my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'});
 876         if ($error == 1){
 877             my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
 878             $updsth->bind_param(1, 'reverted');
 879             $updsth->bind_param(2, $row->{'import_items_id'});
 880             $updsth->execute();
 881             $updsth->finish();
 882             $num_items_deleted++;
 883         }
 884         else {
 885             next;
 886         }
 887     }
 888     $sth->finish();
 889     return $num_items_deleted;
 890 }
 891
 892 =head2 CleanBatch
 893
 894   CleanBatch($batch_id)
 895
 896 Deletes all staged records from the import batch
 897 and sets the status of the batch to 'cleaned'.  Note
 898 that deleting a stage record does *not* affect
 899 any record that has been committed to the database.
 900
 901 =cut
 902
 903 sub CleanBatch {
 904     my $batch_id = shift;
 905     return unless defined $batch_id;
 906
 907     C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
 908     SetImportBatchStatus($batch_id, 'cleaned');
 909 }
 910
 911 =head2 GetAllImportBatches
 912
 913   my $results = GetAllImportBatches();
 914
 915 Returns a references to an array of hash references corresponding
 916 to all import_batches rows (of batch_type 'batch'), sorted in
 917 ascending order by import_batch_id.
 918
 919 =cut
 920
 921 sub  GetAllImportBatches {
 922     my $dbh = C4::Context->dbh;
 923     my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
 924                                     WHERE batch_type IN ('batch', 'webservice')
 925                                     ORDER BY import_batch_id ASC");
 926
 927     my $results = [];
 928     $sth->execute();
 929     while (my $row = $sth->fetchrow_hashref) {
 930         push @$results, $row;
 931     }
 932     $sth->finish();
 933     return $results;
 934 }
 935
 936 =head2 GetStagedWebserviceBatches
 937
 938   my $batch_ids = GetStagedWebserviceBatches();
 939
 940 Returns a references to an array of batch id's
 941 of batch_type 'webservice' that are not imported
 942
 943 =cut
 944
 945 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
 946 SELECT import_batch_id FROM import_batches
 947 WHERE batch_type = 'webservice'
 948 AND import_status = 'staged'
 949 EOQ
 950 sub  GetStagedWebserviceBatches {
 951     my $dbh = C4::Context->dbh;
 952     return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
 953 }
 954
 955 =head2 GetImportBatchRangeDesc
 956
 957   my $results = GetImportBatchRangeDesc($offset, $results_per_group);
 958
 959 Returns a reference to an array of hash references corresponding to
 960 import_batches rows (sorted in descending order by import_batch_id)
 961 start at the given offset.
 962
 963 =cut
 964
 965 sub GetImportBatchRangeDesc {
 966     my ($offset, $results_per_group) = @_;
 967
 968     my $dbh = C4::Context->dbh;
 969     my $query = "SELECT * FROM import_batches
 970                                     WHERE batch_type IN ('batch', 'webservice')
 971                                     ORDER BY import_batch_id DESC";
 972     my @params;
 973     if ($results_per_group){
 974         $query .= " LIMIT ?";
 975         push(@params, $results_per_group);
 976     }
 977     if ($offset){
 978         $query .= " OFFSET ?";
 979         push(@params, $offset);
 980     }
 981     my $sth = $dbh->prepare_cached($query);
 982     $sth->execute(@params);
 983     my $results = $sth->fetchall_arrayref({});
 984     $sth->finish();
 985     return $results;
 986 }
 987
 988 =head2 GetItemNumbersFromImportBatch
 989
 990   my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
 991
 992 =cut
 993
 994 sub GetItemNumbersFromImportBatch {
 995         my ($batch_id) = @_;
 996         my $dbh = C4::Context->dbh;
 997         my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?");
 998         $sth->execute($batch_id);
 999         my @items ;
1000         while ( my ($itm) = $sth->fetchrow_array ) {
1001                 push @items, $itm;
1002         }
1003         return @items;
1004 }
1005
1006 =head2 GetNumberOfImportBatches
1007
1008   my $count = GetNumberOfImportBatches();
1009
1010 =cut
1011
1012 sub GetNumberOfNonZ3950ImportBatches {
1013     my $dbh = C4::Context->dbh;
1014     my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
1015     $sth->execute();
1016     my ($count) = $sth->fetchrow_array();
1017     $sth->finish();
1018     return $count;
1019 }
1020
1021 =head2 GetImportBiblios
1022
1023   my $results = GetImportBiblios($importid);
1024
1025 =cut
1026
1027 sub GetImportBiblios {
1028     my ($import_record_id) = @_;
1029
1030     my $dbh = C4::Context->dbh;
1031     my $query = "SELECT * FROM import_biblios WHERE import_record_id = ?";
1032     return $dbh->selectall_arrayref(
1033         $query,
1034         { Slice => {} },
1035         $import_record_id
1036     );
1037
1038 }
1039
1040 =head2 GetImportRecordsRange
1041
1042   my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group);
1043
1044 Returns a reference to an array of hash references corresponding to
1045 import_biblios/import_auths/import_records rows for a given batch
1046 starting at the given offset.
1047
1048 =cut
1049
1050 sub GetImportRecordsRange {
1051     my ( $batch_id, $offset, $results_per_group, $status, $parameters ) = @_;
1052
1053     my $dbh = C4::Context->dbh;
1054
1055     my $order_by = $parameters->{order_by} || 'import_record_id';
1056     ( $order_by ) = grep( /^$order_by$/, qw( import_record_id title status overlay_status ) ) ? $order_by : 'import_record_id';
1057
1058     my $order_by_direction =
1059       uc( $parameters->{order_by_direction} ) eq 'DESC' ? 'DESC' : 'ASC';
1060
1061     $order_by .= " $order_by_direction, authorized_heading" if $order_by eq 'title';
1062
1063     my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id,
1064                                            record_sequence, status, overlay_status,
1065                                            matched_biblionumber, matched_authid, record_type
1066                                     FROM   import_records
1067                                     LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
1068                                     LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
1069                                     WHERE  import_batch_id = ?";
1070     my @params;
1071     push(@params, $batch_id);
1072     if ($status) {
1073         $query .= " AND status=?";
1074         push(@params,$status);
1075     }
1076
1077     $query.=" ORDER BY $order_by $order_by_direction";
1078
1079     if($results_per_group){
1080         $query .= " LIMIT ?";
1081         push(@params, $results_per_group);
1082     }
1083     if($offset){
1084         $query .= " OFFSET ?";
1085         push(@params, $offset);
1086     }
1087     my $sth = $dbh->prepare_cached($query);
1088     $sth->execute(@params);
1089     my $results = $sth->fetchall_arrayref({});
1090     $sth->finish();
1091     return $results;
1092
1093 }
1094
1095 =head2 GetBestRecordMatch
1096
1097   my $record_id = GetBestRecordMatch($import_record_id);
1098
1099 =cut
1100
1101 sub GetBestRecordMatch {
1102     my ($import_record_id) = @_;
1103
1104     my $dbh = C4::Context->dbh;
1105     my $sth = $dbh->prepare("SELECT candidate_match_id
1106                              FROM   import_record_matches
1107                              JOIN   import_records ON ( import_record_matches.import_record_id = import_records.import_record_id )
1108                              LEFT JOIN biblio ON ( candidate_match_id = biblio.biblionumber )
1109                              LEFT JOIN auth_header ON ( candidate_match_id = auth_header.authid )
1110                              WHERE  import_record_matches.import_record_id = ? AND
1111                              (  (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR
1112                                 (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) )
1113                              ORDER BY score DESC, candidate_match_id DESC");
1114     $sth->execute($import_record_id);
1115     my ($record_id) = $sth->fetchrow_array();
1116     $sth->finish();
1117     return $record_id;
1118 }
1119
1120 =head2 GetImportBatchStatus
1121
1122   my $status = GetImportBatchStatus($batch_id);
1123
1124 =cut
1125
1126 sub GetImportBatchStatus {
1127     my ($batch_id) = @_;
1128
1129     my $dbh = C4::Context->dbh;
1130     my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
1131     $sth->execute($batch_id);
1132     my ($status) = $sth->fetchrow_array();
1133     $sth->finish();
1134     return $status;
1135
1136 }
1137
1138 =head2 SetImportBatchStatus
1139
1140   SetImportBatchStatus($batch_id, $new_status);
1141
1142 =cut
1143
1144 sub SetImportBatchStatus {
1145     my ($batch_id, $new_status) = @_;
1146
1147     my $dbh = C4::Context->dbh;
1148     my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
1149     $sth->execute($new_status, $batch_id);
1150     $sth->finish();
1151
1152 }
1153
1154 =head2 GetImportBatchOverlayAction
1155
1156   my $overlay_action = GetImportBatchOverlayAction($batch_id);
1157
1158 =cut
1159
1160 sub GetImportBatchOverlayAction {
1161     my ($batch_id) = @_;
1162
1163     my $dbh = C4::Context->dbh;
1164     my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
1165     $sth->execute($batch_id);
1166     my ($overlay_action) = $sth->fetchrow_array();
1167     $sth->finish();
1168     return $overlay_action;
1169
1170 }
1171
1172
1173 =head2 SetImportBatchOverlayAction
1174
1175   SetImportBatchOverlayAction($batch_id, $new_overlay_action);
1176
1177 =cut
1178
1179 sub SetImportBatchOverlayAction {
1180     my ($batch_id, $new_overlay_action) = @_;
1181
1182     my $dbh = C4::Context->dbh;
1183     my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
1184     $sth->execute($new_overlay_action, $batch_id);
1185     $sth->finish();
1186
1187 }
1188
1189 =head2 GetImportBatchNoMatchAction
1190
1191   my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1192
1193 =cut
1194
1195 sub GetImportBatchNoMatchAction {
1196     my ($batch_id) = @_;
1197
1198     my $dbh = C4::Context->dbh;
1199     my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1200     $sth->execute($batch_id);
1201     my ($nomatch_action) = $sth->fetchrow_array();
1202     $sth->finish();
1203     return $nomatch_action;
1204
1205 }
1206
1207
1208 =head2 SetImportBatchNoMatchAction
1209
1210   SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1211
1212 =cut
1213
1214 sub SetImportBatchNoMatchAction {
1215     my ($batch_id, $new_nomatch_action) = @_;
1216
1217     my $dbh = C4::Context->dbh;
1218     my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1219     $sth->execute($new_nomatch_action, $batch_id);
1220     $sth->finish();
1221
1222 }
1223
1224 =head2 GetImportBatchItemAction
1225
1226   my $item_action = GetImportBatchItemAction($batch_id);
1227
1228 =cut
1229
1230 sub GetImportBatchItemAction {
1231     my ($batch_id) = @_;
1232
1233     my $dbh = C4::Context->dbh;
1234     my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1235     $sth->execute($batch_id);
1236     my ($item_action) = $sth->fetchrow_array();
1237     $sth->finish();
1238     return $item_action;
1239
1240 }
1241
1242
1243 =head2 SetImportBatchItemAction
1244
1245   SetImportBatchItemAction($batch_id, $new_item_action);
1246
1247 =cut
1248
1249 sub SetImportBatchItemAction {
1250     my ($batch_id, $new_item_action) = @_;
1251
1252     my $dbh = C4::Context->dbh;
1253     my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1254     $sth->execute($new_item_action, $batch_id);
1255     $sth->finish();
1256
1257 }
1258
1259 =head2 GetImportBatchMatcher
1260
1261   my $matcher_id = GetImportBatchMatcher($batch_id);
1262
1263 =cut
1264
1265 sub GetImportBatchMatcher {
1266     my ($batch_id) = @_;
1267
1268     my $dbh = C4::Context->dbh;
1269     my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1270     $sth->execute($batch_id);
1271     my ($matcher_id) = $sth->fetchrow_array();
1272     $sth->finish();
1273     return $matcher_id;
1274
1275 }
1276
1277
1278 =head2 SetImportBatchMatcher
1279
1280   SetImportBatchMatcher($batch_id, $new_matcher_id);
1281
1282 =cut
1283
1284 sub SetImportBatchMatcher {
1285     my ($batch_id, $new_matcher_id) = @_;
1286
1287     my $dbh = C4::Context->dbh;
1288     my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1289     $sth->execute($new_matcher_id, $batch_id);
1290     $sth->finish();
1291
1292 }
1293
1294 =head2 GetImportRecordOverlayStatus
1295
1296   my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1297
1298 =cut
1299
1300 sub GetImportRecordOverlayStatus {
1301     my ($import_record_id) = @_;
1302
1303     my $dbh = C4::Context->dbh;
1304     my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1305     $sth->execute($import_record_id);
1306     my ($overlay_status) = $sth->fetchrow_array();
1307     $sth->finish();
1308     return $overlay_status;
1309
1310 }
1311
1312
1313 =head2 SetImportRecordOverlayStatus
1314
1315   SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1316
1317 =cut
1318
1319 sub SetImportRecordOverlayStatus {
1320     my ($import_record_id, $new_overlay_status) = @_;
1321
1322     my $dbh = C4::Context->dbh;
1323     my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1324     $sth->execute($new_overlay_status, $import_record_id);
1325     $sth->finish();
1326
1327 }
1328
1329 =head2 GetImportRecordStatus
1330
1331   my $overlay_status = GetImportRecordStatus($import_record_id);
1332
1333 =cut
1334
1335 sub GetImportRecordStatus {
1336     my ($import_record_id) = @_;
1337
1338     my $dbh = C4::Context->dbh;
1339     my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1340     $sth->execute($import_record_id);
1341     my ($overlay_status) = $sth->fetchrow_array();
1342     $sth->finish();
1343     return $overlay_status;
1344
1345 }
1346
1347
1348 =head2 SetImportRecordStatus
1349
1350   SetImportRecordStatus($import_record_id, $new_overlay_status);
1351
1352 =cut
1353
1354 sub SetImportRecordStatus {
1355     my ($import_record_id, $new_overlay_status) = @_;
1356
1357     my $dbh = C4::Context->dbh;
1358     my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1359     $sth->execute($new_overlay_status, $import_record_id);
1360     $sth->finish();
1361
1362 }
1363
1364 =head2 GetImportRecordMatches
1365
1366   my $results = GetImportRecordMatches($import_record_id, $best_only);
1367
1368 =cut
1369
1370 sub GetImportRecordMatches {
1371     my $import_record_id = shift;
1372     my $best_only = @_ ? shift : 0;
1373
1374     my $dbh = C4::Context->dbh;
1375     # FIXME currently biblio only
1376     my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber,
1377                                     candidate_match_id, score, record_type
1378                                     FROM import_records
1379                                     JOIN import_record_matches USING (import_record_id)
1380                                     LEFT JOIN biblio ON (biblionumber = candidate_match_id)
1381                                     WHERE import_record_id = ?
1382                                     ORDER BY score DESC, biblionumber DESC");
1383     $sth->bind_param(1, $import_record_id);
1384     my $results = [];
1385     $sth->execute();
1386     while (my $row = $sth->fetchrow_hashref) {
1387         if ($row->{'record_type'} eq 'auth') {
1388             $row->{'authorized_heading'} = C4::AuthoritiesMarc::GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } );
1389         }
1390         next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'});
1391         push @$results, $row;
1392         last if $best_only;
1393     }
1394     $sth->finish();
1395
1396     return $results;
1397
1398 }
1399
1400
1401 =head2 SetImportRecordMatches
1402
1403   SetImportRecordMatches($import_record_id, @matches);
1404
1405 =cut
1406
1407 sub SetImportRecordMatches {
1408     my $import_record_id = shift;
1409     my @matches = @_;
1410
1411     my $dbh = C4::Context->dbh;
1412     my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1413     $delsth->execute($import_record_id);
1414     $delsth->finish();
1415
1416     my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score)
1417                                     VALUES (?, ?, ?)");
1418     foreach my $match (@matches) {
1419         $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'});
1420     }
1421 }
1422
1423
1424 # internal functions
1425
1426 sub _create_import_record {
1427     my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_;
1428
1429     my $dbh = C4::Context->dbh;
1430     my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml,
1431                                                          record_type, encoding, z3950random)
1432                                     VALUES (?, ?, ?, ?, ?, ?, ?)");
1433     $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type),
1434                   $record_type, $encoding, $z3950random);
1435     my $import_record_id = $dbh->{'mysql_insertid'};
1436     $sth->finish();
1437     return $import_record_id;
1438 }
1439
1440 sub _update_import_record_marc {
1441     my ($import_record_id, $marc_record, $marc_type) = @_;
1442
1443     my $dbh = C4::Context->dbh;
1444     my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ?
1445                              WHERE  import_record_id = ?");
1446     $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id);
1447     $sth->finish();
1448 }
1449
1450 sub _add_auth_fields {
1451     my ($import_record_id, $marc_record) = @_;
1452
1453     my $controlnumber;
1454     if ($marc_record->field('001')) {
1455         $controlnumber = $marc_record->field('001')->data();
1456     }
1457     my $authorized_heading = C4::AuthoritiesMarc::GetAuthorizedHeading({ record => $marc_record });
1458     my $dbh = C4::Context->dbh;
1459     my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)");
1460     $sth->execute($import_record_id, $controlnumber, $authorized_heading);
1461     $sth->finish();
1462 }
1463
1464 sub _add_biblio_fields {
1465     my ($import_record_id, $marc_record) = @_;
1466
1467     my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1468     my $dbh = C4::Context->dbh;
1469     # FIXME no controlnumber, originalsource
1470     $isbn = C4::Koha::GetNormalizedISBN($isbn);
1471     my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1472     $sth->execute($import_record_id, $title, $author, $isbn, $issn);
1473     $sth->finish();
1474
1475 }
1476
1477 sub _update_biblio_fields {
1478     my ($import_record_id, $marc_record) = @_;
1479
1480     my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1481     my $dbh = C4::Context->dbh;
1482     # FIXME no controlnumber, originalsource
1483     # FIXME 2 - should regularize normalization of ISBN wherever it is done
1484     $isbn =~ s/\(.*$//;
1485     $isbn =~ tr/ -_//;
1486     $isbn = uc $isbn;
1487     my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1488                              WHERE  import_record_id = ?");
1489     $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1490     $sth->finish();
1491 }
1492
1493 sub _parse_biblio_fields {
1494     my ($marc_record) = @_;
1495
1496     my $dbh = C4::Context->dbh;
1497     my $bibliofields = TransformMarcToKoha($dbh, $marc_record, '');
1498     return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1499
1500 }
1501
1502 sub _update_batch_record_counts {
1503     my ($batch_id) = @_;
1504
1505     my $dbh = C4::Context->dbh;
1506     my $sth = $dbh->prepare_cached("UPDATE import_batches SET
1507                                         num_records = (
1508                                             SELECT COUNT(*)
1509                                             FROM import_records
1510                                             WHERE import_batch_id = import_batches.import_batch_id),
1511                                         num_items = (
1512                                             SELECT COUNT(*)
1513                                             FROM import_records
1514                                             JOIN import_items USING (import_record_id)
1515                                             WHERE import_batch_id = import_batches.import_batch_id
1516                                             AND record_type = 'biblio')
1517                                     WHERE import_batch_id = ?");
1518     $sth->bind_param(1, $batch_id);
1519     $sth->execute();
1520     $sth->finish();
1521 }
1522
1523 sub _get_commit_action {
1524     my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1525
1526     if ($record_type eq 'biblio') {
1527         my ($bib_result, $bib_match, $item_result);
1528
1529         if ($overlay_status ne 'no_match') {
1530             $bib_match = GetBestRecordMatch($import_record_id);
1531             if ($overlay_action eq 'replace') {
1532                 $bib_result  = defined($bib_match) ? 'replace' : 'create_new';
1533             } elsif ($overlay_action eq 'create_new') {
1534                 $bib_result  = 'create_new';
1535             } elsif ($overlay_action eq 'ignore') {
1536                 $bib_result  = 'ignore';
1537             }
1538          if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){
1539                 $item_result = 'create_new';
1540        }
1541       elsif($item_action eq 'replace'){
1542           $item_result = 'replace';
1543           }
1544       else {
1545              $item_result = 'ignore';
1546            }
1547         } else {
1548             $bib_result = $nomatch_action;
1549             $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new')     ? 'create_new' : 'ignore';
1550         }
1551         return ($bib_result, $item_result, $bib_match);
1552     } else { # must be auths
1553         my ($auth_result, $auth_match);
1554
1555         if ($overlay_status ne 'no_match') {
1556             $auth_match = GetBestRecordMatch($import_record_id);
1557             if ($overlay_action eq 'replace') {
1558                 $auth_result  = defined($auth_match) ? 'replace' : 'create_new';
1559             } elsif ($overlay_action eq 'create_new') {
1560                 $auth_result  = 'create_new';
1561             } elsif ($overlay_action eq 'ignore') {
1562                 $auth_result  = 'ignore';
1563             }
1564         } else {
1565             $auth_result = $nomatch_action;
1566         }
1567
1568         return ($auth_result, undef, $auth_match);
1569
1570     }
1571 }
1572
1573 sub _get_revert_action {
1574     my ($overlay_action, $overlay_status, $status) = @_;
1575
1576     my $bib_result;
1577
1578     if ($status eq 'ignored') {
1579         $bib_result = 'ignore';
1580     } else {
1581         if ($overlay_action eq 'create_new') {
1582             $bib_result = 'delete';
1583         } else {
1584             $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1585         }
1586     }
1587     return $bib_result;
1588 }
1589
1590 1;
1591 __END__
1592
1593 =head1 AUTHOR
1594
1595 Koha Development Team <http://koha-community.org/>
1596
1597 Galen Charlton <galen.charlton@liblime.com>
1598
1599 =cut