Merge remote-tracking branch 'origin/new/bug_8204'
[koha.git] / C4 / ImportBatch.pm
bloba64a0d27813e905896d33eda4155ad4a0d5fab14
1 package C4::ImportBatch;
3 # Copyright (C) 2007 LibLime
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 2 of the License, or (at your option) any later
10 # version.
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use strict;
21 use warnings;
23 use C4::Context;
24 use C4::Koha;
25 use C4::Biblio;
26 use C4::Items;
27 use C4::Charset;
29 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
31 BEGIN {
32 # set the version for version checking
33 $VERSION = 3.07.00.049;
34 require Exporter;
35 @ISA = qw(Exporter);
36 @EXPORT = qw(
37 GetZ3950BatchId
38 GetWebserviceBatchId
39 GetImportRecordMarc
40 GetImportRecordMarcXML
41 AddImportBatch
42 GetImportBatch
43 AddBiblioToBatch
44 AddItemsToImportBiblio
45 ModBiblioInBatch
47 BatchStageMarcRecords
48 BatchFindBibDuplicates
49 BatchCommitBibRecords
50 BatchRevertBibRecords
51 CleanBatch
53 GetAllImportBatches
54 GetStagedWebserviceBatches
55 GetImportBatchRangeDesc
56 GetNumberOfNonZ3950ImportBatches
57 GetImportBibliosRange
58 GetItemNumbersFromImportBatch
60 GetImportBatchStatus
61 SetImportBatchStatus
62 GetImportBatchOverlayAction
63 SetImportBatchOverlayAction
64 GetImportBatchNoMatchAction
65 SetImportBatchNoMatchAction
66 GetImportBatchItemAction
67 SetImportBatchItemAction
68 GetImportBatchMatcher
69 SetImportBatchMatcher
70 GetImportRecordOverlayStatus
71 SetImportRecordOverlayStatus
72 GetImportRecordStatus
73 SetImportRecordStatus
74 GetImportRecordMatches
75 SetImportRecordMatches
79 =head1 NAME
81 C4::ImportBatch - manage batches of imported MARC records
83 =head1 SYNOPSIS
85 use C4::ImportBatch;
87 =head1 FUNCTIONS
89 =head2 GetZ3950BatchId
91 my $batchid = GetZ3950BatchId($z3950server);
93 Retrieves the ID of the import batch for the Z39.50
94 reservoir for the given target. If necessary,
95 creates the import batch.
97 =cut
99 sub GetZ3950BatchId {
100 my ($z3950server) = @_;
102 my $dbh = C4::Context->dbh;
103 my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
104 WHERE batch_type = 'z3950'
105 AND file_name = ?");
106 $sth->execute($z3950server);
107 my $rowref = $sth->fetchrow_arrayref();
108 $sth->finish();
109 if (defined $rowref) {
110 return $rowref->[0];
111 } else {
112 my $batch_id = AddImportBatch( {
113 overlay_action => 'create_new',
114 import_status => 'staged',
115 batch_type => 'z3950',
116 file_name => $z3950server,
117 } );
118 return $batch_id;
123 =head2 GetWebserviceBatchId
125 my $batchid = GetWebserviceBatchId();
127 Retrieves the ID of the import batch for webservice.
128 If necessary, creates the import batch.
130 =cut
132 my $WEBSERVICE_BASE_QRY = <<EOQ;
133 SELECT import_batch_id FROM import_batches
134 WHERE batch_type = 'webservice'
135 AND import_status = 'staged'
137 sub GetWebserviceBatchId {
138 my ($params) = @_;
140 my $dbh = C4::Context->dbh;
141 my $sql = $WEBSERVICE_BASE_QRY;
142 my @args;
143 foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
144 if (my $val = $params->{$field}) {
145 $sql .= " AND $field = ?";
146 push @args, $val;
149 my $id = $dbh->selectrow_array($sql, undef, @args);
150 return $id if $id;
152 $params->{batch_type} = 'webservice';
153 $params->{import_status} = 'staged';
154 return AddImportBatch($params);
157 =head2 GetImportRecordMarc
159 my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
161 =cut
163 sub GetImportRecordMarc {
164 my ($import_record_id) = @_;
166 my $dbh = C4::Context->dbh;
167 my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?");
168 $sth->execute($import_record_id);
169 my ($marc, $encoding) = $sth->fetchrow();
170 $sth->finish();
171 return $marc, $encoding;
175 =head2 GetImportRecordMarcXML
177 my $marcxml = GetImportRecordMarcXML($import_record_id);
179 =cut
181 sub GetImportRecordMarcXML {
182 my ($import_record_id) = @_;
184 my $dbh = C4::Context->dbh;
185 my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?");
186 $sth->execute($import_record_id);
187 my ($marcxml) = $sth->fetchrow();
188 $sth->finish();
189 return $marcxml;
193 =head2 AddImportBatch
195 my $batch_id = AddImportBatch($params_hash);
197 =cut
199 sub AddImportBatch {
200 my ($params) = @_;
202 my (@fields, @vals);
203 foreach (qw( matcher_id template_id branchcode
204 overlay_action nomatch_action item_action
205 import_status batch_type file_name comments )) {
206 if (exists $params->{$_}) {
207 push @fields, $_;
208 push @vals, $params->{$_};
211 my $dbh = C4::Context->dbh;
212 $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
213 VALUES (".join( ',', map '?', @fields).")",
214 undef,
215 @vals);
216 return $dbh->{'mysql_insertid'};
219 =head2 GetImportBatch
221 my $row = GetImportBatch($batch_id);
223 Retrieve a hashref of an import_batches row.
225 =cut
227 sub GetImportBatch {
228 my ($batch_id) = @_;
230 my $dbh = C4::Context->dbh;
231 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches WHERE import_batch_id = ?");
232 $sth->bind_param(1, $batch_id);
233 $sth->execute();
234 my $result = $sth->fetchrow_hashref;
235 $sth->finish();
236 return $result;
240 =head2 AddBiblioToBatch
242 my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
243 $marc_record, $encoding, $z3950random, $update_counts);
245 =cut
247 sub AddBiblioToBatch {
248 my $batch_id = shift;
249 my $record_sequence = shift;
250 my $marc_record = shift;
251 my $encoding = shift;
252 my $z3950random = shift;
253 my $update_counts = @_ ? shift : 1;
255 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random);
256 _add_biblio_fields($import_record_id, $marc_record);
257 _update_batch_record_counts($batch_id) if $update_counts;
258 return $import_record_id;
261 =head2 ModBiblioInBatch
263 ModBiblioInBatch($import_record_id, $marc_record);
265 =cut
267 sub ModBiblioInBatch {
268 my ($import_record_id, $marc_record) = @_;
270 _update_import_record_marc($import_record_id, $marc_record);
271 _update_biblio_fields($import_record_id, $marc_record);
275 =head2 BatchStageMarcRecords
277 ($batch_id, $num_records, $num_items, @invalid_records) =
278 BatchStageMarcRecords($encoding, $marc_records, $file_name,
279 $comments, $branch_code, $parse_items,
280 $leave_as_staging,
281 $progress_interval, $progress_callback);
283 =cut
285 sub BatchStageMarcRecords {
286 my $encoding = shift;
287 my $marc_records = shift;
288 my $file_name = shift;
289 my $comments = shift;
290 my $branch_code = shift;
291 my $parse_items = shift;
292 my $leave_as_staging = shift;
294 # optional callback to monitor status
295 # of job
296 my $progress_interval = 0;
297 my $progress_callback = undef;
298 if ($#_ == 1) {
299 $progress_interval = shift;
300 $progress_callback = shift;
301 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
302 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
305 my $batch_id = AddImportBatch( {
306 overlay_action => 'create_new',
307 import_status => 'staging',
308 batch_type => 'batch',
309 file_name => $file_name,
310 comments => $comments,
311 } );
312 if ($parse_items) {
313 SetImportBatchItemAction($batch_id, 'always_add');
314 } else {
315 SetImportBatchItemAction($batch_id, 'ignore');
318 my @invalid_records = ();
319 my $num_valid = 0;
320 my $num_items = 0;
321 # FIXME - for now, we're dealing only with bibs
322 my $rec_num = 0;
323 foreach my $marc_blob (split(/\x1D/, $marc_records)) {
324 $marc_blob =~ s/^\s+//g;
325 $marc_blob =~ s/\s+$//g;
326 next unless $marc_blob;
327 $rec_num++;
328 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
329 &$progress_callback($rec_num);
331 my ($marc_record, $charset_guessed, $char_errors) =
332 MarcToUTF8Record($marc_blob, C4::Context->preference("marcflavour"), $encoding);
334 $encoding = $charset_guessed unless $encoding;
336 my $import_record_id;
337 if (scalar($marc_record->fields()) == 0) {
338 push @invalid_records, $marc_blob;
339 } else {
340 $num_valid++;
341 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
342 if ($parse_items) {
343 my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
344 $num_items += scalar(@import_items_ids);
348 unless ($leave_as_staging) {
349 SetImportBatchStatus($batch_id, 'staged');
351 # FIXME branch_code, number of bibs, number of items
352 _update_batch_record_counts($batch_id);
353 return ($batch_id, $num_valid, $num_items, @invalid_records);
356 =head2 AddItemsToImportBiblio
358 my @import_items_ids = AddItemsToImportBiblio($batch_id,
359 $import_record_id, $marc_record, $update_counts);
361 =cut
363 sub AddItemsToImportBiblio {
364 my $batch_id = shift;
365 my $import_record_id = shift;
366 my $marc_record = shift;
367 my $update_counts = @_ ? shift : 0;
369 my @import_items_ids = ();
371 my $dbh = C4::Context->dbh;
372 my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
373 foreach my $item_field ($marc_record->field($item_tag)) {
374 my $item_marc = MARC::Record->new();
375 $item_marc->leader("00000 a "); # must set Leader/09 to 'a'
376 $item_marc->append_fields($item_field);
377 $marc_record->delete_field($item_field);
378 my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
379 VALUES (?, ?, ?)");
380 $sth->bind_param(1, $import_record_id);
381 $sth->bind_param(2, 'staged');
382 $sth->bind_param(3, $item_marc->as_xml());
383 $sth->execute();
384 push @import_items_ids, $dbh->{'mysql_insertid'};
385 $sth->finish();
388 if ($#import_items_ids > -1) {
389 _update_batch_record_counts($batch_id) if $update_counts;
390 _update_import_record_marc($import_record_id, $marc_record);
392 return @import_items_ids;
395 =head2 BatchFindBibDuplicates
397 my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher,
398 $max_matches, $progress_interval, $progress_callback);
400 Goes through the records loaded in the batch and attempts to
401 find duplicates for each one. Sets the matching status
402 of each record to "no_match" or "auto_match" as appropriate.
404 The $max_matches parameter is optional; if it is not supplied,
405 it defaults to 10.
407 The $progress_interval and $progress_callback parameters are
408 optional; if both are supplied, the sub referred to by
409 $progress_callback will be invoked every $progress_interval
410 records using the number of records processed as the
411 singular argument.
413 =cut
415 sub BatchFindBibDuplicates {
416 my $batch_id = shift;
417 my $matcher = shift;
418 my $max_matches = @_ ? shift : 10;
420 # optional callback to monitor status
421 # of job
422 my $progress_interval = 0;
423 my $progress_callback = undef;
424 if ($#_ == 1) {
425 $progress_interval = shift;
426 $progress_callback = shift;
427 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
428 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
431 my $dbh = C4::Context->dbh;
433 my $sth = $dbh->prepare("SELECT import_record_id, marc
434 FROM import_records
435 JOIN import_biblios USING (import_record_id)
436 WHERE import_batch_id = ?");
437 $sth->execute($batch_id);
438 my $num_with_matches = 0;
439 my $rec_num = 0;
440 while (my $rowref = $sth->fetchrow_hashref) {
441 $rec_num++;
442 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
443 &$progress_callback($rec_num);
445 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
446 my @matches = ();
447 if (defined $matcher) {
448 @matches = $matcher->get_matches($marc_record, $max_matches);
450 if (scalar(@matches) > 0) {
451 $num_with_matches++;
452 SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
453 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
454 } else {
455 SetImportRecordMatches($rowref->{'import_record_id'}, ());
456 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
459 $sth->finish();
460 return $num_with_matches;
463 =head2 BatchCommitBibRecords
465 my ($num_added, $num_updated, $num_items_added, $num_items_errored,
466 $num_ignored) = BatchCommitBibRecords($batch_id, $framework,
467 $progress_interval, $progress_callback);
469 =cut
471 sub BatchCommitBibRecords {
472 my $batch_id = shift;
473 my $framework = shift;
475 # optional callback to monitor status
476 # of job
477 my $progress_interval = 0;
478 my $progress_callback = undef;
479 if ($#_ == 1) {
480 $progress_interval = shift;
481 $progress_callback = shift;
482 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
483 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
486 my $num_added = 0;
487 my $num_updated = 0;
488 my $num_items_added = 0;
489 my $num_items_errored = 0;
490 my $num_ignored = 0;
491 # commit (i.e., save, all records in the batch)
492 # FIXME biblio only at the moment
493 SetImportBatchStatus('importing');
494 my $overlay_action = GetImportBatchOverlayAction($batch_id);
495 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
496 my $item_action = GetImportBatchItemAction($batch_id);
497 my $dbh = C4::Context->dbh;
498 my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding
499 FROM import_records
500 JOIN import_biblios USING (import_record_id)
501 WHERE import_batch_id = ?");
502 $sth->execute($batch_id);
503 my $rec_num = 0;
504 while (my $rowref = $sth->fetchrow_hashref) {
505 $rec_num++;
506 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
507 &$progress_callback($rec_num);
509 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
510 $num_ignored++;
511 next;
514 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
516 # remove any item tags - rely on BatchCommitItems
517 my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
518 foreach my $item_field ($marc_record->field($item_tag)) {
519 $marc_record->delete_field($item_field);
522 # decide what what to do with the bib and item records
523 my ($bib_result, $item_result, $bib_match) =
524 _get_commit_action($overlay_action, $nomatch_action, $item_action,
525 $rowref->{'overlay_status'}, $rowref->{'import_record_id'});
527 if ($bib_result eq 'create_new') {
528 $num_added++;
529 my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, $framework);
530 my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
531 $sth->execute($biblionumber, $rowref->{'import_record_id'});
532 $sth->finish();
533 if ($item_result eq 'create_new') {
534 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
535 $num_items_added += $bib_items_added;
536 $num_items_errored += $bib_items_errored;
538 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
539 } elsif ($bib_result eq 'replace') {
540 $num_updated++;
541 my $biblionumber = $bib_match;
542 my ($count, $oldbiblio) = GetBiblio($biblionumber);
543 my $oldxml = GetXmlBiblio($biblionumber);
545 # remove item fields so that they don't get
546 # added again if record is reverted
547 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'});
548 foreach my $item_field ($old_marc->field($item_tag)) {
549 $old_marc->delete_field($item_field);
552 ModBiblio($marc_record, $biblionumber, $oldbiblio->{'frameworkcode'});
553 my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
554 $sth->execute($old_marc->as_xml(), $rowref->{'import_record_id'});
555 $sth->finish();
556 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
557 $sth2->execute($biblionumber, $rowref->{'import_record_id'});
558 $sth2->finish();
559 if ($item_result eq 'create_new') {
560 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
561 $num_items_added += $bib_items_added;
562 $num_items_errored += $bib_items_errored;
564 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
565 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
566 } elsif ($bib_result eq 'ignore') {
567 $num_ignored++;
568 my $biblionumber = $bib_match;
569 if (defined $biblionumber and $item_result eq 'create_new') {
570 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
571 $num_items_added += $bib_items_added;
572 $num_items_errored += $bib_items_errored;
573 # still need to record the matched biblionumber so that the
574 # items can be reverted
575 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
576 $sth2->execute($biblionumber, $rowref->{'import_record_id'});
577 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
579 SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
582 $sth->finish();
583 SetImportBatchStatus($batch_id, 'imported');
584 return ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored);
587 =head2 BatchCommitItems
589 ($num_items_added, $num_items_errored) =
590 BatchCommitItems($import_record_id, $biblionumber);
592 =cut
594 sub BatchCommitItems {
595 my ($import_record_id, $biblionumber) = @_;
597 my $dbh = C4::Context->dbh;
599 my $num_items_added = 0;
600 my $num_items_errored = 0;
601 my $sth = $dbh->prepare("SELECT import_items_id, import_items.marcxml, encoding
602 FROM import_items
603 JOIN import_records USING (import_record_id)
604 WHERE import_record_id = ?
605 ORDER BY import_items_id");
606 $sth->bind_param(1, $import_record_id);
607 $sth->execute();
608 while (my $row = $sth->fetchrow_hashref()) {
609 my $item_marc = MARC::Record->new_from_xml(StripNonXmlChars($row->{'marcxml'}), 'UTF-8', $row->{'encoding'});
610 # FIXME - duplicate barcode check needs to become part of AddItemFromMarc()
611 my $item = TransformMarcToKoha($dbh, $item_marc);
612 my $duplicate_barcode = exists($item->{'barcode'}) && GetItemnumberFromBarcode($item->{'barcode'});
613 if ($duplicate_barcode) {
614 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, import_error = ? WHERE import_items_id = ?");
615 $updsth->bind_param(1, 'error');
616 $updsth->bind_param(2, 'duplicate item barcode');
617 $updsth->bind_param(3, $row->{'import_items_id'});
618 $updsth->execute();
619 $num_items_errored++;
620 } else {
621 my ($item_biblionumber, $biblioitemnumber, $itemnumber) = AddItemFromMarc($item_marc, $biblionumber);
622 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?");
623 $updsth->bind_param(1, 'imported');
624 $updsth->bind_param(2, $itemnumber);
625 $updsth->bind_param(3, $row->{'import_items_id'});
626 $updsth->execute();
627 $updsth->finish();
628 $num_items_added++;
631 $sth->finish();
632 return ($num_items_added, $num_items_errored);
635 =head2 BatchRevertBibRecords
637 my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
638 $num_ignored) = BatchRevertBibRecords($batch_id);
640 =cut
642 sub BatchRevertBibRecords {
643 my $batch_id = shift;
645 my $num_deleted = 0;
646 my $num_errors = 0;
647 my $num_reverted = 0;
648 my $num_items_deleted = 0;
649 my $num_ignored = 0;
650 # commit (i.e., save, all records in the batch)
651 # FIXME biblio only at the moment
652 SetImportBatchStatus('reverting');
653 my $overlay_action = GetImportBatchOverlayAction($batch_id);
654 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
655 my $dbh = C4::Context->dbh;
656 my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber
657 FROM import_records
658 JOIN import_biblios USING (import_record_id)
659 WHERE import_batch_id = ?");
660 $sth->execute($batch_id);
661 while (my $rowref = $sth->fetchrow_hashref) {
662 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
663 $num_ignored++;
664 next;
667 my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
669 if ($bib_result eq 'delete') {
670 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
671 my $error = DelBiblio($rowref->{'matched_biblionumber'});
672 if (defined $error) {
673 $num_errors++;
674 } else {
675 $num_deleted++;
676 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
678 } elsif ($bib_result eq 'restore') {
679 $num_reverted++;
680 my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'});
681 my $biblionumber = $rowref->{'matched_biblionumber'};
682 my ($count, $oldbiblio) = GetBiblio($biblionumber);
683 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
684 ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
685 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
686 } elsif ($bib_result eq 'ignore') {
687 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
688 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
690 # remove matched_biblionumber only if there is no 'imported' item left
691 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')" );
692 $sth2->execute($rowref->{'import_record_id'});
695 $sth->finish();
696 SetImportBatchStatus($batch_id, 'reverted');
697 return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
700 =head2 BatchRevertItems
702 my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
704 =cut
706 sub BatchRevertItems {
707 my ($import_record_id, $biblionumber) = @_;
709 my $dbh = C4::Context->dbh;
710 my $num_items_deleted = 0;
712 my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
713 FROM import_items
714 JOIN items USING (itemnumber)
715 WHERE import_record_id = ?");
716 $sth->bind_param(1, $import_record_id);
717 $sth->execute();
718 while (my $row = $sth->fetchrow_hashref()) {
719 my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'});
720 if ($error == 1){
721 my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
722 $updsth->bind_param(1, 'reverted');
723 $updsth->bind_param(2, $row->{'import_items_id'});
724 $updsth->execute();
725 $updsth->finish();
726 $num_items_deleted++;
728 else {
729 next;
732 $sth->finish();
733 return $num_items_deleted;
736 =head2 CleanBatch
738 CleanBatch($batch_id)
740 Deletes all staged records from the import batch
741 and sets the status of the batch to 'cleaned'. Note
742 that deleting a stage record does *not* affect
743 any record that has been committed to the database.
745 =cut
747 sub CleanBatch {
748 my $batch_id = shift;
749 return unless defined $batch_id;
751 C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
752 SetImportBatchStatus($batch_id, 'cleaned');
755 =head2 GetAllImportBatches
757 my $results = GetAllImportBatches();
759 Returns a references to an array of hash references corresponding
760 to all import_batches rows (of batch_type 'batch'), sorted in
761 ascending order by import_batch_id.
763 =cut
765 sub GetAllImportBatches {
766 my $dbh = C4::Context->dbh;
767 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
768 WHERE batch_type IN ('batch', 'webservice')
769 ORDER BY import_batch_id ASC");
771 my $results = [];
772 $sth->execute();
773 while (my $row = $sth->fetchrow_hashref) {
774 push @$results, $row;
776 $sth->finish();
777 return $results;
780 =head2 GetStagedWebserviceBatches
782 my $batch_ids = GetStagedWebserviceBatches();
784 Returns a references to an array of batch id's
785 of batch_type 'webservice' that are not imported
787 =cut
789 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
790 SELECT import_batch_id FROM import_batches
791 WHERE batch_type = 'webservice'
792 AND import_status = 'staged'
794 sub GetStagedWebserviceBatches {
795 my $dbh = C4::Context->dbh;
796 return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
799 =head2 GetImportBatchRangeDesc
801 my $results = GetImportBatchRangeDesc($offset, $results_per_group);
803 Returns a reference to an array of hash references corresponding to
804 import_batches rows (sorted in descending order by import_batch_id)
805 start at the given offset.
807 =cut
809 sub GetImportBatchRangeDesc {
810 my ($offset, $results_per_group) = @_;
812 my $dbh = C4::Context->dbh;
813 my $query = "SELECT * FROM import_batches
814 WHERE batch_type IN ('batch', 'webservice')
815 ORDER BY import_batch_id DESC";
816 my @params;
817 if ($results_per_group){
818 $query .= " LIMIT ?";
819 push(@params, $results_per_group);
821 if ($offset){
822 $query .= " OFFSET ?";
823 push(@params, $offset);
825 my $sth = $dbh->prepare_cached($query);
826 $sth->execute(@params);
827 my $results = $sth->fetchall_arrayref({});
828 $sth->finish();
829 return $results;
832 =head2 GetItemNumbersFromImportBatch
834 my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
836 =cut
838 sub GetItemNumbersFromImportBatch {
839 my ($batch_id) = @_;
840 my $dbh = C4::Context->dbh;
841 my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?");
842 $sth->execute($batch_id);
843 my @items ;
844 while ( my ($itm) = $sth->fetchrow_array ) {
845 push @items, $itm;
847 return @items;
850 =head2 GetNumberOfImportBatches
852 my $count = GetNumberOfImportBatches();
854 =cut
856 sub GetNumberOfNonZ3950ImportBatches {
857 my $dbh = C4::Context->dbh;
858 my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
859 $sth->execute();
860 my ($count) = $sth->fetchrow_array();
861 $sth->finish();
862 return $count;
865 =head2 GetImportBibliosRange
867 my $results = GetImportBibliosRange($batch_id, $offset, $results_per_group);
869 Returns a reference to an array of hash references corresponding to
870 import_biblios/import_records rows for a given batch
871 starting at the given offset.
873 =cut
875 sub GetImportBibliosRange {
876 my ($batch_id, $offset, $results_per_group, $status) = @_;
878 my $dbh = C4::Context->dbh;
879 my $query = "SELECT title, author, isbn, issn, import_record_id, record_sequence,
880 status, overlay_status, matched_biblionumber
881 FROM import_records
882 JOIN import_biblios USING (import_record_id)
883 WHERE import_batch_id = ?";
884 my @params;
885 push(@params, $batch_id);
886 if ($status) {
887 $query .= " AND status=?";
888 push(@params,$status);
890 $query.=" ORDER BY import_record_id";
892 if($results_per_group){
893 $query .= " LIMIT ?";
894 push(@params, $results_per_group);
896 if($offset){
897 $query .= " OFFSET ?";
898 push(@params, $offset);
900 my $sth = $dbh->prepare_cached($query);
901 $sth->execute(@params);
902 my $results = $sth->fetchall_arrayref({});
903 $sth->finish();
904 return $results;
908 =head2 GetBestRecordMatch
910 my $record_id = GetBestRecordMatch($import_record_id);
912 =cut
914 sub GetBestRecordMatch {
915 my ($import_record_id) = @_;
917 my $dbh = C4::Context->dbh;
918 my $sth = $dbh->prepare("SELECT candidate_match_id
919 FROM import_record_matches
920 WHERE import_record_id = ?
921 ORDER BY score DESC, candidate_match_id DESC");
922 $sth->execute($import_record_id);
923 my ($record_id) = $sth->fetchrow_array();
924 $sth->finish();
925 return $record_id;
928 =head2 GetImportBatchStatus
930 my $status = GetImportBatchStatus($batch_id);
932 =cut
934 sub GetImportBatchStatus {
935 my ($batch_id) = @_;
937 my $dbh = C4::Context->dbh;
938 my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
939 $sth->execute($batch_id);
940 my ($status) = $sth->fetchrow_array();
941 $sth->finish();
942 return $status;
946 =head2 SetImportBatchStatus
948 SetImportBatchStatus($batch_id, $new_status);
950 =cut
952 sub SetImportBatchStatus {
953 my ($batch_id, $new_status) = @_;
955 my $dbh = C4::Context->dbh;
956 my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
957 $sth->execute($new_status, $batch_id);
958 $sth->finish();
962 =head2 GetImportBatchOverlayAction
964 my $overlay_action = GetImportBatchOverlayAction($batch_id);
966 =cut
968 sub GetImportBatchOverlayAction {
969 my ($batch_id) = @_;
971 my $dbh = C4::Context->dbh;
972 my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
973 $sth->execute($batch_id);
974 my ($overlay_action) = $sth->fetchrow_array();
975 $sth->finish();
976 return $overlay_action;
981 =head2 SetImportBatchOverlayAction
983 SetImportBatchOverlayAction($batch_id, $new_overlay_action);
985 =cut
987 sub SetImportBatchOverlayAction {
988 my ($batch_id, $new_overlay_action) = @_;
990 my $dbh = C4::Context->dbh;
991 my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
992 $sth->execute($new_overlay_action, $batch_id);
993 $sth->finish();
997 =head2 GetImportBatchNoMatchAction
999 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1001 =cut
1003 sub GetImportBatchNoMatchAction {
1004 my ($batch_id) = @_;
1006 my $dbh = C4::Context->dbh;
1007 my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1008 $sth->execute($batch_id);
1009 my ($nomatch_action) = $sth->fetchrow_array();
1010 $sth->finish();
1011 return $nomatch_action;
1016 =head2 SetImportBatchNoMatchAction
1018 SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1020 =cut
1022 sub SetImportBatchNoMatchAction {
1023 my ($batch_id, $new_nomatch_action) = @_;
1025 my $dbh = C4::Context->dbh;
1026 my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1027 $sth->execute($new_nomatch_action, $batch_id);
1028 $sth->finish();
1032 =head2 GetImportBatchItemAction
1034 my $item_action = GetImportBatchItemAction($batch_id);
1036 =cut
1038 sub GetImportBatchItemAction {
1039 my ($batch_id) = @_;
1041 my $dbh = C4::Context->dbh;
1042 my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1043 $sth->execute($batch_id);
1044 my ($item_action) = $sth->fetchrow_array();
1045 $sth->finish();
1046 return $item_action;
1051 =head2 SetImportBatchItemAction
1053 SetImportBatchItemAction($batch_id, $new_item_action);
1055 =cut
1057 sub SetImportBatchItemAction {
1058 my ($batch_id, $new_item_action) = @_;
1060 my $dbh = C4::Context->dbh;
1061 my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1062 $sth->execute($new_item_action, $batch_id);
1063 $sth->finish();
1067 =head2 GetImportBatchMatcher
1069 my $matcher_id = GetImportBatchMatcher($batch_id);
1071 =cut
1073 sub GetImportBatchMatcher {
1074 my ($batch_id) = @_;
1076 my $dbh = C4::Context->dbh;
1077 my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1078 $sth->execute($batch_id);
1079 my ($matcher_id) = $sth->fetchrow_array();
1080 $sth->finish();
1081 return $matcher_id;
1086 =head2 SetImportBatchMatcher
1088 SetImportBatchMatcher($batch_id, $new_matcher_id);
1090 =cut
1092 sub SetImportBatchMatcher {
1093 my ($batch_id, $new_matcher_id) = @_;
1095 my $dbh = C4::Context->dbh;
1096 my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1097 $sth->execute($new_matcher_id, $batch_id);
1098 $sth->finish();
1102 =head2 GetImportRecordOverlayStatus
1104 my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1106 =cut
1108 sub GetImportRecordOverlayStatus {
1109 my ($import_record_id) = @_;
1111 my $dbh = C4::Context->dbh;
1112 my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1113 $sth->execute($import_record_id);
1114 my ($overlay_status) = $sth->fetchrow_array();
1115 $sth->finish();
1116 return $overlay_status;
1121 =head2 SetImportRecordOverlayStatus
1123 SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1125 =cut
1127 sub SetImportRecordOverlayStatus {
1128 my ($import_record_id, $new_overlay_status) = @_;
1130 my $dbh = C4::Context->dbh;
1131 my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1132 $sth->execute($new_overlay_status, $import_record_id);
1133 $sth->finish();
1137 =head2 GetImportRecordStatus
1139 my $overlay_status = GetImportRecordStatus($import_record_id);
1141 =cut
1143 sub GetImportRecordStatus {
1144 my ($import_record_id) = @_;
1146 my $dbh = C4::Context->dbh;
1147 my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1148 $sth->execute($import_record_id);
1149 my ($overlay_status) = $sth->fetchrow_array();
1150 $sth->finish();
1151 return $overlay_status;
1156 =head2 SetImportRecordStatus
1158 SetImportRecordStatus($import_record_id, $new_overlay_status);
1160 =cut
1162 sub SetImportRecordStatus {
1163 my ($import_record_id, $new_overlay_status) = @_;
1165 my $dbh = C4::Context->dbh;
1166 my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1167 $sth->execute($new_overlay_status, $import_record_id);
1168 $sth->finish();
1172 =head2 GetImportRecordMatches
1174 my $results = GetImportRecordMatches($import_record_id, $best_only);
1176 =cut
1178 sub GetImportRecordMatches {
1179 my $import_record_id = shift;
1180 my $best_only = @_ ? shift : 0;
1182 my $dbh = C4::Context->dbh;
1183 # FIXME currently biblio only
1184 my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber, score
1185 FROM import_records
1186 JOIN import_record_matches USING (import_record_id)
1187 JOIN biblio ON (biblionumber = candidate_match_id)
1188 WHERE import_record_id = ?
1189 ORDER BY score DESC, biblionumber DESC");
1190 $sth->bind_param(1, $import_record_id);
1191 my $results = [];
1192 $sth->execute();
1193 while (my $row = $sth->fetchrow_hashref) {
1194 push @$results, $row;
1195 last if $best_only;
1197 $sth->finish();
1199 return $results;
1204 =head2 SetImportRecordMatches
1206 SetImportRecordMatches($import_record_id, @matches);
1208 =cut
1210 sub SetImportRecordMatches {
1211 my $import_record_id = shift;
1212 my @matches = @_;
1214 my $dbh = C4::Context->dbh;
1215 my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1216 $delsth->execute($import_record_id);
1217 $delsth->finish();
1219 my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score)
1220 VALUES (?, ?, ?)");
1221 foreach my $match (@matches) {
1222 $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'});
1227 # internal functions
1229 sub _create_import_record {
1230 my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random) = @_;
1232 my $dbh = C4::Context->dbh;
1233 my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml,
1234 record_type, encoding, z3950random)
1235 VALUES (?, ?, ?, ?, ?, ?, ?)");
1236 $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml(),
1237 $record_type, $encoding, $z3950random);
1238 my $import_record_id = $dbh->{'mysql_insertid'};
1239 $sth->finish();
1240 return $import_record_id;
1243 sub _update_import_record_marc {
1244 my ($import_record_id, $marc_record) = @_;
1246 my $dbh = C4::Context->dbh;
1247 my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ?
1248 WHERE import_record_id = ?");
1249 $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml(C4::Context->preference('marcflavour')), $import_record_id);
1250 $sth->finish();
1253 sub _add_biblio_fields {
1254 my ($import_record_id, $marc_record) = @_;
1256 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1257 my $dbh = C4::Context->dbh;
1258 # FIXME no controlnumber, originalsource
1259 $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public
1260 my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1261 $sth->execute($import_record_id, $title, $author, $isbn, $issn);
1262 $sth->finish();
1266 sub _update_biblio_fields {
1267 my ($import_record_id, $marc_record) = @_;
1269 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1270 my $dbh = C4::Context->dbh;
1271 # FIXME no controlnumber, originalsource
1272 # FIXME 2 - should regularize normalization of ISBN wherever it is done
1273 $isbn =~ s/\(.*$//;
1274 $isbn =~ tr/ -_//;
1275 $isbn = uc $isbn;
1276 my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1277 WHERE import_record_id = ?");
1278 $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1279 $sth->finish();
1282 sub _parse_biblio_fields {
1283 my ($marc_record) = @_;
1285 my $dbh = C4::Context->dbh;
1286 my $bibliofields = TransformMarcToKoha($dbh, $marc_record, '');
1287 return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1291 sub _update_batch_record_counts {
1292 my ($batch_id) = @_;
1294 my $dbh = C4::Context->dbh;
1295 my $sth = $dbh->prepare_cached("UPDATE import_batches SET
1296 num_biblios = (
1297 SELECT COUNT(*)
1298 FROM import_records
1299 WHERE import_batch_id = import_batches.import_batch_id
1300 AND record_type = 'biblio'),
1301 num_items = (
1302 SELECT COUNT(*)
1303 FROM import_records
1304 JOIN import_items USING (import_record_id)
1305 WHERE import_batch_id = import_batches.import_batch_id
1306 AND record_type = 'biblio')
1307 WHERE import_batch_id = ?");
1308 $sth->bind_param(1, $batch_id);
1309 $sth->execute();
1310 $sth->finish();
1313 sub _get_commit_action {
1314 my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_;
1316 my ($bib_result, $bib_match, $item_result);
1318 if ($overlay_status ne 'no_match') {
1319 $bib_match = GetBestRecordMatch($import_record_id);
1320 if ($overlay_action eq 'replace') {
1321 $bib_result = defined($bib_match) ? 'replace' : 'create_new';
1322 } elsif ($overlay_action eq 'create_new') {
1323 $bib_result = 'create_new';
1324 } elsif ($overlay_action eq 'ignore') {
1325 $bib_result = 'ignore';
1327 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
1328 } else {
1329 $bib_result = $nomatch_action;
1330 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
1333 return ($bib_result, $item_result, $bib_match);
1336 sub _get_revert_action {
1337 my ($overlay_action, $overlay_status, $status) = @_;
1339 my $bib_result;
1341 if ($status eq 'ignored') {
1342 $bib_result = 'ignore';
1343 } else {
1344 if ($overlay_action eq 'create_new') {
1345 $bib_result = 'delete';
1346 } else {
1347 $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1350 return $bib_result;
1354 __END__
1356 =head1 AUTHOR
1358 Koha Development Team <http://koha-community.org/>
1360 Galen Charlton <galen.charlton@liblime.com>
1362 =cut