misc/migration_tools/bulkmarcimport.pl

   1 #!/usr/bin/perl
   2 # Import an iso2709 file into Koha 3
   3
   4 use strict;
   5 use warnings;
   6 #use diagnostics;
   7 BEGIN {
   8     # find Koha's Perl modules
   9     # test carefully before changing this
  10     use FindBin;
  11     eval { require "$FindBin::Bin/../kohalib.pl" };
  12 }
  13
  14 # Koha modules used
  15 use MARC::File::USMARC;
  16 use MARC::File::XML;
  17 use MARC::Record;
  18 use MARC::Batch;
  19 use MARC::Charset;
  20
  21 use C4::Context;
  22 use C4::Biblio;
  23 use C4::Koha;
  24 use C4::Debug;
  25 use C4::Charset;
  26 use C4::Items;
  27 use YAML;
  28 use Unicode::Normalize;
  29 use Time::HiRes qw(gettimeofday);
  30 use Getopt::Long;
  31 use IO::File;
  32 use Pod::Usage;
  33
  34 use Koha::SearchEngine;
  35 use Koha::SearchEngine::Search;
  36
  37 use open qw( :std :encoding(UTF-8) );
  38 binmode( STDOUT, ":encoding(UTF-8)" );
  39 my ( $input_marc_file, $number, $offset) = ('',0,0);
  40 my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
  41 my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
  42 my $cleanisbn = 1;
  43 my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
  44 my $framework = '';
  45 my $localcust;
  46
  47 $|=1;
  48
  49 GetOptions(
  50     'commit:f'    => \$commit,
  51     'file:s'    => \$input_marc_file,
  52     'n:f' => \$number,
  53     'o|offset:f' => \$offset,
  54     'h' => \$version,
  55     'd' => \$delete,
  56     't|test' => \$test_parameter,
  57     's' => \$skip_marc8_conversion,
  58     'c:s' => \$char_encoding,
  59     'v:+' => \$verbose,
  60     'fk' => \$fk_off,
  61     'm:s' => \$format,
  62     'l:s' => \$logfile,
  63     'append' => \$append,
  64     'k|keepids:s' => \$keepids,
  65     'b|biblios' => \$biblios,
  66     'a|authorities' => \$authorities,
  67     'authtypes:s' => \$authtypes,
  68     'filter=s@'     => \$filters,
  69     'insert'        => \$insert,
  70     'update'        => \$update,
  71     'all'           => \$all,
  72     'match=s@'    => \$match,
  73     'i|isbn' => \$isbn_check,
  74     'x:s' => \$sourcetag,
  75     'y:s' => \$sourcesubfield,
  76     'idmap:s' => \$idmapfl,
  77     'cleanisbn!'     => \$cleanisbn,
  78     'yaml:s'        => \$yamlfile,
  79     'dedupbarcode' => \$dedup_barcode,
  80     'framework=s' => \$framework,
  81     'custom:s'    => \$localcust,
  82 );
  83 $biblios ||= !$authorities;
  84 $insert  ||= !$update;
  85 my $writemode = ($append) ? "a" : "w";
  86
  87 if ($all) {
  88     $insert = 1;
  89     $update = 1;
  90 }
  91
  92 if ($version || ($input_marc_file eq '')) {
  93     pod2usage( -verbose => 2 );
  94     exit;
  95 }
  96
  97 if(defined $localcust) { #local customize module
  98     if(!-e $localcust) {
  99         $localcust= $localcust||'LocalChanges'; #default name
 100         $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
 101         $localcust=~ s/\.pm$//;           #remove extension
 102         my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
 103         if(-e $fqcust) {
 104             $localcust= $fqcust;
 105         }
 106         else {
 107             print "WARNING: customize module $localcust.pm not found!\n";
 108             exit 1;
 109         }
 110     }
 111     require $localcust if $localcust;
 112     $localcust=\&customize if $localcust;
 113 }
 114
 115 my $dbh = C4::Context->dbh;
 116 my $heading_fields=get_heading_fields();
 117
 118 if (defined $idmapfl) {
 119   open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
 120 }
 121
 122 if ((not defined $sourcesubfield) && (not defined $sourcetag)){
 123   $sourcetag="910";
 124   $sourcesubfield="a";
 125 }
 126
 127
 128 # Disable logging for the biblios and authorities import operation. It would unnecessarily
 129 # slow the import
 130
 131 # Disable the syspref cache so we can change logging settings
 132 C4::Context->disable_syspref_cache();
 133 # Save current CataloguingLog and AuthoritiesLog sysprefs values
 134 my $CataloguingLog = C4::Context->preference( 'CataloguingLog' );
 135 my $AuthoritiesLog = C4::Context->preference( 'AuthoritiesLog' );
 136 # Disable logging for both
 137 C4::Context->set_preference( 'CataloguingLog', 0 );
 138 C4::Context->set_preference( 'AuthoritiesLog', 0 );
 139
 140 if ($fk_off) {
 141         $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
 142 }
 143
 144
 145 if ($delete) {
 146         if ($biblios){
 147         print "deleting biblios\n";
 148         $dbh->do("truncate biblio");
 149         $dbh->do("truncate biblioitems");
 150         $dbh->do("truncate items");
 151         }
 152         else {
 153         print "deleting authorities\n";
 154         $dbh->do("truncate auth_header");
 155         }
 156     $dbh->do("truncate zebraqueue");
 157 }
 158
 159
 160
 161 if ($test_parameter) {
 162     print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
 163 }
 164
 165 my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
 166
 167 print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
 168 my $starttime = gettimeofday;
 169 my $batch;
 170 my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
 171 if (defined $format && $format =~ /XML/i) {
 172     # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
 173     # appears to try to convert incoming XML records from MARC-8
 174     # to UTF-8.  Setting the BinaryEncoding key turns that off
 175     # TODO: see what happens to ISO-8859-1 XML files.
 176     # TODO: determine if MARC::Batch can be fixed to handle
 177     #       XML records properly -- it probably should be
 178     #       be using a proper push or pull XML parser to
 179     #       extract the records, not using regexes to look
 180     #       for <record>.*</record>.
 181     $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
 182     my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
 183 #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
 184     $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
 185     $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
 186     $batch = MARC::Batch->new( 'XML', $fh );
 187 } else {
 188     $batch = MARC::Batch->new( 'USMARC', $fh );
 189 }
 190 $batch->warnings_off();
 191 $batch->strict_off();
 192 my $i=0;
 193 my $commitnum = $commit ? $commit : 50;
 194 my $yamlhash;
 195
 196 # Skip file offset
 197 if ( $offset ) {
 198     print "Skipping file offset: $offset records\n";
 199     $batch->next() while ($offset--);
 200 }
 201
 202 my ($tagid,$subfieldid);
 203 if ($authorities){
 204           $tagid='001';
 205 }
 206 else {
 207    ( $tagid, $subfieldid ) =
 208             GetMarcFromKohaField( "biblio.biblionumber", $framework );
 209         $tagid||="001";
 210 }
 211
 212 # the SQL query to search on isbn
 213 my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
 214
 215 $dbh->{AutoCommit} = 0;
 216 my $loghandle;
 217 if ($logfile){
 218    $loghandle= IO::File->new($logfile, $writemode) ;
 219    print $loghandle "id;operation;status\n";
 220 }
 221
 222 my $searcher = Koha::SearchEngine::Search->new(
 223     {
 224         index => (
 225               $authorities
 226             ? $Koha::SearchEngine::AUTHORITIES_INDEX
 227             : $Koha::SearchEngine::BIBLIOS_INDEX
 228         )
 229     }
 230 );
 231
 232 RECORD: while (  ) {
 233     my $record;
 234     # get records
 235     eval { $record = $batch->next() };
 236     if ( $@ ) {
 237         print "Bad MARC record $i: $@ skipped\n";
 238         # FIXME - because MARC::Batch->next() combines grabbing the next
 239         # blob and parsing it into one operation, a correctable condition
 240         # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
 241         # from because we don't have access to the original blob.  Note
 242         # that the staging import can deal with this condition (via
 243         # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
 244         next;
 245     }
 246     # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
 247     last unless ( $record );
 248     $i++;
 249     if( ($verbose//1)==1 ) { #no dot for verbose==2
 250         print "." . ( $i % 100==0 ? "\n$i" : '' );
 251     }
 252
 253     # transcode the record to UTF8 if needed & applicable.
 254     if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
 255         # FIXME update condition
 256         my ($guessed_charset, $charset_errors);
 257          ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
 258         if ($guessed_charset eq 'failed') {
 259             warn "ERROR: failed to perform character conversion for record $i\n";
 260             next RECORD;
 261         }
 262     }
 263     SetUTF8Flag($record);
 264     &$localcust($record) if $localcust;
 265     my $isbn;
 266     # remove trailing - in isbn (only for biblios, of course)
 267     if ($biblios && $cleanisbn) {
 268         my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
 269         my $field = $record->field($tag);
 270         my $isbn = $field && $field->subfield('a');
 271         if ( $isbn ) {
 272             $isbn =~ s/-//g;
 273             $field->update('a' => $isbn);
 274         }
 275     }
 276     my $id;
 277     # search for duplicates (based on Local-number)
 278     my $originalid;
 279     $originalid = GetRecordId( $record, $tagid, $subfieldid );
 280     if ($match) {
 281         require C4::Search;
 282         my $query = build_query( $match, $record );
 283         my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
 284         $debug && warn $query;
 285         my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
 286         # changed to warn so able to continue with one broken record
 287         if ( defined $error ) {
 288             warn "unable to search the database for duplicates : $error";
 289             printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
 290             next RECORD;
 291         }
 292         $debug && warn "$query $server : $totalhits";
 293         if ( $results && scalar(@$results) == 1 ) {
 294             my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
 295             SetUTF8Flag($marcrecord);
 296             $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
 297             if ( $authorities && $marcFlavour ) {
 298                 #Skip if authority in database is the same as the on in database
 299                 if ( $marcrecord->field('005') && $record->field('005') &&
 300                      $marcrecord->field('005')->data && $record->field('005')->data &&
 301                      $marcrecord->field('005')->data >= $record->field('005')->data ) {
 302                     if ($yamlfile) {
 303                         $yamlhash->{$originalid}->{'authid'} = $id;
 304
 305                         # we recover all subfields of the heading authorities
 306                         my @subfields;
 307                         foreach my $field ( $marcrecord->field("2..") ) {
 308                             push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
 309                         }
 310                         $yamlhash->{$originalid}->{'subfields'} = \@subfields;
 311                     }
 312                     next;
 313                 }
 314             }
 315         } elsif ( $results && scalar(@$results) > 1 ) {
 316             $debug && warn "more than one match for $query";
 317         } else {
 318             $debug && warn "nomatch for $query";
 319         }
 320     }
 321     if ($keepids && $originalid) {
 322             my $storeidfield;
 323             if ( length($keepids) == 3 ) {
 324                 $storeidfield = MARC::Field->new( $keepids, $originalid );
 325             } else {
 326                 $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
 327             }
 328             $record->insert_fields_ordered($storeidfield);
 329             $record->delete_field( $record->field($tagid) );
 330     }
 331     foreach my $stringfilter (@$filters) {
 332         if ( length($stringfilter) == 3 ) {
 333             foreach my $field ( $record->field($stringfilter) ) {
 334                 $record->delete_field($field);
 335                 $debug && warn "removed : ", $field->as_string;
 336             }
 337         } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
 338             my $removetag = $1;
 339             my $removesubfield = $2;
 340             my $removematch = $3;
 341             if ( ( $removetag > "010" ) && $removesubfield ) {
 342                 foreach my $field ( $record->field($removetag) ) {
 343                     $field->delete_subfield( code => "$removesubfield", match => $removematch );
 344                     $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
 345                 }
 346             }
 347         }
 348     }
 349     unless ($test_parameter) {
 350         if ($authorities){
 351             use C4::AuthoritiesMarc;
 352             my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
 353             my $authid= ($id?$id:GuessAuthId($record));
 354             if ($authid && GetAuthority($authid) && $update ){
 355             ## Authority has an id and is in database : Replace
 356                 eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
 357                 if ($@){
 358                     warn "Problem with authority $authid Cannot Modify";
 359                                         printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
 360                 }
 361                                 else{
 362                                         printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
 363                                 }
 364             }
 365             elsif (defined $authid) {
 366             ## An authid is defined but no authority in database : add
 367                 eval { ( $authid ) = AddAuthority($record,$authid, $authtypecode) };
 368                 if ($@){
 369                     warn "Problem with authority $authid Cannot Add ".$@;
 370                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
 371                 }
 372                                 else{
 373                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
 374                                 }
 375             }
 376                 else {
 377             ## True insert in database
 378                 eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
 379                 if ($@){
 380                     warn "Problem with authority $authid Cannot Add".$@;
 381                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
 382                 }
 383                                 else{
 384                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
 385                                 }
 386                 }
 387             if ($yamlfile) {
 388             $yamlhash->{$originalid}->{'authid'} = $authid;
 389             my @subfields;
 390             foreach my $field ( $record->field("2..") ) {
 391                 push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
 392             }
 393             $yamlhash->{$originalid}->{'subfields'} = \@subfields;
 394             }
 395         }
 396         else {
 397             my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
 398             $biblionumber = $id;
 399             # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
 400             if (!$biblionumber && $isbn_check && $isbn) {
 401     #         warn "search ISBN : $isbn";
 402                 $sth_isbn->execute($isbn);
 403                 ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
 404             }
 405                 if (defined $idmapfl) {
 406                                 if ($sourcetag < "010"){
 407                                         if ($record->field($sourcetag)){
 408                                           my $source = $record->field($sourcetag)->data();
 409                                           printf(IDMAP "%s|%s\n",$source,$biblionumber);
 410                                         }
 411                             } else {
 412                                         my $source=$record->subfield($sourcetag,$sourcesubfield);
 413                                         printf(IDMAP "%s|%s\n",$source,$biblionumber);
 414                           }
 415                         }
 416                                         # create biblio, unless we already have it ( either match or isbn )
 417             if ($biblionumber) {
 418                 eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
 419                 if ($update) {
 420                     eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber) ) };
 421                     if ($@) {
 422                         warn "ERROR: Edit biblio $biblionumber failed: $@\n";
 423                         printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
 424                         next RECORD;
 425                     } else {
 426                         printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
 427                     }
 428                 } else {
 429                     printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
 430                 }
 431             } else {
 432                 if ($insert) {
 433                     eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
 434                     if ($@) {
 435                         warn "ERROR: Adding biblio $biblionumber failed: $@\n";
 436                         printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
 437                         next RECORD;
 438                     } else {
 439                         printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
 440                     }
 441                 } else {
 442                     printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
 443                 }
 444             }
 445             eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
 446             my $error_adding = $@;
 447             # Work on a clone so that if there are real errors, we can maybe
 448             # fix them up later.
 449                         my $clone_record = $record->clone();
 450             C4::Biblio::_strip_item_fields($clone_record, '');
 451             # This sets the marc fields if there was an error, and also calls
 452             # defer_marc_save.
 453             ModBiblioMarc( $clone_record, $biblionumber, $framework );
 454             if ( $error_adding ) {
 455                 warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
 456                                 printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
 457                 # if we failed because of an exception, assume that
 458                 # the MARC columns in biblioitems were not set.
 459                 next RECORD;
 460             }
 461                         else{
 462                                 printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
 463                         }
 464             if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
 465                 # Find the record called 'barcode'
 466                 my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode', $framework);
 467                 # Now remove any items that didn't have a duplicate_barcode error,
 468                 # erase the barcodes on items that did, and re-add those items.
 469                 my %dupes;
 470                 foreach my $i (0 .. $#{$errors_ref}) {
 471                     my $ref = $errors_ref->[$i];
 472                     if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
 473                         $dupes{$ref->{item_sequence}} = 1;
 474                         # Delete the error message because we're going to
 475                         # retry this one.
 476                         delete $errors_ref->[$i];
 477                     }
 478                 }
 479                 my $seq = 0;
 480                 foreach my $field ($record->field($tag)) {
 481                     $seq++;
 482                     if ($dupes{$seq}) {
 483                         # Here we remove the barcode
 484                         $field->delete_subfield(code => $sub);
 485                     } else {
 486                         # otherwise we delete the field because we don't want
 487                         # two of them
 488                         $record->delete_fields($field);
 489                     }
 490                 }
 491                 # Now re-add the record as before, adding errors to the prev list
 492                 my $more_errors;
 493                 eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
 494                 if ( $@ ) {
 495                     warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
 496                     printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
 497                     # if we failed because of an exception, assume that
 498                     # the MARC columns in biblioitems were not set.
 499                     ModBiblioMarc( $record, $biblionumber, $framework );
 500                     next RECORD;
 501                 } else {
 502                     printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
 503                 }
 504                 push @$errors_ref, @{ $more_errors };
 505             }
 506             if ($#{ $errors_ref } > -1) {
 507                 report_item_errors($biblionumber, $errors_ref);
 508             }
 509             $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
 510         }
 511         $dbh->commit() if (0 == $i % $commitnum);
 512     }
 513     print $record->as_formatted()."\n" if ($verbose//0)==2;
 514     last if $i == $number;
 515 }
 516 $dbh->commit();
 517 $dbh->{AutoCommit} = 1;
 518
 519
 520 if ($fk_off) {
 521         $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
 522 }
 523
 524 # Restore CataloguingLog
 525 C4::Context->set_preference( 'CataloguingLog', $CataloguingLog );
 526 # Restore AuthoritiesLog
 527 C4::Context->set_preference( 'AuthoritiesLog', $AuthoritiesLog );
 528
 529 my $timeneeded = gettimeofday - $starttime;
 530 print "\n$i MARC records done in $timeneeded seconds\n";
 531 if ($logfile){
 532   print $loghandle "file : $input_marc_file\n";
 533   print $loghandle "$i MARC records done in $timeneeded seconds\n";
 534   $loghandle->close;
 535 }
 536 if ($yamlfile) {
 537     open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
 538     print $yamlfileout Dump($yamlhash);
 539 }
 540 exit 0;
 541
 542 sub GetRecordId{
 543         my $marcrecord=shift;
 544         my $tag=shift;
 545         my $subfield=shift;
 546         my $id;
 547         if ($tag lt "010"){
 548                 return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
 549         }
 550         elsif ($subfield){
 551                 if ($marcrecord->field($tag)){
 552                         return $marcrecord->subfield($tag,$subfield);
 553                 }
 554         }
 555         return $id;
 556 }
 557 sub build_query {
 558         my $match = shift;
 559         my $record=shift;
 560         my @searchstrings;
 561         foreach my $matchingpoint (@$match){
 562           my $string = build_simplequery($matchingpoint,$record);
 563           push @searchstrings,$string if (length($string)>0);
 564         }
 565     my $QParser;
 566     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 567     my $op;
 568     if ($QParser) {
 569         $op = '&&';
 570     } else {
 571         $op = 'and';
 572     }
 573     return join(" $op ",@searchstrings);
 574 }
 575 sub build_simplequery {
 576         my $element=shift;
 577         my $record=shift;
 578     my @searchstrings;
 579     my ($index,$recorddata)=split /,/,$element;
 580     if ($recorddata=~/(\d{3})(.*)/) {
 581         my ($tag,$subfields) =($1,$2);
 582         foreach my $field ($record->field($tag)){
 583                   if (length($field->as_string("$subfields"))>0){
 584               push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
 585                   }
 586         }
 587     }
 588     my $QParser;
 589     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 590     my $op;
 591     if ($QParser) {
 592         $op = '&&';
 593     } else {
 594         $op = 'and';
 595     }
 596     return join(" $op ",@searchstrings);
 597 }
 598 sub report_item_errors {
 599     my $biblionumber = shift;
 600     my $errors_ref = shift;
 601
 602     foreach my $error (@{ $errors_ref }) {
 603         next if !$error;
 604         my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
 605         my $error_code = $error->{'error_code'};
 606         $error_code =~ s/_/ /g;
 607         $msg .= "$error_code $error->{'error_information'}";
 608         print $msg, "\n";
 609     }
 610 }
 611 sub printlog{
 612         my $logelements=shift;
 613     print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
 614 }
 615 sub get_heading_fields{
 616     my $headingfields;
 617     if ($authtypes){
 618         $headingfields=YAML::LoadFile($authtypes);
 619         $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
 620         $debug && warn YAML::Dump($headingfields);
 621     }
 622     unless ($headingfields){
 623         $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
 624         $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
 625     }
 626     return $headingfields;
 627 }
 628
 629 =head1 NAME
 630
 631 bulkmarcimport.pl - Import bibliographic/authority records into Koha
 632
 633 =head1 USAGE
 634
 635  $ export KOHA_CONF=/etc/koha.conf
 636  $ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\
 637     -file /home/jmf/koha.mrc -n 3000
 638
 639 =head1 WARNING
 640
 641 Don't use this script before you've entered and checked your MARC parameters
 642 tables twice (or more!). Otherwise, the import won't work correctly and you
 643 will get invalid data.
 644
 645 =head1 DESCRIPTION
 646
 647 =over
 648
 649 =item  B<-h>
 650
 651 This version/help screen
 652
 653 =item B<-b, -biblios>
 654
 655 Type of import: bibliographic records
 656
 657 =item B<-a, -authorities>
 658
 659 Type of import: authority records
 660
 661 =item B<-file>=I<FILE>
 662
 663 The I<FILE> to import
 664
 665 =item  B<-v>
 666
 667 Verbose mode. 1 means "some infos", 2 means "MARC dumping"
 668
 669 =item B<-fk>
 670
 671 Turn off foreign key checks during import.
 672
 673 =item B<-n>=I<NUMBER>
 674
 675 The I<NUMBER> of records to import. If missing, all the file is imported
 676
 677 =item B<-o, -offset>=I<NUMBER>
 678
 679 File offset before importing, ie I<NUMBER> of records to skip.
 680
 681 =item B<-commit>=I<NUMBER>
 682
 683 The I<NUMBER> of records to wait before performing a 'commit' operation
 684
 685 =item B<-l>
 686
 687 File logs actions done for each record and their status into file
 688
 689 =item B<-append>
 690
 691 If specified, data will be appended to the logfile. If not, the logfile will be erased for each execution.
 692
 693 =item B<-t, -test>
 694
 695 Test mode: parses the file, saying what he would do, but doing nothing.
 696
 697 =item B<-s>
 698
 699 Skip automatic conversion of MARC-8 to UTF-8.  This option is provided for
 700 debugging.
 701
 702 =item B<-c>=I<CHARACTERISTIC>
 703
 704 The I<CHARACTERISTIC> MARC flavour. At the moment, only I<MARC21> and
 705 I<UNIMARC> are supported. MARC21 by default.
 706
 707 =item B<-d>
 708
 709 Delete EVERYTHING related to biblio in koha-DB before import. Tables: biblio,
 710 biblioitems, items
 711
 712 =item B<-m>=I<FORMAT>
 713
 714 Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
 715
 716 =item B<-authtypes>
 717
 718 file yamlfile with authoritiesTypes and distinguishable record field in order
 719 to store the correct authtype
 720
 721 =item B<-yaml>
 722
 723 yaml file  format a yaml file with ids
 724
 725 =item B<-filter>
 726
 727 list of fields that will not be imported. Can be any from 000 to 999 or field,
 728 subfield and subfield's matching value such as 200avalue
 729
 730 =item B<-insert>
 731
 732 if set, only insert when possible
 733
 734 =item B<-update>
 735
 736 if set, only updates (any biblio should have a matching record)
 737
 738 =item B<-all>
 739
 740 if set, do whatever is required
 741
 742 =item B<-k, -keepids>=<FIELD>
 743
 744 Field store ids in I<FIELD> (useful for authorities, where 001 contains the
 745 authid for Koha, that can contain a very valuable info for authorities coming
 746 from LOC or BNF. useless for biblios probably)
 747
 748 =item B<-match>=<FIELD>
 749
 750 I<FIELD> matchindex,fieldtomatch matchpoint to use to deduplicate fieldtomatch
 751 can be either 001 to 999 or field and list of subfields as such 100abcde
 752
 753 =item B<-i,-isbn>
 754
 755 If set, a search will be done on isbn, and, if the same isbn is found, the
 756 biblio is not added. It's another method to deduplicate.  B<-match> & B<-isbn>
 757 can be both set.
 758
 759 =item B<-cleanisbn>
 760
 761 Clean ISBN fields from entering biblio records, ie removes hyphens. By default,
 762 ISBN are cleaned. --nocleanisbn will keep ISBN unchanged.
 763
 764 =item B<-x>=I<TAG>
 765
 766 Source bib I<TAG> for reporting the source bib number
 767
 768 =item B<-y>=I<SUBFIELD>
 769
 770 Source I<SUBFIELD> for reporting the source bib number
 771
 772 =item B<-idmap>=I<FILE>
 773
 774 I<FILE> for the koha bib and source id
 775
 776 =item B<-keepids>
 777
 778 Store ids in 009 (useful for authorities, where 001 contains the authid for
 779 Koha, that can contain a very valuable info for authorities coming from LOC or
 780 BNF. useless for biblios probably)
 781
 782 =item B<-dedupbarcode>
 783
 784 If set, whenever a duplicate barcode is detected, it is removed and the attempt
 785 to add the record is retried, thereby giving the record a blank barcode. This
 786 is useful when something has set barcodes to be a biblio ID, or similar
 787 (usually other software.)
 788
 789 =item B<-framework>
 790
 791 This is the code for the framework that the requested records will have attached
 792 to them when they are created. If not specified, then the default framework
 793 will be used.
 794
 795 =item B<-custom>=I<MODULE>
 796
 797 This parameter allows you to use a local module with a customize subroutine
 798 that is called for each MARC record.
 799 If no filename is passed, LocalChanges.pm is assumed to be in the
 800 migration_tools subdirectory. You may pass an absolute file name or a file name
 801 from the migration_tools directory.
 802
 803 =back
 804
 805 =cut
 806