misc/migration_tools/bulkmarcimport.pl

   1 #!/usr/bin/perl
   2 # Import an iso2709 file into Koha 3
   3
   4 use strict;
   5 use warnings;
   6 #use diagnostics;
   7 BEGIN {
   8     # find Koha's Perl modules
   9     # test carefully before changing this
  10     use FindBin;
  11     eval { require "$FindBin::Bin/../kohalib.pl" };
  12 }
  13
  14 # Koha modules used
  15 use MARC::File::USMARC;
  16 use MARC::File::XML;
  17 use MARC::Record;
  18 use MARC::Batch;
  19 use MARC::Charset;
  20
  21 use C4::Context;
  22 use C4::Biblio;
  23 use C4::Koha;
  24 use C4::Debug;
  25 use C4::Charset;
  26 use C4::Items;
  27 use YAML;
  28 use Unicode::Normalize;
  29 use Time::HiRes qw(gettimeofday);
  30 use Getopt::Long;
  31 use IO::File;
  32 use Pod::Usage;
  33
  34 use Koha::Biblios;
  35 use Koha::SearchEngine;
  36 use Koha::SearchEngine::Search;
  37
  38 use open qw( :std :encoding(UTF-8) );
  39 binmode( STDOUT, ":encoding(UTF-8)" );
  40 my ( $input_marc_file, $number, $offset) = ('',0,0);
  41 my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
  42 my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
  43 my $cleanisbn = 1;
  44 my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
  45 my $framework = '';
  46 my $localcust;
  47
  48 $|=1;
  49
  50 GetOptions(
  51     'commit:f'    => \$commit,
  52     'file:s'    => \$input_marc_file,
  53     'n:f' => \$number,
  54     'o|offset:f' => \$offset,
  55     'h' => \$version,
  56     'd' => \$delete,
  57     't|test' => \$test_parameter,
  58     's' => \$skip_marc8_conversion,
  59     'c:s' => \$char_encoding,
  60     'v:+' => \$verbose,
  61     'fk' => \$fk_off,
  62     'm:s' => \$format,
  63     'l:s' => \$logfile,
  64     'append' => \$append,
  65     'k|keepids:s' => \$keepids,
  66     'b|biblios' => \$biblios,
  67     'a|authorities' => \$authorities,
  68     'authtypes:s' => \$authtypes,
  69     'filter=s@'     => \$filters,
  70     'insert'        => \$insert,
  71     'update'        => \$update,
  72     'all'           => \$all,
  73     'match=s@'    => \$match,
  74     'i|isbn' => \$isbn_check,
  75     'x:s' => \$sourcetag,
  76     'y:s' => \$sourcesubfield,
  77     'idmap:s' => \$idmapfl,
  78     'cleanisbn!'     => \$cleanisbn,
  79     'yaml:s'        => \$yamlfile,
  80     'dedupbarcode' => \$dedup_barcode,
  81     'framework=s' => \$framework,
  82     'custom:s'    => \$localcust,
  83 );
  84 $biblios ||= !$authorities;
  85 $insert  ||= !$update;
  86 my $writemode = ($append) ? "a" : "w";
  87
  88 pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) if $biblios && $authorities;
  89
  90 if ($all) {
  91     $insert = 1;
  92     $update = 1;
  93 }
  94
  95 if ($version || ($input_marc_file eq '')) {
  96     pod2usage( -verbose => 2 );
  97     exit;
  98 }
  99
 100 if(defined $localcust) { #local customize module
 101     if(!-e $localcust) {
 102         $localcust= $localcust||'LocalChanges'; #default name
 103         $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
 104         $localcust=~ s/\.pm$//;           #remove extension
 105         my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
 106         if(-e $fqcust) {
 107             $localcust= $fqcust;
 108         }
 109         else {
 110             print "WARNING: customize module $localcust.pm not found!\n";
 111             exit 1;
 112         }
 113     }
 114     require $localcust if $localcust;
 115     $localcust=\&customize if $localcust;
 116 }
 117
 118 my $dbh = C4::Context->dbh;
 119 my $heading_fields=get_heading_fields();
 120
 121 if (defined $idmapfl) {
 122   open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
 123 }
 124
 125 if ((not defined $sourcesubfield) && (not defined $sourcetag)){
 126   $sourcetag="910";
 127   $sourcesubfield="a";
 128 }
 129
 130
 131 # Disable logging for the biblios and authorities import operation. It would unnecessarily
 132 # slow the import
 133
 134 # Disable the syspref cache so we can change logging settings
 135 C4::Context->disable_syspref_cache();
 136 # Save current CataloguingLog and AuthoritiesLog sysprefs values
 137 my $CataloguingLog = C4::Context->preference( 'CataloguingLog' );
 138 my $AuthoritiesLog = C4::Context->preference( 'AuthoritiesLog' );
 139 # Disable logging for both
 140 C4::Context->set_preference( 'CataloguingLog', 0 );
 141 C4::Context->set_preference( 'AuthoritiesLog', 0 );
 142
 143 if ($fk_off) {
 144         $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
 145 }
 146
 147
 148 if ($delete) {
 149         if ($biblios){
 150         print "deleting biblios\n";
 151         $dbh->do("truncate biblio");
 152         $dbh->do("truncate biblioitems");
 153         $dbh->do("truncate items");
 154         }
 155         else {
 156         print "deleting authorities\n";
 157         $dbh->do("truncate auth_header");
 158         }
 159     $dbh->do("truncate zebraqueue");
 160 }
 161
 162
 163
 164 if ($test_parameter) {
 165     print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
 166 }
 167
 168 my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
 169
 170 print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
 171 my $starttime = gettimeofday;
 172 my $batch;
 173 my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
 174 if (defined $format && $format =~ /XML/i) {
 175     # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
 176     # appears to try to convert incoming XML records from MARC-8
 177     # to UTF-8.  Setting the BinaryEncoding key turns that off
 178     # TODO: see what happens to ISO-8859-1 XML files.
 179     # TODO: determine if MARC::Batch can be fixed to handle
 180     #       XML records properly -- it probably should be
 181     #       be using a proper push or pull XML parser to
 182     #       extract the records, not using regexes to look
 183     #       for <record>.*</record>.
 184     $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
 185     my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
 186 #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
 187     $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
 188     $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
 189     $batch = MARC::Batch->new( 'XML', $fh );
 190 } else {
 191     $batch = MARC::Batch->new( 'USMARC', $fh );
 192 }
 193 $batch->warnings_off();
 194 $batch->strict_off();
 195 my $i=0;
 196 my $commitnum = $commit ? $commit : 50;
 197 my $yamlhash;
 198
 199 # Skip file offset
 200 if ( $offset ) {
 201     print "Skipping file offset: $offset records\n";
 202     $batch->next() while ($offset--);
 203 }
 204
 205 my ($tagid,$subfieldid);
 206 if ($authorities){
 207           $tagid='001';
 208 }
 209 else {
 210    ( $tagid, $subfieldid ) =
 211             GetMarcFromKohaField( "biblio.biblionumber", $framework );
 212         $tagid||="001";
 213 }
 214
 215 # the SQL query to search on isbn
 216 my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
 217
 218 $dbh->{AutoCommit} = 0;
 219 my $loghandle;
 220 if ($logfile){
 221    $loghandle= IO::File->new($logfile, $writemode) ;
 222    print $loghandle "id;operation;status\n";
 223 }
 224
 225 my $searcher = Koha::SearchEngine::Search->new(
 226     {
 227         index => (
 228               $authorities
 229             ? $Koha::SearchEngine::AUTHORITIES_INDEX
 230             : $Koha::SearchEngine::BIBLIOS_INDEX
 231         )
 232     }
 233 );
 234
 235 RECORD: while (  ) {
 236     my $record;
 237     # get records
 238     eval { $record = $batch->next() };
 239     if ( $@ ) {
 240         print "Bad MARC record $i: $@ skipped\n";
 241         # FIXME - because MARC::Batch->next() combines grabbing the next
 242         # blob and parsing it into one operation, a correctable condition
 243         # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
 244         # from because we don't have access to the original blob.  Note
 245         # that the staging import can deal with this condition (via
 246         # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
 247         next;
 248     }
 249     # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
 250     last unless ( $record );
 251     $i++;
 252     if( ($verbose//1)==1 ) { #no dot for verbose==2
 253         print "." . ( $i % 100==0 ? "\n$i" : '' );
 254     }
 255
 256     # transcode the record to UTF8 if needed & applicable.
 257     if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
 258         # FIXME update condition
 259         my ($guessed_charset, $charset_errors);
 260          ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
 261         if ($guessed_charset eq 'failed') {
 262             warn "ERROR: failed to perform character conversion for record $i\n";
 263             next RECORD;
 264         }
 265     }
 266     SetUTF8Flag($record);
 267     &$localcust($record) if $localcust;
 268     my $isbn;
 269     # remove trailing - in isbn (only for biblios, of course)
 270     if ($biblios && $cleanisbn) {
 271         my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
 272         my $field = $record->field($tag);
 273         my $isbn = $field && $field->subfield('a');
 274         if ( $isbn ) {
 275             $isbn =~ s/-//g;
 276             $field->update('a' => $isbn);
 277         }
 278     }
 279     my $id;
 280     # search for duplicates (based on Local-number)
 281     my $originalid;
 282     $originalid = GetRecordId( $record, $tagid, $subfieldid );
 283     if ($match) {
 284         require C4::Search;
 285         my $query = build_query( $match, $record );
 286         my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
 287         $debug && warn $query;
 288         my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
 289         # changed to warn so able to continue with one broken record
 290         if ( defined $error ) {
 291             warn "unable to search the database for duplicates : $error";
 292             printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
 293             next RECORD;
 294         }
 295         $debug && warn "$query $server : $totalhits";
 296         if ( $results && scalar(@$results) == 1 ) {
 297             my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
 298             SetUTF8Flag($marcrecord);
 299             $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
 300             if ( $authorities && $marcFlavour ) {
 301                 #Skip if authority in database is the same as the on in database
 302                 if ( $marcrecord->field('005') && $record->field('005') &&
 303                      $marcrecord->field('005')->data && $record->field('005')->data &&
 304                      $marcrecord->field('005')->data >= $record->field('005')->data ) {
 305                     if ($yamlfile) {
 306                         $yamlhash->{$originalid}->{'authid'} = $id;
 307
 308                         # we recover all subfields of the heading authorities
 309                         my @subfields;
 310                         foreach my $field ( $marcrecord->field("2..") ) {
 311                             push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
 312                         }
 313                         $yamlhash->{$originalid}->{'subfields'} = \@subfields;
 314                     }
 315                     next;
 316                 }
 317             }
 318         } elsif ( $results && scalar(@$results) > 1 ) {
 319             $debug && warn "more than one match for $query";
 320         } else {
 321             $debug && warn "nomatch for $query";
 322         }
 323     }
 324     if ($keepids && $originalid) {
 325             my $storeidfield;
 326             if ( length($keepids) == 3 ) {
 327                 $storeidfield = MARC::Field->new( $keepids, $originalid );
 328             } else {
 329                 $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
 330             }
 331             $record->insert_fields_ordered($storeidfield);
 332             $record->delete_field( $record->field($tagid) );
 333     }
 334     foreach my $stringfilter (@$filters) {
 335         if ( length($stringfilter) == 3 ) {
 336             foreach my $field ( $record->field($stringfilter) ) {
 337                 $record->delete_field($field);
 338                 $debug && warn "removed : ", $field->as_string;
 339             }
 340         } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
 341             my $removetag = $1;
 342             my $removesubfield = $2;
 343             my $removematch = $3;
 344             if ( ( $removetag > "010" ) && $removesubfield ) {
 345                 foreach my $field ( $record->field($removetag) ) {
 346                     $field->delete_subfield( code => "$removesubfield", match => $removematch );
 347                     $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
 348                 }
 349             }
 350         }
 351     }
 352     unless ($test_parameter) {
 353         if ($authorities){
 354             use C4::AuthoritiesMarc;
 355             my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
 356             my $authid= ($id?$id:GuessAuthId($record));
 357             if ($authid && GetAuthority($authid) && $update ){
 358             ## Authority has an id and is in database : Replace
 359                 eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
 360                 if ($@){
 361                     warn "Problem with authority $authid Cannot Modify";
 362                                         printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
 363                 }
 364                                 else{
 365                                         printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
 366                                 }
 367             }
 368             elsif (defined $authid) {
 369             ## An authid is defined but no authority in database : add
 370                 eval { ( $authid ) = AddAuthority($record,$authid, $authtypecode) };
 371                 if ($@){
 372                     warn "Problem with authority $authid Cannot Add ".$@;
 373                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
 374                 }
 375                                 else{
 376                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
 377                                 }
 378             }
 379                 else {
 380             ## True insert in database
 381                 eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
 382                 if ($@){
 383                     warn "Problem with authority $authid Cannot Add".$@;
 384                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
 385                 }
 386                                 else{
 387                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
 388                                 }
 389                 }
 390             if ($yamlfile) {
 391             $yamlhash->{$originalid}->{'authid'} = $authid;
 392             my @subfields;
 393             foreach my $field ( $record->field("2..") ) {
 394                 push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
 395             }
 396             $yamlhash->{$originalid}->{'subfields'} = \@subfields;
 397             }
 398         }
 399         else {
 400             my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
 401             $biblionumber = $id;
 402             # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
 403             if (!$biblionumber && $isbn_check && $isbn) {
 404     #         warn "search ISBN : $isbn";
 405                 $sth_isbn->execute($isbn);
 406                 ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
 407             }
 408                 if (defined $idmapfl) {
 409                                 if ($sourcetag < "010"){
 410                                         if ($record->field($sourcetag)){
 411                                           my $source = $record->field($sourcetag)->data();
 412                                           printf(IDMAP "%s|%s\n",$source,$biblionumber);
 413                                         }
 414                             } else {
 415                                         my $source=$record->subfield($sourcetag,$sourcesubfield);
 416                                         printf(IDMAP "%s|%s\n",$source,$biblionumber);
 417                           }
 418                         }
 419                                         # create biblio, unless we already have it ( either match or isbn )
 420             if ($biblionumber) {
 421                 eval{
 422                     $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
 423                 };
 424                 if ($update) {
 425                     eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber) ) };
 426                     if ($@) {
 427                         warn "ERROR: Edit biblio $biblionumber failed: $@\n";
 428                         printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
 429                         next RECORD;
 430                     } else {
 431                         printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
 432                     }
 433                 } else {
 434                     printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
 435                 }
 436             } else {
 437                 if ($insert) {
 438                     eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
 439                     if ($@) {
 440                         warn "ERROR: Adding biblio $biblionumber failed: $@\n";
 441                         printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
 442                         next RECORD;
 443                     } else {
 444                         printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
 445                     }
 446                 } else {
 447                     printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
 448                 }
 449             }
 450             eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
 451             my $error_adding = $@;
 452             # Work on a clone so that if there are real errors, we can maybe
 453             # fix them up later.
 454                         my $clone_record = $record->clone();
 455             C4::Biblio::_strip_item_fields($clone_record, '');
 456             # This sets the marc fields if there was an error, and also calls
 457             # defer_marc_save.
 458             ModBiblioMarc( $clone_record, $biblionumber, $framework );
 459             if ( $error_adding ) {
 460                 warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
 461                                 printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
 462                 # if we failed because of an exception, assume that
 463                 # the MARC columns in biblioitems were not set.
 464                 next RECORD;
 465             }
 466                         else{
 467                                 printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
 468                         }
 469             if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
 470                 # Find the record called 'barcode'
 471                 my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode', $framework);
 472                 # Now remove any items that didn't have a duplicate_barcode error,
 473                 # erase the barcodes on items that did, and re-add those items.
 474                 my %dupes;
 475                 foreach my $i (0 .. $#{$errors_ref}) {
 476                     my $ref = $errors_ref->[$i];
 477                     if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
 478                         $dupes{$ref->{item_sequence}} = 1;
 479                         # Delete the error message because we're going to
 480                         # retry this one.
 481                         delete $errors_ref->[$i];
 482                     }
 483                 }
 484                 my $seq = 0;
 485                 foreach my $field ($record->field($tag)) {
 486                     $seq++;
 487                     if ($dupes{$seq}) {
 488                         # Here we remove the barcode
 489                         $field->delete_subfield(code => $sub);
 490                     } else {
 491                         # otherwise we delete the field because we don't want
 492                         # two of them
 493                         $record->delete_fields($field);
 494                     }
 495                 }
 496                 # Now re-add the record as before, adding errors to the prev list
 497                 my $more_errors;
 498                 eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
 499                 if ( $@ ) {
 500                     warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
 501                     printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
 502                     # if we failed because of an exception, assume that
 503                     # the MARC columns in biblioitems were not set.
 504                     ModBiblioMarc( $record, $biblionumber, $framework );
 505                     next RECORD;
 506                 } else {
 507                     printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
 508                 }
 509                 push @$errors_ref, @{ $more_errors };
 510             }
 511             if ($#{ $errors_ref } > -1) {
 512                 report_item_errors($biblionumber, $errors_ref);
 513             }
 514             $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
 515         }
 516         $dbh->commit() if (0 == $i % $commitnum);
 517     }
 518     print $record->as_formatted()."\n" if ($verbose//0)==2;
 519     last if $i == $number;
 520 }
 521 $dbh->commit();
 522 $dbh->{AutoCommit} = 1;
 523
 524
 525 if ($fk_off) {
 526         $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
 527 }
 528
 529 # Restore CataloguingLog
 530 C4::Context->set_preference( 'CataloguingLog', $CataloguingLog );
 531 # Restore AuthoritiesLog
 532 C4::Context->set_preference( 'AuthoritiesLog', $AuthoritiesLog );
 533
 534 my $timeneeded = gettimeofday - $starttime;
 535 print "\n$i MARC records done in $timeneeded seconds\n";
 536 if ($logfile){
 537   print $loghandle "file : $input_marc_file\n";
 538   print $loghandle "$i MARC records done in $timeneeded seconds\n";
 539   $loghandle->close;
 540 }
 541 if ($yamlfile) {
 542     open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
 543     print $yamlfileout Dump($yamlhash);
 544 }
 545 exit 0;
 546
 547 sub GetRecordId{
 548         my $marcrecord=shift;
 549         my $tag=shift;
 550         my $subfield=shift;
 551         my $id;
 552         if ($tag lt "010"){
 553                 return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
 554         }
 555         elsif ($subfield){
 556                 if ($marcrecord->field($tag)){
 557                         return $marcrecord->subfield($tag,$subfield);
 558                 }
 559         }
 560         return $id;
 561 }
 562 sub build_query {
 563         my $match = shift;
 564         my $record=shift;
 565         my @searchstrings;
 566         foreach my $matchingpoint (@$match){
 567           my $string = build_simplequery($matchingpoint,$record);
 568           push @searchstrings,$string if (length($string)>0);
 569         }
 570     my $QParser;
 571     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 572     my $op;
 573     if ($QParser) {
 574         $op = '&&';
 575     } else {
 576         $op = 'and';
 577     }
 578     return join(" $op ",@searchstrings);
 579 }
 580 sub build_simplequery {
 581         my $element=shift;
 582         my $record=shift;
 583     my @searchstrings;
 584     my ($index,$recorddata)=split /,/,$element;
 585     if ($recorddata=~/(\d{3})(.*)/) {
 586         my ($tag,$subfields) =($1,$2);
 587         foreach my $field ($record->field($tag)){
 588                   if (length($field->as_string("$subfields"))>0){
 589               push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
 590                   }
 591         }
 592     }
 593     my $QParser;
 594     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 595     my $op;
 596     if ($QParser) {
 597         $op = '&&';
 598     } else {
 599         $op = 'and';
 600     }
 601     return join(" $op ",@searchstrings);
 602 }
 603 sub report_item_errors {
 604     my $biblionumber = shift;
 605     my $errors_ref = shift;
 606
 607     foreach my $error (@{ $errors_ref }) {
 608         next if !$error;
 609         my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
 610         my $error_code = $error->{'error_code'};
 611         $error_code =~ s/_/ /g;
 612         $msg .= "$error_code $error->{'error_information'}";
 613         print $msg, "\n";
 614     }
 615 }
 616 sub printlog{
 617         my $logelements=shift;
 618     print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
 619 }
 620 sub get_heading_fields{
 621     my $headingfields;
 622     if ($authtypes){
 623         $headingfields=YAML::LoadFile($authtypes);
 624         $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
 625         $debug && warn YAML::Dump($headingfields);
 626     }
 627     unless ($headingfields){
 628         $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
 629         $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
 630     }
 631     return $headingfields;
 632 }
 633
 634 =head1 NAME
 635
 636 bulkmarcimport.pl - Import bibliographic/authority records into Koha
 637
 638 =head1 USAGE
 639
 640  $ export KOHA_CONF=/etc/koha.conf
 641  $ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\
 642     -file /home/jmf/koha.mrc -n 3000
 643
 644 =head1 WARNING
 645
 646 Don't use this script before you've entered and checked your MARC parameters
 647 tables twice (or more!). Otherwise, the import won't work correctly and you
 648 will get invalid data.
 649
 650 =head1 DESCRIPTION
 651
 652 =over
 653
 654 =item  B<-h>
 655
 656 This version/help screen
 657
 658 =item B<-b, -biblios>
 659
 660 Type of import: bibliographic records
 661
 662 =item B<-a, -authorities>
 663
 664 Type of import: authority records
 665
 666 =item B<-file>=I<FILE>
 667
 668 The I<FILE> to import
 669
 670 =item  B<-v>
 671
 672 Verbose mode. 1 means "some infos", 2 means "MARC dumping"
 673
 674 =item B<-fk>
 675
 676 Turn off foreign key checks during import.
 677
 678 =item B<-n>=I<NUMBER>
 679
 680 The I<NUMBER> of records to import. If missing, all the file is imported
 681
 682 =item B<-o, -offset>=I<NUMBER>
 683
 684 File offset before importing, ie I<NUMBER> of records to skip.
 685
 686 =item B<-commit>=I<NUMBER>
 687
 688 The I<NUMBER> of records to wait before performing a 'commit' operation
 689
 690 =item B<-l>
 691
 692 File logs actions done for each record and their status into file
 693
 694 =item B<-append>
 695
 696 If specified, data will be appended to the logfile. If not, the logfile will be erased for each execution.
 697
 698 =item B<-t, -test>
 699
 700 Test mode: parses the file, saying what it would do, but doing nothing.
 701
 702 =item B<-s>
 703
 704 Skip automatic conversion of MARC-8 to UTF-8.  This option is provided for
 705 debugging.
 706
 707 =item B<-c>=I<CHARACTERISTIC>
 708
 709 The I<CHARACTERISTIC> MARC flavour. At the moment, only I<MARC21> and
 710 I<UNIMARC> are supported. MARC21 by default.
 711
 712 =item B<-d>
 713
 714 Delete EVERYTHING related to biblio in koha-DB before import. Tables: biblio,
 715 biblioitems, items
 716
 717 =item B<-m>=I<FORMAT>
 718
 719 Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
 720
 721 =item B<-authtypes>
 722
 723 file yamlfile with authoritiesTypes and distinguishable record field in order
 724 to store the correct authtype
 725
 726 =item B<-yaml>
 727
 728 yaml file  format a yaml file with ids
 729
 730 =item B<-filter>
 731
 732 list of fields that will not be imported. Can be any from 000 to 999 or field,
 733 subfield and subfield's matching value such as 200avalue
 734
 735 =item B<-insert>
 736
 737 if set, only insert when possible
 738
 739 =item B<-update>
 740
 741 if set, only updates (any biblio should have a matching record)
 742
 743 =item B<-all>
 744
 745 if set, do whatever is required
 746
 747 =item B<-k, -keepids>=<FIELD>
 748
 749 Field store ids in I<FIELD> (useful for authorities, where 001 contains the
 750 authid for Koha, that can contain a very valuable info for authorities coming
 751 from LOC or BNF. useless for biblios probably)
 752
 753 =item B<-match>=<FIELD>
 754
 755 I<FIELD> matchindex,fieldtomatch matchpoint to use to deduplicate fieldtomatch
 756 can be either 001 to 999 or field and list of subfields as such 100abcde
 757
 758 =item B<-i,-isbn>
 759
 760 If set, a search will be done on isbn, and, if the same isbn is found, the
 761 biblio is not added. It's another method to deduplicate.  B<-match> & B<-isbn>
 762 can be both set.
 763
 764 =item B<-cleanisbn>
 765
 766 Clean ISBN fields from entering biblio records, ie removes hyphens. By default,
 767 ISBN are cleaned. --nocleanisbn will keep ISBN unchanged.
 768
 769 =item B<-x>=I<TAG>
 770
 771 Source bib I<TAG> for reporting the source bib number
 772
 773 =item B<-y>=I<SUBFIELD>
 774
 775 Source I<SUBFIELD> for reporting the source bib number
 776
 777 =item B<-idmap>=I<FILE>
 778
 779 I<FILE> for the koha bib and source id
 780
 781 =item B<-keepids>
 782
 783 Store ids in 009 (useful for authorities, where 001 contains the authid for
 784 Koha, that can contain a very valuable info for authorities coming from LOC or
 785 BNF. useless for biblios probably)
 786
 787 =item B<-dedupbarcode>
 788
 789 If set, whenever a duplicate barcode is detected, it is removed and the attempt
 790 to add the record is retried, thereby giving the record a blank barcode. This
 791 is useful when something has set barcodes to be a biblio ID, or similar
 792 (usually other software.)
 793
 794 =item B<-framework>
 795
 796 This is the code for the framework that the requested records will have attached
 797 to them when they are created. If not specified, then the default framework
 798 will be used.
 799
 800 =item B<-custom>=I<MODULE>
 801
 802 This parameter allows you to use a local module with a customize subroutine
 803 that is called for each MARC record.
 804 If no filename is passed, LocalChanges.pm is assumed to be in the
 805 migration_tools subdirectory. You may pass an absolute file name or a file name
 806 from the migration_tools directory.
 807
 808 =back
 809
 810 =cut
 811