misc/migration_tools/bulkmarcimport.pl

   1 #!/usr/bin/perl
   2 # Import an iso2709 file into Koha 3
   3
   4 use Modern::Perl;
   5 #use diagnostics;
   6 BEGIN {
   7     # find Koha's Perl modules
   8     # test carefully before changing this
   9     use FindBin;
  10     eval { require "$FindBin::Bin/../kohalib.pl" };
  11 }
  12
  13 # Koha modules used
  14 use MARC::File::USMARC;
  15 use MARC::File::XML;
  16 use MARC::Record;
  17 use MARC::Batch;
  18 use MARC::Charset;
  19
  20 use Koha::Script;
  21 use C4::Context;
  22 use C4::Biblio;
  23 use C4::Koha;
  24 use C4::Debug;
  25 use C4::Charset;
  26 use C4::Items;
  27 use C4::MarcModificationTemplates;
  28
  29 use YAML;
  30 use Unicode::Normalize;
  31 use Time::HiRes qw(gettimeofday);
  32 use Getopt::Long;
  33 use IO::File;
  34 use Pod::Usage;
  35
  36 use Koha::Biblios;
  37 use Koha::SearchEngine;
  38 use Koha::SearchEngine::Search;
  39
  40 use open qw( :std :encoding(UTF-8) );
  41 binmode( STDOUT, ":encoding(UTF-8)" );
  42 my ( $input_marc_file, $number, $offset) = ('',0,0);
  43 my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
  44 my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
  45 my $cleanisbn = 1;
  46 my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
  47 my $framework = '';
  48 my $localcust;
  49 my $marc_mod_template = '';
  50 my $marc_mod_template_id = -1;
  51
  52 $|=1;
  53
  54 GetOptions(
  55     'commit:f'    => \$commit,
  56     'file:s'    => \$input_marc_file,
  57     'n:f' => \$number,
  58     'o|offset:f' => \$offset,
  59     'h' => \$version,
  60     'd' => \$delete,
  61     't|test' => \$test_parameter,
  62     's' => \$skip_marc8_conversion,
  63     'c:s' => \$char_encoding,
  64     'v:+' => \$verbose,
  65     'fk' => \$fk_off,
  66     'm:s' => \$format,
  67     'l:s' => \$logfile,
  68     'append' => \$append,
  69     'k|keepids:s' => \$keepids,
  70     'b|biblios' => \$biblios,
  71     'a|authorities' => \$authorities,
  72     'authtypes:s' => \$authtypes,
  73     'filter=s@'     => \$filters,
  74     'insert'        => \$insert,
  75     'update'        => \$update,
  76     'all'           => \$all,
  77     'match=s@'    => \$match,
  78     'i|isbn' => \$isbn_check,
  79     'x:s' => \$sourcetag,
  80     'y:s' => \$sourcesubfield,
  81     'idmap:s' => \$idmapfl,
  82     'cleanisbn!'     => \$cleanisbn,
  83     'yaml:s'        => \$yamlfile,
  84     'dedupbarcode' => \$dedup_barcode,
  85     'framework=s' => \$framework,
  86     'custom:s'    => \$localcust,
  87     'marcmodtemplate:s' => \$marc_mod_template,
  88 );
  89 $biblios ||= !$authorities;
  90 $insert  ||= !$update;
  91 my $writemode = ($append) ? "a" : "w";
  92
  93 pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) if $biblios && $authorities;
  94
  95 if ($all) {
  96     $insert = 1;
  97     $update = 1;
  98 }
  99
 100 if ($version || ($input_marc_file eq '')) {
 101     pod2usage( -verbose => 2 );
 102     exit;
 103 }
 104 if( $update && !( $match || $isbn_check ) ) {
 105     warn "Using -update without -match or -isbn seems to be useless.\n";
 106 }
 107
 108 if(defined $localcust) { #local customize module
 109     if(!-e $localcust) {
 110         $localcust= $localcust||'LocalChanges'; #default name
 111         $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
 112         $localcust=~ s/\.pm$//;           #remove extension
 113         my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
 114         if(-e $fqcust) {
 115             $localcust= $fqcust;
 116         }
 117         else {
 118             print "WARNING: customize module $localcust.pm not found!\n";
 119             exit 1;
 120         }
 121     }
 122     require $localcust if $localcust;
 123     $localcust=\&customize if $localcust;
 124 }
 125
 126 if($marc_mod_template ne '') {
 127     my @templates = GetModificationTemplates();
 128     foreach my $this_template (@templates) {
 129         if($this_template->{'name'} eq $marc_mod_template) {
 130             if($marc_mod_template_id < 0) {
 131                 $marc_mod_template_id = $this_template->{'template_id'};
 132             } else {
 133                 print "WARNING: MARC modification template name " .
 134                 "'$marc_mod_template' matches multiple templates. " .
 135                 "Please rename these templates\n";
 136                 exit 1;
 137             }
 138         }
 139     }
 140     if($marc_mod_template_id < 0) {
 141         die "Can't located MARC modification template '$marc_mod_template'\n";
 142     } else {
 143         print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose;
 144     }
 145 }
 146
 147 my $dbh = C4::Context->dbh;
 148 my $heading_fields=get_heading_fields();
 149
 150 my $idmapfh;
 151 if (defined $idmapfl) {
 152   open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
 153 }
 154
 155 if ((not defined $sourcesubfield) && (not defined $sourcetag)){
 156   $sourcetag="910";
 157   $sourcesubfield="a";
 158 }
 159
 160
 161 # Disable logging for the biblios and authorities import operation. It would unnecessarily
 162 # slow the import
 163 $ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
 164 $ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
 165
 166 if ($fk_off) {
 167         $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
 168 }
 169
 170
 171 if ($delete) {
 172         if ($biblios){
 173         print "deleting biblios\n";
 174         $dbh->do("DELETE FROM biblio");
 175         $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
 176         $dbh->do("DELETE FROM biblioitems");
 177         $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
 178         $dbh->do("DELETE FROM items");
 179         $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
 180         }
 181         else {
 182         print "deleting authorities\n";
 183         $dbh->do("truncate auth_header");
 184         }
 185     $dbh->do("truncate zebraqueue");
 186 }
 187
 188
 189
 190 if ($test_parameter) {
 191     print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
 192 }
 193
 194 my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
 195
 196 # The definition of $searcher must be before MARC::Batch->new
 197 my $searcher = Koha::SearchEngine::Search->new(
 198     {
 199         index => (
 200               $authorities
 201             ? $Koha::SearchEngine::AUTHORITIES_INDEX
 202             : $Koha::SearchEngine::BIBLIOS_INDEX
 203         )
 204     }
 205 );
 206
 207 print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
 208 my $starttime = gettimeofday;
 209 my $batch;
 210 my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
 211 if (defined $format && $format =~ /XML/i) {
 212     # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
 213     # appears to try to convert incoming XML records from MARC-8
 214     # to UTF-8.  Setting the BinaryEncoding key turns that off
 215     # TODO: see what happens to ISO-8859-1 XML files.
 216     # TODO: determine if MARC::Batch can be fixed to handle
 217     #       XML records properly -- it probably should be
 218     #       be using a proper push or pull XML parser to
 219     #       extract the records, not using regexes to look
 220     #       for <record>.*</record>.
 221     $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
 222     my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
 223 #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
 224     $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
 225     $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
 226     $batch = MARC::Batch->new( 'XML', $fh );
 227 } else {
 228     $batch = MARC::Batch->new( 'USMARC', $fh );
 229 }
 230 $batch->warnings_off();
 231 $batch->strict_off();
 232 my $i=0;
 233 my $commitnum = $commit ? $commit : 50;
 234 my $yamlhash;
 235
 236 # Skip file offset
 237 if ( $offset ) {
 238     print "Skipping file offset: $offset records\n";
 239     $batch->next() while ($offset--);
 240 }
 241
 242 my ($tagid,$subfieldid);
 243 if ($authorities){
 244           $tagid='001';
 245 }
 246 else {
 247    ( $tagid, $subfieldid ) =
 248             GetMarcFromKohaField( "biblio.biblionumber" );
 249         $tagid||="001";
 250 }
 251
 252 # the SQL query to search on isbn
 253 my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
 254
 255 $dbh->{AutoCommit} = 0;
 256 my $loghandle;
 257 if ($logfile){
 258    $loghandle= IO::File->new($logfile, $writemode) ;
 259    print $loghandle "id;operation;status\n";
 260 }
 261
 262 RECORD: while (  ) {
 263     my $record;
 264     # get records
 265     eval { $record = $batch->next() };
 266     if ( $@ ) {
 267         print "Bad MARC record $i: $@ skipped\n";
 268         # FIXME - because MARC::Batch->next() combines grabbing the next
 269         # blob and parsing it into one operation, a correctable condition
 270         # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
 271         # from because we don't have access to the original blob.  Note
 272         # that the staging import can deal with this condition (via
 273         # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
 274         next;
 275     }
 276     # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
 277     last unless ( $record );
 278     $i++;
 279     if( ($verbose//1)==1 ) { #no dot for verbose==2
 280         print "." . ( $i % 100==0 ? "\n$i" : '' );
 281     }
 282
 283     # transcode the record to UTF8 if needed & applicable.
 284     if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
 285         # FIXME update condition
 286         my ($guessed_charset, $charset_errors);
 287          ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
 288         if ($guessed_charset eq 'failed') {
 289             warn "ERROR: failed to perform character conversion for record $i\n";
 290             next RECORD;
 291         }
 292     }
 293     SetUTF8Flag($record);
 294     if($marc_mod_template_id > 0) {
 295     print "Modifying MARC\n" if $verbose;
 296     ModifyRecordWithTemplate( $marc_mod_template_id, $record );
 297     }
 298     &$localcust($record) if $localcust;
 299     my $isbn;
 300     # remove trailing - in isbn (only for biblios, of course)
 301     if( $biblios ) {
 302         my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
 303         my $field = $record->field($tag);
 304         $isbn = $field && $field->subfield('a');
 305         if ( $isbn && $cleanisbn ) {
 306             $isbn =~ s/-//g;
 307             $field->update('a' => $isbn);
 308         }
 309     }
 310     my $id;
 311     # search for duplicates (based on Local-number)
 312     my $originalid;
 313     $originalid = GetRecordId( $record, $tagid, $subfieldid );
 314     if ($match) {
 315         require C4::Search;
 316         my $query = build_query( $match, $record );
 317         my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
 318         $debug && warn $query;
 319         my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
 320         # changed to warn so able to continue with one broken record
 321         if ( defined $error ) {
 322             warn "unable to search the database for duplicates : $error";
 323             printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
 324             next RECORD;
 325         }
 326         $debug && warn "$query $server : $totalhits";
 327         if ( $results && scalar(@$results) == 1 ) {
 328             my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
 329             SetUTF8Flag($marcrecord);
 330             $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
 331             if ( $authorities && $marcFlavour ) {
 332                 #Skip if authority in database is the same as the on in database
 333                 if ( $marcrecord->field('005') && $record->field('005') &&
 334                      $marcrecord->field('005')->data && $record->field('005')->data &&
 335                      $marcrecord->field('005')->data >= $record->field('005')->data ) {
 336                     if ($yamlfile) {
 337                         $yamlhash->{$originalid}->{'authid'} = $id;
 338
 339                         # we recover all subfields of the heading authorities
 340                         my @subfields;
 341                         foreach my $field ( $marcrecord->field("2..") ) {
 342                             push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
 343                         }
 344                         $yamlhash->{$originalid}->{'subfields'} = \@subfields;
 345                         $yamlhash->{$originalid}->{'updated'} = 0;
 346                     }
 347                     next;
 348                 }
 349             }
 350         } elsif ( $results && scalar(@$results) > 1 ) {
 351             $debug && warn "more than one match for $query";
 352         } else {
 353             $debug && warn "nomatch for $query";
 354         }
 355     }
 356     if ($keepids && $originalid) {
 357             my $storeidfield;
 358             if ( length($keepids) == 3 ) {
 359                 $storeidfield = MARC::Field->new( $keepids, $originalid );
 360             } else {
 361                 $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
 362             }
 363             $record->insert_fields_ordered($storeidfield);
 364             $record->delete_field( $record->field($tagid) );
 365     }
 366     foreach my $stringfilter (@$filters) {
 367         if ( length($stringfilter) == 3 ) {
 368             foreach my $field ( $record->field($stringfilter) ) {
 369                 $record->delete_field($field);
 370                 $debug && warn "removed : ", $field->as_string;
 371             }
 372         } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
 373             my $removetag = $1;
 374             my $removesubfield = $2;
 375             my $removematch = $3;
 376             if ( ( $removetag > "010" ) && $removesubfield ) {
 377                 foreach my $field ( $record->field($removetag) ) {
 378                     $field->delete_subfield( code => "$removesubfield", match => $removematch );
 379                     $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
 380                 }
 381             }
 382         }
 383     }
 384     unless ($test_parameter) {
 385         if ($authorities){
 386             use C4::AuthoritiesMarc;
 387             my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
 388             my $authid= ($id?$id:GuessAuthId($record));
 389             if ($authid && GetAuthority($authid) && $update ){
 390             ## Authority has an id and is in database : Replace
 391                 eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
 392                 if ($@){
 393                     warn "Problem with authority $authid Cannot Modify";
 394                                         printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
 395                 }
 396                                 else{
 397                                         printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
 398                                 }
 399             }
 400                 else {
 401             ## True insert in database
 402                 eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
 403                 if ($@){
 404                     warn "Problem with authority $authid Cannot Add".$@;
 405                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
 406                 }
 407                                 else{
 408                                         printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
 409                                 }
 410                 }
 411             if ($yamlfile) {
 412             $yamlhash->{$originalid}->{'authid'} = $authid;
 413             my @subfields;
 414             foreach my $field ( $record->field("2..") ) {
 415                 push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
 416             }
 417             $yamlhash->{$originalid}->{'subfields'} = \@subfields;
 418             $yamlhash->{$originalid}->{'updated'} = 1;
 419             }
 420         }
 421         else {
 422             my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
 423             $biblionumber = $id;
 424             # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
 425             if (!$biblionumber && $isbn_check && $isbn) {
 426     #         warn "search ISBN : $isbn";
 427                 $sth_isbn->execute($isbn);
 428                 ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
 429             }
 430                 if (defined $idmapfl) {
 431                                 if ($sourcetag < "010"){
 432                                         if ($record->field($sourcetag)){
 433                                           my $source = $record->field($sourcetag)->data();
 434                       printf($idmapfh "%s|%s\n",$source,$biblionumber);
 435                                         }
 436                             } else {
 437                                         my $source=$record->subfield($sourcetag,$sourcesubfield);
 438                     printf($idmapfh "%s|%s\n",$source,$biblionumber);
 439                           }
 440                         }
 441                                         # create biblio, unless we already have it ( either match or isbn )
 442             if ($biblionumber) {
 443                 eval{
 444                     $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
 445                 };
 446                 if ($update) {
 447                     eval { ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber) ) };
 448                     if ($@) {
 449                         warn "ERROR: Edit biblio $biblionumber failed: $@\n";
 450                         printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
 451                         next RECORD;
 452                     } else {
 453                         printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
 454                     }
 455                 } else {
 456                     printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
 457                 }
 458             } else {
 459                 if ($insert) {
 460                     eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
 461                     if ($@) {
 462                         warn "ERROR: Adding biblio $biblionumber failed: $@\n";
 463                         printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
 464                         next RECORD;
 465                     } else {
 466                         printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
 467                     }
 468                 } else {
 469                     warn "WARNING: Updating record ".($id||$originalid)." failed";
 470                     printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
 471                     next RECORD;
 472                 }
 473             }
 474             eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
 475             my $error_adding = $@;
 476             # Work on a clone so that if there are real errors, we can maybe
 477             # fix them up later.
 478                         my $clone_record = $record->clone();
 479             C4::Biblio::_strip_item_fields($clone_record, '');
 480             # This sets the marc fields if there was an error, and also calls
 481             # defer_marc_save.
 482             ModBiblioMarc( $clone_record, $biblionumber, $framework );
 483             if ( $error_adding ) {
 484                 warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
 485                                 printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
 486                 # if we failed because of an exception, assume that
 487                 # the MARC columns in biblioitems were not set.
 488                 next RECORD;
 489             }
 490                         else{
 491                                 printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
 492                         }
 493             if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
 494                 # Find the record called 'barcode'
 495                 my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
 496                 # Now remove any items that didn't have a duplicate_barcode error,
 497                 # erase the barcodes on items that did, and re-add those items.
 498                 my %dupes;
 499                 foreach my $i (0 .. $#{$errors_ref}) {
 500                     my $ref = $errors_ref->[$i];
 501                     if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
 502                         $dupes{$ref->{item_sequence}} = 1;
 503                         # Delete the error message because we're going to
 504                         # retry this one.
 505                         delete $errors_ref->[$i];
 506                     }
 507                 }
 508                 my $seq = 0;
 509                 foreach my $field ($record->field($tag)) {
 510                     $seq++;
 511                     if ($dupes{$seq}) {
 512                         # Here we remove the barcode
 513                         $field->delete_subfield(code => $sub);
 514                     } else {
 515                         # otherwise we delete the field because we don't want
 516                         # two of them
 517                         $record->delete_fields($field);
 518                     }
 519                 }
 520                 # Now re-add the record as before, adding errors to the prev list
 521                 my $more_errors;
 522                 eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
 523                 if ( $@ ) {
 524                     warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
 525                     printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
 526                     # if we failed because of an exception, assume that
 527                     # the MARC columns in biblioitems were not set.
 528                     ModBiblioMarc( $record, $biblionumber, $framework );
 529                     next RECORD;
 530                 } else {
 531                     printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
 532                 }
 533                 push @$errors_ref, @{ $more_errors };
 534             }
 535             if ($#{ $errors_ref } > -1) {
 536                 report_item_errors($biblionumber, $errors_ref);
 537             }
 538             $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
 539         }
 540         $dbh->commit() if (0 == $i % $commitnum);
 541     }
 542     print $record->as_formatted()."\n" if ($verbose//0)==2;
 543     last if $i == $number;
 544 }
 545 $dbh->commit();
 546 $dbh->{AutoCommit} = 1;
 547
 548
 549 if ($fk_off) {
 550         $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
 551 }
 552
 553 # Restore CataloguingLog and AuthoritiesLog
 554 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog};
 555 delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
 556
 557 my $timeneeded = gettimeofday - $starttime;
 558 print "\n$i MARC records done in $timeneeded seconds\n";
 559 if ($logfile){
 560   print $loghandle "file : $input_marc_file\n";
 561   print $loghandle "$i MARC records done in $timeneeded seconds\n";
 562   $loghandle->close;
 563 }
 564 if ($yamlfile) {
 565     open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
 566     print $yamlfileout Dump($yamlhash);
 567 }
 568 exit 0;
 569
 570 sub GetRecordId{
 571         my $marcrecord=shift;
 572         my $tag=shift;
 573         my $subfield=shift;
 574         my $id;
 575         if ($tag lt "010"){
 576                 return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
 577         }
 578         elsif ($subfield){
 579                 if ($marcrecord->field($tag)){
 580                         return $marcrecord->subfield($tag,$subfield);
 581                 }
 582         }
 583         return $id;
 584 }
 585 sub build_query {
 586         my $match = shift;
 587         my $record=shift;
 588         my @searchstrings;
 589         foreach my $matchingpoint (@$match){
 590           my $string = build_simplequery($matchingpoint,$record);
 591           push @searchstrings,$string if (length($string)>0);
 592         }
 593     my $op = 'and';
 594     return join(" $op ",@searchstrings);
 595 }
 596 sub build_simplequery {
 597         my $element=shift;
 598         my $record=shift;
 599     my @searchstrings;
 600     my ($index,$recorddata)=split /,/,$element;
 601     if ($recorddata=~/(\d{3})(.*)/) {
 602         my ($tag,$subfields) =($1,$2);
 603         foreach my $field ($record->field($tag)){
 604                   if (length($field->as_string("$subfields"))>0){
 605               push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
 606                   }
 607         }
 608     }
 609     my $op = 'and';
 610     return join(" $op ",@searchstrings);
 611 }
 612 sub report_item_errors {
 613     my $biblionumber = shift;
 614     my $errors_ref = shift;
 615
 616     foreach my $error (@{ $errors_ref }) {
 617         next if !$error;
 618         my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
 619         my $error_code = $error->{'error_code'};
 620         $error_code =~ s/_/ /g;
 621         $msg .= "$error_code $error->{'error_information'}";
 622         print $msg, "\n";
 623     }
 624 }
 625 sub printlog{
 626         my $logelements=shift;
 627     print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
 628 }
 629 sub get_heading_fields{
 630     my $headingfields;
 631     if ($authtypes){
 632         $headingfields=YAML::LoadFile($authtypes);
 633         $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
 634         $debug && warn YAML::Dump($headingfields);
 635     }
 636     unless ($headingfields){
 637         $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
 638         $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
 639     }
 640     return $headingfields;
 641 }
 642
 643 =head1 NAME
 644
 645 bulkmarcimport.pl - Import bibliographic/authority records into Koha
 646
 647 =head1 USAGE
 648
 649  $ export KOHA_CONF=/etc/koha.conf
 650  $ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\
 651     -file /home/jmf/koha.mrc -n 3000
 652
 653 =head1 WARNING
 654
 655 Don't use this script before you've entered and checked your MARC parameters
 656 tables twice (or more!). Otherwise, the import won't work correctly and you
 657 will get invalid data.
 658
 659 =head1 DESCRIPTION
 660
 661 =over
 662
 663 =item  B<-h>
 664
 665 This version/help screen
 666
 667 =item B<-b, -biblios>
 668
 669 Type of import: bibliographic records
 670
 671 =item B<-a, -authorities>
 672
 673 Type of import: authority records
 674
 675 =item B<-file>=I<FILE>
 676
 677 The I<FILE> to import
 678
 679 =item  B<-v>
 680
 681 Verbose mode. 1 means "some infos", 2 means "MARC dumping"
 682
 683 =item B<-fk>
 684
 685 Turn off foreign key checks during import.
 686
 687 =item B<-n>=I<NUMBER>
 688
 689 The I<NUMBER> of records to import. If missing, all the file is imported
 690
 691 =item B<-o, -offset>=I<NUMBER>
 692
 693 File offset before importing, ie I<NUMBER> of records to skip.
 694
 695 =item B<-commit>=I<NUMBER>
 696
 697 The I<NUMBER> of records to wait before performing a 'commit' operation
 698
 699 =item B<-l>
 700
 701 File logs actions done for each record and their status into file
 702
 703 =item B<-append>
 704
 705 If specified, data will be appended to the logfile. If not, the logfile will be erased for each execution.
 706
 707 =item B<-t, -test>
 708
 709 Test mode: parses the file, saying what it would do, but doing nothing.
 710
 711 =item B<-s>
 712
 713 Skip automatic conversion of MARC-8 to UTF-8.  This option is provided for
 714 debugging.
 715
 716 =item B<-c>=I<CHARACTERISTIC>
 717
 718 The I<CHARACTERISTIC> MARC flavour. At the moment, only I<MARC21> and
 719 I<UNIMARC> are supported. MARC21 by default.
 720
 721 =item B<-d>
 722
 723 Delete EVERYTHING related to biblio in koha-DB before import. Tables: biblio,
 724 biblioitems, items
 725
 726 =item B<-m>=I<FORMAT>
 727
 728 Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
 729
 730 =item B<-authtypes>
 731
 732 file yamlfile with authoritiesTypes and distinguishable record field in order
 733 to store the correct authtype
 734
 735 =item B<-yaml>
 736
 737 yaml file  format a yaml file with ids
 738
 739 =item B<-filter>
 740
 741 list of fields that will not be imported. Can be any from 000 to 999 or field,
 742 subfield and subfield's matching value such as 200avalue
 743
 744 =item B<-insert>
 745
 746 if set, only insert when possible
 747
 748 =item B<-update>
 749
 750 if set, only updates (any biblio should have a matching record)
 751
 752 =item B<-all>
 753
 754 if set, do whatever is required
 755
 756 =item B<-k, -keepids>=<FIELD>
 757
 758 Field store ids in I<FIELD> (useful for authorities, where 001 contains the
 759 authid for Koha, that can contain a very valuable info for authorities coming
 760 from LOC or BNF. useless for biblios probably)
 761
 762 =item B<-match>=<FIELD>
 763
 764 I<FIELD> matchindex,fieldtomatch matchpoint to use to deduplicate fieldtomatch
 765 can be either 001 to 999 or field and list of subfields as such 100abcde
 766
 767 =item B<-i,-isbn>
 768
 769 If set, a search will be done on isbn, and, if the same isbn is found, the
 770 biblio is not added. It's another method to deduplicate.  B<-match> & B<-isbn>
 771 can be both set.
 772
 773 =item B<-cleanisbn>
 774
 775 Clean ISBN fields from entering biblio records, ie removes hyphens. By default,
 776 ISBN are cleaned. --nocleanisbn will keep ISBN unchanged.
 777
 778 =item B<-x>=I<TAG>
 779
 780 Source bib I<TAG> for reporting the source bib number
 781
 782 =item B<-y>=I<SUBFIELD>
 783
 784 Source I<SUBFIELD> for reporting the source bib number
 785
 786 =item B<-idmap>=I<FILE>
 787
 788 I<FILE> for the koha bib and source id
 789
 790 =item B<-keepids>
 791
 792 Store ids in 009 (useful for authorities, where 001 contains the authid for
 793 Koha, that can contain a very valuable info for authorities coming from LOC or
 794 BNF. useless for biblios probably)
 795
 796 =item B<-dedupbarcode>
 797
 798 If set, whenever a duplicate barcode is detected, it is removed and the attempt
 799 to add the record is retried, thereby giving the record a blank barcode. This
 800 is useful when something has set barcodes to be a biblio ID, or similar
 801 (usually other software.)
 802
 803 =item B<-framework>
 804
 805 This is the code for the framework that the requested records will have attached
 806 to them when they are created. If not specified, then the default framework
 807 will be used.
 808
 809 =item B<-custom>=I<MODULE>
 810
 811 This parameter allows you to use a local module with a customize subroutine
 812 that is called for each MARC record.
 813 If no filename is passed, LocalChanges.pm is assumed to be in the
 814 migration_tools subdirectory. You may pass an absolute file name or a file name
 815 from the migration_tools directory.
 816
 817 =item B<-marcmodtemplate>=I<TEMPLATE>
 818
 819 This parameter allows you to specify the name of an existing MARC
 820 modification template to apply as the MARC records are imported (these
 821 templates are created in the "MARC modification templates" tool in Koha).
 822 If not specified, no MARC modification templates are used (default).
 823
 824 =back
 825
 826 =cut
 827