misc/migration_tools/build_oai_sets.pl

   1 #!/usr/bin/perl
   2
   3 # Copyright 2011 BibLibre
   4 #
   5 # This file is part of Koha.
   6 #
   7 # Koha is free software; you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # Koha is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  19
  20 =head1 DESCRIPTION
  21
  22 This script build OAI-PMH sets (to be used by opac/oai.pl) according to sets
  23 and mappings defined in Koha. It reads informations from oai_sets and
  24 oai_sets_mappings, and then fill table oai_sets_biblios with builded infos.
  25
  26 =head1 USAGE
  27
  28     build_oai_sets.pl [-h] [-v] [-r] [-i] [-l LENGTH [-o OFFSET]]
  29         -h          Print help message;
  30         -v          Be verbose
  31         -r          Truncate table oai_sets_biblios before inserting new rows
  32         -i          Embed items informations, mandatory if you defined mappings
  33                     on item fields
  34         -l LENGTH   Process LENGTH biblios
  35         -o OFFSET   If LENGTH is defined, start processing from OFFSET
  36
  37 =cut
  38
  39 use Modern::Perl;
  40 use MARC::Record;
  41 use MARC::File::XML;
  42 use List::MoreUtils qw/uniq/;
  43 use Getopt::Std;
  44
  45 use C4::Context;
  46 use C4::Charset qw/StripNonXmlChars/;
  47 use C4::Biblio;
  48 use C4::OAI::Sets;
  49
  50 my %opts;
  51 $Getopt::Std::STANDARD_HELP_VERSION = 1;
  52 my $go = getopts('vo:l:ihr', \%opts);
  53
  54 if(!$go or $opts{h}){
  55     &print_usage;
  56     exit;
  57 }
  58
  59 my $verbose = $opts{v};
  60 my $offset = $opts{o};
  61 my $length = $opts{l};
  62 my $embed_items = $opts{i};
  63 my $reset = $opts{r};
  64
  65 my $dbh = C4::Context->dbh;
  66
  67 # Get OAI sets mappings
  68 my $mappings = GetOAISetsMappings;
  69
  70 # Get all biblionumbers and marcxml
  71 print "Retrieving biblios... " if $verbose;
  72 my $query = qq{
  73     SELECT biblionumber, metadata
  74     FROM biblio_metadata
  75     WHERE format='marcxml'
  76     AND marcflavour = ?
  77 };
  78 if($length) {
  79     $query .= "LIMIT $length";
  80     if($offset) {
  81         $query .= " OFFSET $offset";
  82     }
  83 }
  84 my $sth = $dbh->prepare($query);
  85 $sth->execute( C4::Context->preference('marcflavour') );
  86 my $results = $sth->fetchall_arrayref({});
  87 print "done.\n" if $verbose;
  88
  89 # Build lists of parents sets
  90 my $sets = GetOAISets;
  91 my $parentsets;
  92 foreach my $set (@$sets) {
  93     my $setSpec = $set->{'spec'};
  94     while($setSpec =~ /^(.+):(.+)$/) {
  95         my $parent = $1;
  96         my $parent_set = GetOAISetBySpec($parent);
  97         if($parent_set) {
  98             push @{ $parentsets->{$set->{'id'}} }, $parent_set->{'id'};
  99             $setSpec = $parent;
 100         } else {
 101             last;
 102         }
 103     }
 104 }
 105
 106 my $num_biblios = scalar @$results;
 107 my $i = 1;
 108 my $sets_biblios = {};
 109 foreach my $res (@$results) {
 110     my $biblionumber = $res->{'biblionumber'};
 111     my $marcxml = $res->{'metadata'};
 112     if($verbose and $i % 1000 == 0) {
 113         my $percent = ($i * 100) / $num_biblios;
 114         $percent = sprintf("%.2f", $percent);
 115         say "Progression: $i/$num_biblios ($percent %)";
 116     }
 117     # The following lines are copied from GetMarcBiblio
 118     # We don't call GetMarcBiblio to avoid a sql query to be executed each time
 119     $marcxml = StripNonXmlChars($marcxml);
 120     MARC::File::XML->default_record_format(C4::Context->preference('marcflavour'));
 121     my $record;
 122     eval {
 123         $record = MARC::Record::new_from_xml($marcxml, "utf8", C4::Context->preference('marcflavour'));
 124     };
 125     if($@) {
 126         warn "(biblio $biblionumber) Error while creating record from marcxml: $@";
 127         next;
 128     }
 129     if($embed_items) {
 130         C4::Biblio::EmbedItemsInMarcBiblio($record, $biblionumber);
 131     }
 132
 133     my @biblio_sets = CalcOAISetsBiblio($record, $mappings);
 134     foreach my $set_id (@biblio_sets) {
 135         push @{ $sets_biblios->{$set_id} }, $biblionumber;
 136         foreach my $parent_set_id ( @{ $parentsets->{$set_id} } ) {
 137             push @{ $sets_biblios->{$parent_set_id} }, $biblionumber;
 138         }
 139     }
 140     $i++;
 141 }
 142 say "Progression: done." if $verbose;
 143
 144 say "Summary:";
 145 foreach my $set_id (keys %$sets_biblios) {
 146     $sets_biblios->{$set_id} = [ uniq @{ $sets_biblios->{$set_id} } ];
 147     my $set = GetOAISet($set_id);
 148     my $setSpec = $set->{'spec'};
 149     say "Set '$setSpec': ". scalar(@{$sets_biblios->{$set_id}}) ." biblios";
 150 }
 151
 152 print "Updating database... ";
 153 if($reset) {
 154     ModOAISetsBiblios( {} );
 155 }
 156 AddOAISetsBiblios($sets_biblios);
 157 print "done.\n";
 158
 159 sub print_usage {
 160     print "build_oai_sets.pl: Build OAI-PMH sets, according to mappings defined in Koha\n";
 161     print "Usage: build_oai_sets.pl [-h] [-v] [-i] [-l LENGTH [-o OFFSET]]\n\n";
 162     print "\t-h\t\tPrint this help and exit\n";
 163     print "\t-v\t\tBe verbose\n";
 164     print "\t-i\t\tEmbed items informations, mandatory if you defined mappings on item fields\n";
 165     print "\t-l LENGTH\tProcess LENGTH biblios\n";
 166     print "\t-o OFFSET\tIf LENGTH is defined, start processing from OFFSET\n\n";
 167 }