Koha/SearchEngine/Elasticsearch.pm

   1 package Koha::SearchEngine::Elasticsearch;
   2
   3 # Copyright 2015 Catalyst IT
   4 #
   5 # This file is part of Koha.
   6 #
   7 # Koha is free software; you can redistribute it and/or modify it under the
   8 # terms of the GNU General Public License as published by the Free Software
   9 # Foundation; either version 3 of the License, or (at your option) any later
  10 # version.
  11 #
  12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
  13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  14 # A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License along
  17 # with Koha; if not, write to the Free Software Foundation, Inc.,
  18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19
  20 use base qw(Class::Accessor);
  21
  22 use C4::Context;
  23
  24 use Koha::Database;
  25 use Koha::SearchFields;
  26 use Koha::SearchMarcMaps;
  27
  28 use Carp;
  29 use JSON;
  30 use Modern::Perl;
  31 use Readonly;
  32 use YAML::Syck;
  33
  34 use Data::Dumper;    # TODO remove
  35
  36 __PACKAGE__->mk_ro_accessors(qw( index ));
  37 __PACKAGE__->mk_accessors(qw( sort_fields ));
  38
  39 # Constants to refer to the standard index names
  40 Readonly our $BIBLIOS_INDEX     => 'biblios';
  41 Readonly our $AUTHORITIES_INDEX => 'authorities';
  42
  43 =head1 NAME
  44
  45 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
  46
  47 =head1 ACCESSORS
  48
  49 =over 4
  50
  51 =item index
  52
  53 The name of the index to use, generally 'biblios' or 'authorities'.
  54
  55 =back
  56
  57 =head1 FUNCTIONS
  58
  59 =cut
  60
  61 sub new {
  62     my $class = shift @_;
  63     my $self = $class->SUPER::new(@_);
  64     # Check for a valid index
  65     croak('No index name provided') unless $self->index;
  66     return $self;
  67 }
  68
  69 =head2 get_elasticsearch_params
  70
  71     my $params = $self->get_elasticsearch_params();
  72
  73 This provides a hashref that contains the parameters for connecting to the
  74 ElasicSearch servers, in the form:
  75
  76     {
  77         'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
  78         'index_name' => 'koha_instance_index',
  79     }
  80
  81 This is configured by the following in the C<config> block in koha-conf.xml:
  82
  83     <elasticsearch>
  84         <server>127.0.0.1:9200</server>
  85         <server>anotherserver:9200</server>
  86         <index_name>koha_instance</index_name>
  87     </elasticsearch>
  88
  89 =cut
  90
  91 sub get_elasticsearch_params {
  92     my ($self) = @_;
  93
  94     # Copy the hash so that we're not modifying the original
  95     my $conf = C4::Context->config('elasticsearch');
  96     die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
  97     my $es = { %{ $conf } };
  98
  99     # Helpfully, the multiple server lines end up in an array for us anyway
 100     # if there are multiple ones, but not if there's only one.
 101     my $server = $es->{server};
 102     delete $es->{server};
 103     if ( ref($server) eq 'ARRAY' ) {
 104
 105         # store it called 'nodes' (which is used by newer Search::Elasticsearch)
 106         $es->{nodes} = $server;
 107     }
 108     elsif ($server) {
 109         $es->{nodes} = [$server];
 110     }
 111     else {
 112         die "No elasticsearch servers were specified in koha-conf.xml.\n";
 113     }
 114     die "No elasticserver index_name was specified in koha-conf.xml.\n"
 115       if ( !$es->{index_name} );
 116     # Append the name of this particular index to our namespace
 117     $es->{index_name} .= '_' . $self->index;
 118
 119     $es->{key_prefix} = 'es_';
 120     return $es;
 121 }
 122
 123 =head2 get_elasticsearch_settings
 124
 125     my $settings = $self->get_elasticsearch_settings();
 126
 127 This provides the settings provided to elasticsearch when an index is created.
 128 These can do things like define tokenisation methods.
 129
 130 A hashref containing the settings is returned.
 131
 132 =cut
 133
 134 sub get_elasticsearch_settings {
 135     my ($self) = @_;
 136
 137     # Ultimately this should come from a file or something, and not be
 138     # hardcoded.
 139     my $settings = {
 140         index => {
 141             analysis => {
 142                 analyzer => {
 143                     analyser_phrase => {
 144                         tokenizer => 'keyword',
 145                         filter    => ['lowercase'],
 146                     },
 147                     analyser_standard => {
 148                         tokenizer => 'standard',
 149                         filter    => ['lowercase'],
 150                     },
 151                     default => {
 152                         tokenizer => 'keyword',
 153                         filter    => ['lowercase'],
 154                     },
 155                 },
 156             }
 157         }
 158     };
 159     return $settings;
 160 }
 161
 162 =head2 get_elasticsearch_mappings
 163
 164     my $mappings = $self->get_elasticsearch_mappings();
 165
 166 This provides the mappings that get passed to elasticsearch when an index is
 167 created.
 168
 169 =cut
 170
 171 sub get_elasticsearch_mappings {
 172     my ($self) = @_;
 173
 174     # TODO cache in the object?
 175     my $mappings = {
 176         data => {
 177             properties => {
 178                 record => {
 179                     store          => "yes",
 180                     include_in_all => JSON::false,
 181                     type           => "string",
 182                 },
 183             }
 184         }
 185     };
 186     my %sort_fields;
 187     my $marcflavour = lc C4::Context->preference('marcflavour');
 188     $self->_foreach_mapping(
 189         sub {
 190             my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
 191             return if $marc_type ne $marcflavour;
 192
 193             # TODO if this gets any sort of complexity to it, it should
 194             # be broken out into its own function.
 195
 196             # TODO be aware of date formats, but this requires pre-parsing
 197             # as ES will simply reject anything with an invalid date.
 198             my $es_type =
 199               $type eq 'boolean'
 200               ? 'boolean'
 201               : 'string';
 202
 203             if ($es_type eq 'boolean') {
 204                 $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_boolean( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
 205                 return; #Boolean cannot have facets nor sorting nor suggestions
 206             } else {
 207                 $mappings->{data}{properties}{$name} = _elasticsearch_mapping_for_default( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
 208             }
 209
 210             if ($facet) {
 211                 $mappings->{data}{properties}{ $name . '__facet' } = {
 212                     type  => "string",
 213                     index => "not_analyzed",
 214                 };
 215             }
 216             if ($suggestible) {
 217                 $mappings->{data}{properties}{ $name . '__suggestion' } = {
 218                     type => 'completion',
 219                     analyzer => 'simple',
 220                     search_analyzer => 'simple',
 221                 };
 222             }
 223             # Sort may be true, false, or undef. Here we care if it's
 224             # anything other than undef.
 225             if (defined $sort) {
 226                 $mappings->{data}{properties}{ $name . '__sort' } = {
 227                     search_analyzer => "analyser_phrase",
 228                     analyzer  => "analyser_phrase",
 229                     type            => "string",
 230                     include_in_all  => JSON::false,
 231                     fields          => {
 232                         phrase => {
 233                             search_analyzer => "analyser_phrase",
 234                             analyzer  => "analyser_phrase",
 235                             type            => "string",
 236                         },
 237                     },
 238                 };
 239                 $sort_fields{$name} = 1;
 240             }
 241         }
 242     );
 243     $self->sort_fields(\%sort_fields);
 244     return $mappings;
 245 }
 246
 247 =head2 _elasticsearch_mapping_for_*
 248
 249 Get the ES mappings for the given data type or a special mapping case
 250
 251 Receives the same parameters from the $self->_foreach_mapping() dispatcher
 252
 253 =cut
 254
 255 sub _elasticsearch_mapping_for_boolean {
 256     my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
 257
 258     return {
 259         type            => $type,
 260         null_value      => 0,
 261     };
 262 }
 263
 264 sub _elasticsearch_mapping_for_default {
 265     my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
 266
 267     return {
 268         search_analyzer => "analyser_standard",
 269         analyzer        => "analyser_standard",
 270         type            => $type,
 271         fields          => {
 272             phrase => {
 273                 search_analyzer => "analyser_phrase",
 274                 analyzer        => "analyser_phrase",
 275                 type            => "string",
 276             },
 277             raw => {
 278                 type    => "string",
 279                 index   => "not_analyzed",
 280             }
 281         },
 282     };
 283 }
 284
 285 sub reset_elasticsearch_mappings {
 286     my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
 287     my $indexes = LoadFile( $mappings_yaml );
 288
 289     while ( my ( $index_name, $fields ) = each %$indexes ) {
 290         while ( my ( $field_name, $data ) = each %$fields ) {
 291             my $field_type = $data->{type};
 292             my $field_label = $data->{label};
 293             my $mappings = $data->{mappings};
 294             my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
 295             for my $mapping ( @$mappings ) {
 296                 my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
 297                 $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } );
 298             }
 299         }
 300     }
 301 }
 302
 303 # This overrides the accessor provided by Class::Accessor so that if
 304 # sort_fields isn't set, then it'll generate it.
 305 sub sort_fields {
 306     my $self = shift;
 307
 308     if (@_) {
 309         $self->_sort_fields_accessor(@_);
 310         return;
 311     }
 312     my $val = $self->_sort_fields_accessor();
 313     return $val if $val;
 314
 315     # This will populate the accessor as a side effect
 316     $self->get_elasticsearch_mappings();
 317     return $self->_sort_fields_accessor();
 318 }
 319
 320 # Provides the rules for data conversion.
 321 sub get_fixer_rules {
 322     my ($self) = @_;
 323
 324     my $marcflavour = lc C4::Context->preference('marcflavour');
 325     my @rules;
 326
 327     $self->_foreach_mapping(
 328         sub {
 329             my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
 330             return if $marc_type ne $marcflavour;
 331             my $options = '';
 332
 333             # There's a bug when using 'split' with something that
 334             # selects a range
 335             # The split makes everything into nested arrays, but that's not
 336             # really a big deal, ES doesn't mind.
 337             $options = '-split => 1' unless $marc_field =~ m|_/| || $type eq 'sum';
 338             push @rules, "marc_map('$marc_field','${name}', $options)";
 339             if ($facet) {
 340                 push @rules, "marc_map('$marc_field','${name}__facet', $options)";
 341             }
 342             if ($suggestible) {
 343                 push @rules,
 344                     #"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; #must not have nested data structures in .input
 345                     "marc_map('$marc_field','${name}__suggestion.input.\$append')";
 346             }
 347             if ( $type eq 'boolean' ) {
 348
 349                 # boolean gets special handling, basically if it doesn't exist,
 350                 # it's added and set to false. Otherwise we can't query it.
 351                 push @rules,
 352                   "unless exists('$name') add_field('$name', 0) end";
 353             }
 354             if ($type eq 'sum' ) {
 355                 push @rules, "sum('$name')";
 356             }
 357             # Sort is a bit special as it can be true, false, undef. For
 358             # fixer rules, we care about "true", or "undef" if there is
 359             # special handling of this field from other one. "undef" means
 360             # to do the default thing, which is make it sortable.
 361             if ($self->sort_fields()->{$name}) {
 362                 if ($sort || !defined $sort) {
 363                     push @rules, "marc_map('$marc_field','${name}__sort', $options)";
 364                 }
 365             }
 366         }
 367     );
 368
 369     push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');
 370     return \@rules;
 371 }
 372
 373 =head2 _foreach_mapping
 374
 375     $self->_foreach_mapping(
 376         sub {
 377             my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
 378                 $marc_field )
 379               = @_;
 380             return unless $marc_type eq 'marc21';
 381             print "Data comes from: " . $marc_field . "\n";
 382         }
 383     );
 384
 385 This allows you to apply a function to each entry in the elasticsearch mappings
 386 table, in order to build the mappings for whatever is needed.
 387
 388 In the provided function, the files are:
 389
 390 =over 4
 391
 392 =item C<$name>
 393
 394 The field name for elasticsearch (corresponds to the 'mapping' column in the
 395 database.
 396
 397 =item C<$type>
 398
 399 The type for this value, e.g. 'string'.
 400
 401 =item C<$facet>
 402
 403 True if this value should be facetised. This only really makes sense if the
 404 field is understood by the facet processing code anyway.
 405
 406 =item C<$sort>
 407
 408 True if this is a field that a) needs special sort handling, and b) if it
 409 should be sorted on. False if a) but not b). Undef if not a). This allows,
 410 for example, author to be sorted on but not everything marked with "author"
 411 to be included in that sort.
 412
 413 =item C<$marc_type>
 414
 415 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
 416 'unimarc', 'normarc'.
 417
 418 =item C<$marc_field>
 419
 420 A string that describes the MARC field that contains the data to extract.
 421 These are of a form suited to Catmandu's MARC fixers.
 422
 423 =back
 424
 425 =cut
 426
 427 sub _foreach_mapping {
 428     my ( $self, $sub ) = @_;
 429
 430     # TODO use a caching framework here
 431     my $search_fields = Koha::Database->schema->resultset('SearchField')->search(
 432         {
 433             'search_marc_map.index_name' => $self->index,
 434         },
 435         {   join => { search_marc_to_fields => 'search_marc_map' },
 436             '+select' => [
 437                 'search_marc_to_fields.facet',
 438                 'search_marc_to_fields.suggestible',
 439                 'search_marc_to_fields.sort',
 440                 'search_marc_map.marc_type',
 441                 'search_marc_map.marc_field',
 442             ],
 443             '+as'     => [
 444                 'facet',
 445                 'suggestible',
 446                 'sort',
 447                 'marc_type',
 448                 'marc_field',
 449             ],
 450         }
 451     );
 452
 453     while ( my $search_field = $search_fields->next ) {
 454         $sub->(
 455             $search_field->name,
 456             $search_field->type,
 457             $search_field->get_column('facet'),
 458             $search_field->get_column('suggestible'),
 459             $search_field->get_column('sort'),
 460             $search_field->get_column('marc_type'),
 461             $search_field->get_column('marc_field'),
 462         );
 463     }
 464 }
 465
 466 =head2 process_error
 467
 468     die process_error($@);
 469
 470 This parses an Elasticsearch error message and produces a human-readable
 471 result from it. This result is probably missing all the useful information
 472 that you might want in diagnosing an issue, so the warning is also logged.
 473
 474 Note that currently the resulting message is not internationalised. This
 475 will happen eventually by some method or other.
 476
 477 =cut
 478
 479 sub process_error {
 480     my ($self, $msg) = @_;
 481
 482     warn $msg; # simple logging
 483
 484     # This is super-primitive
 485     return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
 486
 487     return "Unable to perform your search. Please try again.\n";
 488 }
 489
 490 1;
 491
 492 __END__
 493
 494 =head1 AUTHOR
 495
 496 =over 4
 497
 498 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
 499
 500 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
 501
 502 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>
 503
 504 =back
 505
 506 =cut