1 package Koha
::SearchEngine
::Elasticsearch
;
3 # Copyright 2015 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use base
qw(Class::Accessor);
25 use Koha
::Exceptions
::Config
;
26 use Koha
::SearchFields
;
27 use Koha
::SearchMarcMaps
;
33 use Search
::Elasticsearch
;
37 use Data
::Dumper
; # TODO remove
39 __PACKAGE__
->mk_ro_accessors(qw( index ));
40 __PACKAGE__
->mk_accessors(qw( sort_fields ));
42 # Constants to refer to the standard index names
43 Readonly
our $BIBLIOS_INDEX => 'biblios';
44 Readonly
our $AUTHORITIES_INDEX => 'authorities';
48 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
56 The name of the index to use, generally 'biblios' or 'authorities'.
66 my $self = $class->SUPER::new
(@_);
67 # Check for a valid index
68 croak
('No index name provided') unless $self->index;
72 =head2 get_elasticsearch_params
74 my $params = $self->get_elasticsearch_params();
76 This provides a hashref that contains the parameters for connecting to the
77 ElasicSearch servers, in the form:
80 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
81 'index_name' => 'koha_instance_index',
84 This is configured by the following in the C<config> block in koha-conf.xml:
87 <server>127.0.0.1:9200</server>
88 <server>anotherserver:9200</server>
89 <index_name>koha_instance</index_name>
94 sub get_elasticsearch_params
{
97 # Copy the hash so that we're not modifying the original
98 my $conf = C4
::Context
->config('elasticsearch');
99 die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
100 my $es = { %{ $conf } };
102 # Helpfully, the multiple server lines end up in an array for us anyway
103 # if there are multiple ones, but not if there's only one.
104 my $server = $es->{server
};
105 delete $es->{server
};
106 if ( ref($server) eq 'ARRAY' ) {
108 # store it called 'nodes' (which is used by newer Search::Elasticsearch)
109 $es->{nodes
} = $server;
112 $es->{nodes
} = [$server];
115 die "No elasticsearch servers were specified in koha-conf.xml.\n";
117 die "No elasticserver index_name was specified in koha-conf.xml.\n"
118 if ( !$es->{index_name
} );
119 # Append the name of this particular index to our namespace
120 $es->{index_name
} .= '_' . $self->index;
122 $es->{key_prefix
} = 'es_';
126 =head2 get_elasticsearch_settings
128 my $settings = $self->get_elasticsearch_settings();
130 This provides the settings provided to elasticsearch when an index is created.
131 These can do things like define tokenisation methods.
133 A hashref containing the settings is returned.
137 sub get_elasticsearch_settings
{
140 # Ultimately this should come from a file or something, and not be
147 tokenizer
=> 'icu_tokenizer',
148 filter
=> ['icu_folding'],
150 analyser_standard
=> {
151 tokenizer
=> 'icu_tokenizer',
152 filter
=> ['icu_folding'],
161 =head2 get_elasticsearch_mappings
163 my $mappings = $self->get_elasticsearch_mappings();
165 This provides the mappings that get passed to elasticsearch when an index is
170 sub get_elasticsearch_mappings
{
173 # TODO cache in the object?
176 _all
=> {type
=> "string", analyzer
=> "analyser_standard"},
180 include_in_all
=> JSON
::false
,
187 my $marcflavour = lc C4
::Context
->preference('marcflavour');
188 $self->_foreach_mapping(
190 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
191 return if $marc_type ne $marcflavour;
192 # TODO if this gets any sort of complexity to it, it should
193 # be broken out into its own function.
195 # TODO be aware of date formats, but this requires pre-parsing
196 # as ES will simply reject anything with an invalid date.
202 if ($es_type eq 'boolean') {
203 $mappings->{data
}{properties
}{$name} = _elasticsearch_mapping_for_boolean
( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
204 return; #Boolean cannot have facets nor sorting nor suggestions
206 $mappings->{data
}{properties
}{$name} = _elasticsearch_mapping_for_default
( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
210 $mappings->{data
}{properties
}{ $name . '__facet' } = {
215 $mappings->{data
}{properties
}{ $name . '__suggestion' } = {
216 type
=> 'completion',
217 analyzer
=> 'simple',
218 search_analyzer
=> 'simple',
221 # Sort is a bit special as it can be true, false, undef.
222 # We care about "true" or "undef",
223 # "undef" means to do the default thing, which is make it sortable.
224 if ($sort || !defined $sort) {
225 $mappings->{data
}{properties
}{ $name . '__sort' } = {
226 search_analyzer
=> "analyser_phrase",
227 analyzer
=> "analyser_phrase",
229 include_in_all
=> JSON
::false
,
236 $sort_fields{$name} = 1;
240 $self->sort_fields(\
%sort_fields);
244 =head2 _elasticsearch_mapping_for_*
246 Get the ES mappings for the given data type or a special mapping case
248 Receives the same parameters from the $self->_foreach_mapping() dispatcher
252 sub _elasticsearch_mapping_for_boolean
{
253 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
261 sub _elasticsearch_mapping_for_default
{
262 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
265 search_analyzer
=> "analyser_standard",
266 analyzer
=> "analyser_standard",
270 search_analyzer
=> "analyser_phrase",
271 analyzer
=> "analyser_phrase",
281 sub reset_elasticsearch_mappings
{
282 my $mappings_yaml = C4
::Context
->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
283 my $indexes = LoadFile
( $mappings_yaml );
285 while ( my ( $index_name, $fields ) = each %$indexes ) {
286 while ( my ( $field_name, $data ) = each %$fields ) {
287 my $field_type = $data->{type
};
288 my $field_label = $data->{label
};
289 my $mappings = $data->{mappings
};
290 my $search_field = Koha
::SearchFields
->find_or_create({ name
=> $field_name, label
=> $field_label, type
=> $field_type }, { key
=> 'name' });
291 for my $mapping ( @
$mappings ) {
292 my $marc_field = Koha
::SearchMarcMaps
->find_or_create({ index_name
=> $index_name, marc_type
=> $mapping->{marc_type
}, marc_field
=> $mapping->{marc_field
} });
293 $search_field->add_to_search_marc_maps($marc_field, { facet
=> $mapping->{facet
} || 0, suggestible
=> $mapping->{suggestible
} || 0, sort => $mapping->{sort} } );
299 # This overrides the accessor provided by Class::Accessor so that if
300 # sort_fields isn't set, then it'll generate it.
304 $self->_sort_fields_accessor(@_);
307 my $val = $self->_sort_fields_accessor();
310 # This will populate the accessor as a side effect
311 $self->get_elasticsearch_mappings();
312 return $self->_sort_fields_accessor();
315 # Provides the rules for data conversion.
316 sub get_fixer_rules
{
319 my $marcflavour = lc C4
::Context
->preference('marcflavour');
322 $self->_foreach_mapping(
324 my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
325 return if $marc_type ne $marcflavour;
328 # There's a bug when using 'split' with something that
330 # The split makes everything into nested arrays, but that's not
331 # really a big deal, ES doesn't mind.
332 $options = '-split => 1' unless $marc_field =~ m
|_
/| || $type eq 'sum';
333 push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
335 push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
339 #"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; #must not have nested data structures in .input
340 "marc_map('$marc_field','${name}__suggestion.input.\$append')";
342 if ( $type eq 'boolean' ) {
344 # boolean gets special handling, basically if it doesn't exist,
345 # it's added and set to false. Otherwise we can't query it.
347 "unless exists('$name') add_field('$name', 0) end";
349 if ($type eq 'sum' ) {
350 push @rules, "sum('$name')";
352 if ($self->sort_fields()->{$name}) {
353 if ($sort || !defined $sort) {
354 push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";
360 push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');
364 =head2 _foreach_mapping
366 $self->_foreach_mapping(
368 my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
371 return unless $marc_type eq 'marc21';
372 print "Data comes from: " . $marc_field . "\n";
376 This allows you to apply a function to each entry in the elasticsearch mappings
377 table, in order to build the mappings for whatever is needed.
379 In the provided function, the files are:
385 The field name for elasticsearch (corresponds to the 'mapping' column in the
390 The type for this value, e.g. 'string'.
394 True if this value should be facetised. This only really makes sense if the
395 field is understood by the facet processing code anyway.
399 True if this is a field that a) needs special sort handling, and b) if it
400 should be sorted on. False if a) but not b). Undef if not a). This allows,
401 for example, author to be sorted on but not everything marked with "author"
402 to be included in that sort.
406 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
407 'unimarc', 'normarc'.
411 A string that describes the MARC field that contains the data to extract.
412 These are of a form suited to Catmandu's MARC fixers.
418 sub _foreach_mapping
{
419 my ( $self, $sub ) = @_;
421 # TODO use a caching framework here
422 my $search_fields = Koha
::Database
->schema->resultset('SearchField')->search(
424 'search_marc_map.index_name' => $self->index,
426 { join => { search_marc_to_fields
=> 'search_marc_map' },
428 'search_marc_to_fields.facet',
429 'search_marc_to_fields.suggestible',
430 'search_marc_to_fields.sort',
431 'search_marc_map.marc_type',
432 'search_marc_map.marc_field',
444 while ( my $search_field = $search_fields->next ) {
448 $search_field->get_column('facet'),
449 $search_field->get_column('suggestible'),
450 $search_field->get_column('sort'),
451 $search_field->get_column('marc_type'),
452 $search_field->get_column('marc_field'),
459 die process_error($@);
461 This parses an Elasticsearch error message and produces a human-readable
462 result from it. This result is probably missing all the useful information
463 that you might want in diagnosing an issue, so the warning is also logged.
465 Note that currently the resulting message is not internationalised. This
466 will happen eventually by some method or other.
471 my ($self, $msg) = @_;
473 warn $msg; # simple logging
475 # This is super-primitive
476 return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
478 return "Unable to perform your search. Please try again.\n";
481 =head2 _read_configuration
483 my $conf = _read_configuration();
485 Reads the I<configuration file> and returns a hash structure with the
486 configuration information. It raises an exception if mandatory entries
489 The hashref structure has the following form:
492 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
493 'index_name' => 'koha_instance',
496 This is configured by the following in the C<config> block in koha-conf.xml:
499 <server>127.0.0.1:9200</server>
500 <server>anotherserver:9200</server>
501 <index_name>koha_instance</index_name>
506 sub _read_configuration
{
510 my $conf = C4
::Context
->config('elasticsearch');
511 Koha
::Exceptions
::Config
::MissingEntry
->throw(
512 "Missing 'elasticsearch' block in config file")
513 unless defined $conf;
515 if ( $conf && $conf->{server
} ) {
516 my $nodes = $conf->{server
};
517 if ( ref($nodes) eq 'ARRAY' ) {
518 $configuration->{nodes
} = $nodes;
521 $configuration->{nodes
} = [$nodes];
525 Koha
::Exceptions
::Config
::MissingEntry
->throw(
526 "Missing 'server' entry in config file for elasticsearch");
529 if ( defined $conf->{index_name
} ) {
530 $configuration->{index_name
} = $conf->{index_name
};
533 Koha
::Exceptions
::Config
::MissingEntry
->throw(
534 "Missing 'index_name' entry in config file for elasticsearch");
537 return $configuration;
548 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
550 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
552 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>