1 package Koha
::SearchEngine
::Elasticsearch
;
3 # Copyright 2015 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use base
qw(Class::Accessor);
25 use Koha
::SearchFields
;
26 use Koha
::SearchMarcMaps
;
34 use Data
::Dumper
; # TODO remove
36 __PACKAGE__
->mk_ro_accessors(qw( index ));
37 __PACKAGE__
->mk_accessors(qw( sort_fields ));
39 # Constants to refer to the standard index names
40 Readonly
our $BIBLIOS_INDEX => 'biblios';
41 Readonly
our $AUTHORITIES_INDEX => 'authorities';
45 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
53 The name of the index to use, generally 'biblios' or 'authorities'.
63 my $self = $class->SUPER::new
(@_);
64 # Check for a valid index
65 croak
('No index name provided') unless $self->index;
69 =head2 get_elasticsearch_params
71 my $params = $self->get_elasticsearch_params();
73 This provides a hashref that contains the parameters for connecting to the
74 ElasicSearch servers, in the form:
77 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
78 'index_name' => 'koha_instance_index',
81 This is configured by the following in the C<config> block in koha-conf.xml:
84 <server>127.0.0.1:9200</server>
85 <server>anotherserver:9200</server>
86 <index_name>koha_instance</index_name>
91 sub get_elasticsearch_params
{
94 # Copy the hash so that we're not modifying the original
95 my $conf = C4
::Context
->config('elasticsearch');
96 die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
97 my $es = { %{ $conf } };
99 # Helpfully, the multiple server lines end up in an array for us anyway
100 # if there are multiple ones, but not if there's only one.
101 my $server = $es->{server
};
102 delete $es->{server
};
103 if ( ref($server) eq 'ARRAY' ) {
105 # store it called 'nodes' (which is used by newer Search::Elasticsearch)
106 $es->{nodes
} = $server;
109 $es->{nodes
} = [$server];
112 die "No elasticsearch servers were specified in koha-conf.xml.\n";
114 die "No elasticserver index_name was specified in koha-conf.xml.\n"
115 if ( !$es->{index_name
} );
116 # Append the name of this particular index to our namespace
117 $es->{index_name
} .= '_' . $self->index;
119 $es->{key_prefix
} = 'es_';
123 =head2 get_elasticsearch_settings
125 my $settings = $self->get_elasticsearch_settings();
127 This provides the settings provided to elasticsearch when an index is created.
128 These can do things like define tokenisation methods.
130 A hashref containing the settings is returned.
134 sub get_elasticsearch_settings
{
137 # Ultimately this should come from a file or something, and not be
144 tokenizer
=> 'keyword',
145 filter
=> ['lowercase'],
147 analyser_standard
=> {
148 tokenizer
=> 'standard',
149 filter
=> ['lowercase'],
158 =head2 get_elasticsearch_mappings
160 my $mappings = $self->get_elasticsearch_mappings();
162 This provides the mappings that get passed to elasticsearch when an index is
167 sub get_elasticsearch_mappings
{
170 # TODO cache in the object?
176 include_in_all
=> JSON
::false
,
183 my $marcflavour = lc C4
::Context
->preference('marcflavour');
184 $self->_foreach_mapping(
186 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
187 return if $marc_type ne $marcflavour;
188 # TODO if this gets any sort of complexity to it, it should
189 # be broken out into its own function.
191 # TODO be aware of date formats, but this requires pre-parsing
192 # as ES will simply reject anything with an invalid date.
198 if ($es_type eq 'boolean') {
199 $mappings->{data
}{properties
}{$name} = _elasticsearch_mapping_for_boolean
( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
200 return; #Boolean cannot have facets nor sorting nor suggestions
202 $mappings->{data
}{properties
}{$name} = _elasticsearch_mapping_for_default
( $name, $es_type, $facet, $suggestible, $sort, $marc_type );
206 $mappings->{data
}{properties
}{ $name . '__facet' } = {
211 $mappings->{data
}{properties
}{ $name . '__suggestion' } = {
212 type
=> 'completion',
213 analyzer
=> 'simple',
214 search_analyzer
=> 'simple',
217 # Sort may be true, false, or undef. Here we care if it's
218 # anything other than undef.
220 $mappings->{data
}{properties
}{ $name . '__sort' } = {
221 search_analyzer
=> "analyser_phrase",
222 analyzer
=> "analyser_phrase",
224 include_in_all
=> JSON
::false
,
227 search_analyzer
=> "analyser_phrase",
228 analyzer
=> "analyser_phrase",
233 $sort_fields{$name} = 1;
237 $self->sort_fields(\
%sort_fields);
241 =head2 _elasticsearch_mapping_for_*
243 Get the ES mappings for the given data type or a special mapping case
245 Receives the same parameters from the $self->_foreach_mapping() dispatcher
249 sub _elasticsearch_mapping_for_boolean
{
250 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
258 sub _elasticsearch_mapping_for_default
{
259 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
262 search_analyzer
=> "analyser_standard",
263 analyzer
=> "analyser_standard",
267 search_analyzer
=> "analyser_phrase",
268 analyzer
=> "analyser_phrase",
278 sub reset_elasticsearch_mappings
{
279 my $mappings_yaml = C4
::Context
->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
280 my $indexes = LoadFile
( $mappings_yaml );
282 while ( my ( $index_name, $fields ) = each %$indexes ) {
283 while ( my ( $field_name, $data ) = each %$fields ) {
284 my $field_type = $data->{type
};
285 my $field_label = $data->{label
};
286 my $mappings = $data->{mappings
};
287 my $search_field = Koha
::SearchFields
->find_or_create({ name
=> $field_name, label
=> $field_label, type
=> $field_type }, { key
=> 'name' });
288 for my $mapping ( @
$mappings ) {
289 my $marc_field = Koha
::SearchMarcMaps
->find_or_create({ index_name
=> $index_name, marc_type
=> $mapping->{marc_type
}, marc_field
=> $mapping->{marc_field
} });
290 $search_field->add_to_search_marc_maps($marc_field, { facet
=> $mapping->{facet
} || 0, suggestible
=> $mapping->{suggestible
} || 0, sort => $mapping->{sort} } );
296 # This overrides the accessor provided by Class::Accessor so that if
297 # sort_fields isn't set, then it'll generate it.
302 $self->_sort_fields_accessor(@_);
305 my $val = $self->_sort_fields_accessor();
308 # This will populate the accessor as a side effect
309 $self->get_elasticsearch_mappings();
310 return $self->_sort_fields_accessor();
313 # Provides the rules for data conversion.
314 sub get_fixer_rules
{
317 my $marcflavour = lc C4
::Context
->preference('marcflavour');
320 $self->_foreach_mapping(
322 my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
323 return if $marc_type ne $marcflavour;
326 # There's a bug when using 'split' with something that
328 # The split makes everything into nested arrays, but that's not
329 # really a big deal, ES doesn't mind.
330 $options = '-split => 1' unless $marc_field =~ m
|_
/| || $type eq 'sum';
331 push @rules, "marc_map('$marc_field','${name}', $options)";
333 push @rules, "marc_map('$marc_field','${name}__facet', $options)";
337 #"marc_map('$marc_field','${name}__suggestion.input.\$append', $options)"; #must not have nested data structures in .input
338 "marc_map('$marc_field','${name}__suggestion.input.\$append')";
340 if ( $type eq 'boolean' ) {
342 # boolean gets special handling, basically if it doesn't exist,
343 # it's added and set to false. Otherwise we can't query it.
345 "unless exists('$name') add_field('$name', 0) end";
347 if ($type eq 'sum' ) {
348 push @rules, "sum('$name')";
350 # Sort is a bit special as it can be true, false, undef. For
351 # fixer rules, we care about "true", or "undef" if there is
352 # special handling of this field from other one. "undef" means
353 # to do the default thing, which is make it sortable.
354 if ($self->sort_fields()->{$name}) {
355 if ($sort || !defined $sort) {
356 push @rules, "marc_map('$marc_field','${name}__sort', $options)";
362 push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');
366 =head2 _foreach_mapping
368 $self->_foreach_mapping(
370 my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
373 return unless $marc_type eq 'marc21';
374 print "Data comes from: " . $marc_field . "\n";
378 This allows you to apply a function to each entry in the elasticsearch mappings
379 table, in order to build the mappings for whatever is needed.
381 In the provided function, the files are:
387 The field name for elasticsearch (corresponds to the 'mapping' column in the
392 The type for this value, e.g. 'string'.
396 True if this value should be facetised. This only really makes sense if the
397 field is understood by the facet processing code anyway.
401 True if this is a field that a) needs special sort handling, and b) if it
402 should be sorted on. False if a) but not b). Undef if not a). This allows,
403 for example, author to be sorted on but not everything marked with "author"
404 to be included in that sort.
408 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
409 'unimarc', 'normarc'.
413 A string that describes the MARC field that contains the data to extract.
414 These are of a form suited to Catmandu's MARC fixers.
420 sub _foreach_mapping
{
421 my ( $self, $sub ) = @_;
423 # TODO use a caching framework here
424 my $search_fields = Koha
::Database
->schema->resultset('SearchField')->search(
426 'search_marc_map.index_name' => $self->index,
428 { join => { search_marc_to_fields
=> 'search_marc_map' },
430 'search_marc_to_fields.facet',
431 'search_marc_to_fields.suggestible',
432 'search_marc_to_fields.sort',
433 'search_marc_map.marc_type',
434 'search_marc_map.marc_field',
446 while ( my $search_field = $search_fields->next ) {
450 $search_field->get_column('facet'),
451 $search_field->get_column('suggestible'),
452 $search_field->get_column('sort'),
453 $search_field->get_column('marc_type'),
454 $search_field->get_column('marc_field'),
461 die process_error($@);
463 This parses an Elasticsearch error message and produces a human-readable
464 result from it. This result is probably missing all the useful information
465 that you might want in diagnosing an issue, so the warning is also logged.
467 Note that currently the resulting message is not internationalised. This
468 will happen eventually by some method or other.
473 my ($self, $msg) = @_;
475 warn $msg; # simple logging
477 # This is super-primitive
478 return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
480 return "Unable to perform your search. Please try again.\n";
491 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
493 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
495 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>