1 package Koha
::SearchEngine
::Elasticsearch
;
3 # Copyright 2015 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use base
qw(Class::Accessor);
25 use Koha
::SearchFields
;
26 use Koha
::SearchMarcMaps
;
34 use Data
::Dumper
; # TODO remove
36 __PACKAGE__
->mk_ro_accessors(qw( index ));
37 __PACKAGE__
->mk_accessors(qw( sort_fields ));
39 # Constants to refer to the standard index names
40 Readonly
our $BIBLIOS_INDEX => 'biblios';
41 Readonly
our $AUTHORITIES_INDEX => 'authorities';
45 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
53 The name of the index to use, generally 'biblios' or 'authorities'.
63 my $self = $class->SUPER::new
(@_);
64 # Check for a valid index
65 croak
('No index name provided') unless $self->index;
69 =head2 get_elasticsearch_params
71 my $params = $self->get_elasticsearch_params();
73 This provides a hashref that contains the parameters for connecting to the
74 ElasicSearch servers, in the form:
77 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
78 'index_name' => 'koha_instance_index',
81 This is configured by the following in the C<config> block in koha-conf.xml:
84 <server>127.0.0.1:9200</server>
85 <server>anotherserver:9200</server>
86 <index_name>koha_instance</index_name>
91 sub get_elasticsearch_params
{
94 # Copy the hash so that we're not modifying the original
95 my $conf = C4
::Context
->config('elasticsearch');
96 die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
97 my $es = { %{ $conf } };
99 # Helpfully, the multiple server lines end up in an array for us anyway
100 # if there are multiple ones, but not if there's only one.
101 my $server = $es->{server
};
102 delete $es->{server
};
103 if ( ref($server) eq 'ARRAY' ) {
105 # store it called 'nodes' (which is used by newer Search::Elasticsearch)
106 $es->{nodes
} = $server;
109 $es->{nodes
} = [$server];
112 die "No elasticsearch servers were specified in koha-conf.xml.\n";
114 die "No elasticserver index_name was specified in koha-conf.xml.\n"
115 if ( !$es->{index_name
} );
116 # Append the name of this particular index to our namespace
117 $es->{index_name
} .= '_' . $self->index;
121 =head2 get_elasticsearch_settings
123 my $settings = $self->get_elasticsearch_settings();
125 This provides the settings provided to elasticsearch when an index is created.
126 These can do things like define tokenisation methods.
128 A hashref containing the settings is returned.
132 sub get_elasticsearch_settings
{
135 # Ultimately this should come from a file or something, and not be
142 tokenizer
=> 'keyword',
143 filter
=> ['lowercase'],
145 analyser_standard
=> {
146 tokenizer
=> 'standard',
147 filter
=> ['lowercase'],
156 =head2 get_elasticsearch_mappings
158 my $mappings = $self->get_elasticsearch_mappings();
160 This provides the mappings that get passed to elasticsearch when an index is
165 sub get_elasticsearch_mappings
{
168 # TODO cache in the object?
174 include_in_all
=> JSON
::false
,
178 search_analyzer
=> "analyser_phrase",
179 index_analyzer
=> "analyser_phrase",
186 my $marcflavour = lc C4
::Context
->preference('marcflavour');
187 $self->_foreach_mapping(
189 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
190 return if $marc_type ne $marcflavour;
191 # TODO if this gets any sort of complexity to it, it should
192 # be broken out into its own function.
194 # TODO be aware of date formats, but this requires pre-parsing
195 # as ES will simply reject anything with an invalid date.
200 $mappings->{data
}{properties
}{$name} = {
201 search_analyzer
=> "analyser_standard",
202 index_analyzer
=> "analyser_standard",
206 search_analyzer
=> "analyser_phrase",
207 index_analyzer
=> "analyser_phrase",
209 copy_to
=> "_all.phrase",
213 "index" => "not_analyzed",
217 $mappings->{data
}{properties
}{$name}{null_value
} = 0
218 if $type eq 'boolean';
220 $mappings->{data
}{properties
}{ $name . '__facet' } = {
222 index => "not_analyzed",
226 $mappings->{data
}{properties
}{ $name . '__suggestion' } = {
227 type
=> 'completion',
228 index_analyzer
=> 'simple',
229 search_analyzer
=> 'simple',
232 # Sort may be true, false, or undef. Here we care if it's
233 # anything other than undef.
235 $mappings->{data
}{properties
}{ $name . '__sort' } = {
236 search_analyzer
=> "analyser_phrase",
237 index_analyzer
=> "analyser_phrase",
239 include_in_all
=> JSON
::false
,
242 search_analyzer
=> "analyser_phrase",
243 index_analyzer
=> "analyser_phrase",
248 $sort_fields{$name} = 1;
252 $self->sort_fields(\
%sort_fields);
256 sub reset_elasticsearch_mappings
{
257 my $mappings_yaml = C4
::Context
->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
258 my $indexes = LoadFile
( $mappings_yaml );
260 while ( my ( $index_name, $fields ) = each %$indexes ) {
261 while ( my ( $field_name, $data ) = each %$fields ) {
262 my $field_type = $data->{type
};
263 my $field_label = $data->{label
};
264 my $mappings = $data->{mappings
};
265 my $search_field = Koha
::SearchFields
->find_or_create({ name
=> $field_name, label
=> $field_label, type
=> $field_type }, { key
=> 'name' });
266 for my $mapping ( @
$mappings ) {
267 my $marc_field = Koha
::SearchMarcMaps
->find_or_create({ index_name
=> $index_name, marc_type
=> $mapping->{marc_type
}, marc_field
=> $mapping->{marc_field
} });
268 $search_field->add_to_search_marc_maps($marc_field, { facet
=> $mapping->{facet
}, suggestible
=> $mapping->{suggestible
}, sort => $mapping->{sort} } );
274 # This overrides the accessor provided by Class::Accessor so that if
275 # sort_fields isn't set, then it'll generate it.
280 $self->_sort_fields_accessor(@_);
283 my $val = $self->_sort_fields_accessor();
286 # This will populate the accessor as a side effect
287 $self->get_elasticsearch_mappings();
288 return $self->_sort_fields_accessor();
291 # Provides the rules for data conversion.
292 sub get_fixer_rules
{
295 my $marcflavour = lc C4
::Context
->preference('marcflavour');
297 $self->_foreach_mapping(
299 my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
300 return if $marc_type ne $marcflavour;
303 # There's a bug when using 'split' with something that
305 # The split makes everything into nested arrays, but that's not
306 # really a big deal, ES doesn't mind.
307 $options = '-split => 1' unless $marc_field =~ m
|_
/| || $type eq 'sum';
308 push @rules, "marc_map('$marc_field','${name}', $options)";
310 push @rules, "marc_map('$marc_field','${name}__facet', $options)";
314 "marc_map('$marc_field','${name}__suggestion.input.\$append', $options)";
316 if ( $type eq 'boolean' ) {
318 # boolean gets special handling, basically if it doesn't exist,
319 # it's added and set to false. Otherwise we can't query it.
321 "unless exists('$name') add_field('$name', 0) end";
323 if ($type eq 'sum' ) {
324 push @rules, "sum('$name')";
326 # Sort is a bit special as it can be true, false, undef. For
327 # fixer rules, we care about "true", or "undef" if there is
328 # special handling of this field from other one. "undef" means
329 # to do the default thing, which is make it sortable.
330 if ($self->sort_fields()->{$name}) {
331 if ($sort || !defined $sort) {
332 push @rules, "marc_map('$marc_field','${name}__sort', $options)";
340 =head2 _foreach_mapping
342 $self->_foreach_mapping(
344 my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
347 return unless $marc_type eq 'marc21';
348 print "Data comes from: " . $marc_field . "\n";
352 This allows you to apply a function to each entry in the elasticsearch mappings
353 table, in order to build the mappings for whatever is needed.
355 In the provided function, the files are:
361 The field name for elasticsearch (corresponds to the 'mapping' column in the
366 The type for this value, e.g. 'string'.
370 True if this value should be facetised. This only really makes sense if the
371 field is understood by the facet processing code anyway.
375 True if this is a field that a) needs special sort handling, and b) if it
376 should be sorted on. False if a) but not b). Undef if not a). This allows,
377 for example, author to be sorted on but not everything marked with "author"
378 to be included in that sort.
382 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
383 'unimarc', 'normarc'.
387 A string that describes the MARC field that contains the data to extract.
388 These are of a form suited to Catmandu's MARC fixers.
394 sub _foreach_mapping
{
395 my ( $self, $sub ) = @_;
397 # TODO use a caching framework here
398 my $search_fields = Koha
::Database
->schema->resultset('SearchField')->search(
400 'search_marc_map.index_name' => $self->index,
402 { join => { search_marc_to_fields
=> 'search_marc_map' },
404 'search_marc_to_fields.facet',
405 'search_marc_to_fields.suggestible',
406 'search_marc_to_fields.sort',
407 'search_marc_map.marc_type',
408 'search_marc_map.marc_field',
420 while ( my $search_field = $search_fields->next ) {
424 $search_field->get_column('facet'),
425 $search_field->get_column('suggestible'),
426 $search_field->get_column('sort'),
427 $search_field->get_column('marc_type'),
428 $search_field->get_column('marc_field'),
435 die process_error($@);
437 This parses an Elasticsearch error message and produces a human-readable
438 result from it. This result is probably missing all the useful information
439 that you might want in diagnosing an issue, so the warning is also logged.
441 Note that currently the resulting message is not internationalised. This
442 will happen eventually by some method or other.
447 my ($self, $msg) = @_;
449 warn $msg; # simple logging
451 # This is super-primitive
452 return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
454 return "Unable to perform your search. Please try again.\n";
465 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
467 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
469 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>