1 package Koha
::SearchEngine
::Elasticsearch
;
3 # Copyright 2015 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use base
qw(Class::Accessor);
25 use Koha
::Exceptions
::Config
;
26 use Koha
::SearchFields
;
27 use Koha
::SearchMarcMaps
;
34 use Search
::Elasticsearch
;
38 use List
::Util
qw( sum0 reduce );
41 use Encode
qw(encode);
44 __PACKAGE__
->mk_ro_accessors(qw( index ));
45 __PACKAGE__
->mk_accessors(qw( sort_fields ));
47 # Constants to refer to the standard index names
48 Readonly
our $BIBLIOS_INDEX => 'biblios';
49 Readonly
our $AUTHORITIES_INDEX => 'authorities';
53 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
61 The name of the index to use, generally 'biblios' or 'authorities'.
71 my $self = $class->SUPER::new
(@_);
72 # Check for a valid index
73 Koha
::Exceptions
::MissingParameter
->throw('No index name provided') unless $self->index;
77 =head2 get_elasticsearch
79 my $elasticsearch_client = $self->get_elasticsearch();
81 Returns a C<Search::Elasticsearch> client. The client is cached on a C<Koha::SearchEngine::ElasticSearch>
82 instance level and will be reused if method is called multiple times.
86 sub get_elasticsearch
{
88 unless (defined $self->{elasticsearch
}) {
89 my $conf = $self->get_elasticsearch_params();
90 $self->{elasticsearch
} = Search
::Elasticsearch
->new($conf);
92 return $self->{elasticsearch
};
95 =head2 get_elasticsearch_params
97 my $params = $self->get_elasticsearch_params();
99 This provides a hashref that contains the parameters for connecting to the
100 ElasicSearch servers, in the form:
103 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
104 'index_name' => 'koha_instance_index',
107 This is configured by the following in the C<config> block in koha-conf.xml:
110 <server>127.0.0.1:9200</server>
111 <server>anotherserver:9200</server>
112 <index_name>koha_instance</index_name>
117 sub get_elasticsearch_params
{
120 # Copy the hash so that we're not modifying the original
121 my $conf = C4
::Context
->config('elasticsearch');
122 die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
123 my $es = { %{ $conf } };
125 # Helpfully, the multiple server lines end up in an array for us anyway
126 # if there are multiple ones, but not if there's only one.
127 my $server = $es->{server
};
128 delete $es->{server
};
129 if ( ref($server) eq 'ARRAY' ) {
131 # store it called 'nodes' (which is used by newer Search::Elasticsearch)
132 $es->{nodes
} = $server;
135 $es->{nodes
} = [$server];
138 die "No elasticsearch servers were specified in koha-conf.xml.\n";
140 die "No elasticsearch index_name was specified in koha-conf.xml.\n"
141 if ( !$es->{index_name
} );
142 # Append the name of this particular index to our namespace
143 $es->{index_name
} .= '_' . $self->index;
145 $es->{key_prefix
} = 'es_';
146 $es->{client
} //= '5_0::Direct';
147 $es->{cxn_pool
} //= 'Sniff';
148 $es->{request_timeout
} //= 60;
153 =head2 get_elasticsearch_settings
155 my $settings = $self->get_elasticsearch_settings();
157 This provides the settings provided to Elasticsearch when an index is created.
158 These can do things like define tokenization methods.
160 A hashref containing the settings is returned.
164 sub get_elasticsearch_settings
{
167 # Use state to speed up repeated calls
168 state $settings = undef;
169 if (!defined $settings) {
170 my $config_file = C4
::Context
->config('elasticsearch_index_config');
171 $config_file ||= C4
::Context
->config('intranetdir') . '/admin/searchengine/elasticsearch/index_config.yaml';
172 $settings = LoadFile
( $config_file );
178 =head2 get_elasticsearch_mappings
180 my $mappings = $self->get_elasticsearch_mappings();
182 This provides the mappings that get passed to Elasticsearch when an index is
187 sub get_elasticsearch_mappings
{
190 # Use state to speed up repeated calls
194 if (!defined $all_mappings{$self->index}) {
195 $sort_fields{$self->index} = {};
196 # Clone the general mapping to break ties with the original hash
198 data
=> clone
(_get_elasticsearch_field_config
('general', ''))
200 my $marcflavour = lc C4
::Context
->preference('marcflavour');
201 $self->_foreach_mapping(
203 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
205 return if $marc_type ne $marcflavour;
206 # TODO if this gets any sort of complexity to it, it should
207 # be broken out into its own function.
209 # TODO be aware of date formats, but this requires pre-parsing
210 # as ES will simply reject anything with an invalid date.
211 my $es_type = 'text';
212 if ($type eq 'boolean') {
213 $es_type = 'boolean';
214 } elsif ($type eq 'number' || $type eq 'sum') {
215 $es_type = 'integer';
216 } elsif ($type eq 'isbn' || $type eq 'stdno') {
220 $mappings->{data
}{properties
}{$name} = _get_elasticsearch_field_config
('search', $es_type);
223 $mappings->{data
}{properties
}{ $name . '__facet' } = _get_elasticsearch_field_config
('facet', $es_type);
226 $mappings->{data
}{properties
}{ $name . '__suggestion' } = _get_elasticsearch_field_config
('suggestible', $es_type);
228 # Sort is a bit special as it can be true, false, undef.
229 # We care about "true" or "undef",
230 # "undef" means to do the default thing, which is make it sortable.
231 if (!defined $sort || $sort) {
232 $mappings->{data
}{properties
}{ $name . '__sort' } = _get_elasticsearch_field_config
('sort', $es_type);
233 $sort_fields{$self->index}{$name} = 1;
237 $all_mappings{$self->index} = $mappings;
239 $self->sort_fields(\
%{$sort_fields{$self->index}});
241 return $all_mappings{$self->index};
244 =head2 _get_elasticsearch_field_config
246 Get the Elasticsearch field config for the given purpose and data type.
248 $mapping = _get_elasticsearch_field_config('search', 'text');
252 sub _get_elasticsearch_field_config
{
254 my ( $purpose, $type ) = @_;
256 # Use state to speed up repeated calls
257 state $settings = undef;
258 if (!defined $settings) {
259 my $config_file = C4
::Context
->config('elasticsearch_field_config');
260 $config_file ||= C4
::Context
->config('intranetdir') . '/admin/searchengine/elasticsearch/field_config.yaml';
261 $settings = LoadFile
( $config_file );
264 if (!defined $settings->{$purpose}) {
265 die "Field purpose $purpose not defined in field config";
268 return $settings->{$purpose};
270 if (defined $settings->{$purpose}{$type}) {
271 return $settings->{$purpose}{$type};
273 if (defined $settings->{$purpose}{'default'}) {
274 return $settings->{$purpose}{'default'};
279 sub reset_elasticsearch_mappings
{
280 my ( $reset_fields ) = @_;
281 my $mappings_yaml = C4
::Context
->config('elasticsearch_index_mappings');
282 $mappings_yaml ||= C4
::Context
->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
283 my $indexes = LoadFile
( $mappings_yaml );
285 while ( my ( $index_name, $fields ) = each %$indexes ) {
286 while ( my ( $field_name, $data ) = each %$fields ) {
287 my $field_type = $data->{type
};
288 my $field_label = $data->{label
};
289 my $mappings = $data->{mappings
};
290 my $facet_order = $data->{facet_order
};
291 my $search_field = Koha
::SearchFields
->find_or_create({
293 label
=> $field_label,
299 $search_field->update(
301 facet_order
=> $facet_order
304 for my $mapping ( @
$mappings ) {
305 my $marc_field = Koha
::SearchMarcMaps
->find_or_create({ index_name
=> $index_name, marc_type
=> $mapping->{marc_type
}, marc_field
=> $mapping->{marc_field
} });
306 $search_field->add_to_search_marc_maps($marc_field, { facet
=> $mapping->{facet
} || 0, suggestible
=> $mapping->{suggestible
} || 0, sort => $mapping->{sort} } );
312 # This overrides the accessor provided by Class::Accessor so that if
313 # sort_fields isn't set, then it'll generate it.
317 $self->_sort_fields_accessor(@_);
320 my $val = $self->_sort_fields_accessor();
323 # This will populate the accessor as a side effect
324 $self->get_elasticsearch_mappings();
325 return $self->_sort_fields_accessor();
328 =head2 _process_mappings($mappings, $data, $record_document, $altscript)
330 $self->_process_mappings($mappings, $marc_field_data, $record_document, 0)
332 Process all C<$mappings> targets operating on a specific MARC field C<$data>.
333 Since we group all mappings by MARC field targets C<$mappings> will contain
334 all targets for C<$data> and thus we need to fetch the MARC field only once.
335 C<$mappings> will be applied to C<$record_document> and new field values added.
336 The method has no return value.
342 Arrayref of mappings containing arrayrefs in the format
343 [C<$target>, C<$options>] where C<$target> is the name of the target field and
344 C<$options> is a hashref containing processing directives for this particular
349 The source data from a MARC record field.
351 =item C<$record_document>
353 Hashref representing the Elasticsearch document on which mappings should be
358 A boolean value indicating whether an alternate script presentation is being
365 sub _process_mappings
{
366 my ($_self, $mappings, $data, $record_document, $altscript) = @_;
367 foreach my $mapping (@
{$mappings}) {
368 my ($target, $options) = @
{$mapping};
370 # Don't process sort fields for alternate scripts
371 my $sort = $target =~ /__sort$/;
372 if ($sort && $altscript) {
376 # Copy (scalar) data since can have multiple targets
377 # with differing options for (possibly) mutating data
378 # so need a different copy for each
380 $record_document->{$target} //= [];
381 if (defined $options->{substr}) {
382 my ($start, $length) = @
{$options->{substr}};
383 $_data = length($data) > $start ?
substr $data, $start, $length : '';
385 if (defined $options->{value_callbacks
}) {
386 $_data = reduce
{ $b->($a) } ($_data, @
{$options->{value_callbacks
}});
388 if (defined $options->{property
}) {
390 $options->{property
} => $_data
393 push @
{$record_document->{$target}}, $_data;
397 =head2 marc_records_to_documents($marc_records)
399 my @record_documents = $self->marc_records_to_documents($marc_records);
401 Using mappings stored in database convert C<$marc_records> to Elasticsearch documents.
403 Returns array of hash references, representing Elasticsearch documents,
404 acceptable as body payload in C<Search::Elasticsearch> requests.
408 =item C<$marc_documents>
410 Reference to array of C<MARC::Record> objects to be converted to Elasticsearch documents.
416 sub marc_records_to_documents
{
417 my ($self, $records) = @_;
418 my $rules = $self->_get_marc_mapping_rules();
419 my $control_fields_rules = $rules->{control_fields
};
420 my $data_fields_rules = $rules->{data_fields
};
421 my $marcflavour = lc C4
::Context
->preference('marcflavour');
423 my @record_documents;
425 foreach my $record (@
{$records}) {
426 my $record_document = {};
427 my $mappings = $rules->{leader
};
429 $self->_process_mappings($mappings, $record->leader(), $record_document, 0);
431 foreach my $field ($record->fields()) {
432 if ($field->is_control_field()) {
433 my $mappings = $control_fields_rules->{$field->tag()};
435 $self->_process_mappings($mappings, $field->data(), $record_document, 0);
439 my $tag = $field->tag();
440 # Handle alternate scripts in MARC 21
442 if ($marcflavour eq 'marc21' && $tag eq '880') {
443 my $sub6 = $field->subfield('6');
444 if ($sub6 =~ /^(...)-\d+/) {
450 my $data_field_rules = $data_fields_rules->{$tag};
452 if ($data_field_rules) {
453 my $subfields_mappings = $data_field_rules->{subfields
};
454 my $wildcard_mappings = $subfields_mappings->{'*'};
455 foreach my $subfield ($field->subfields()) {
456 my ($code, $data) = @
{$subfield};
457 my $mappings = $subfields_mappings->{$code} // [];
458 if ($wildcard_mappings) {
459 $mappings = [@
{$mappings}, @
{$wildcard_mappings}];
462 $self->_process_mappings($mappings, $data, $record_document, $altscript);
466 my $subfields_join_mappings = $data_field_rules->{subfields_join
};
467 if ($subfields_join_mappings) {
468 foreach my $subfields_group (keys %{$subfields_join_mappings}) {
469 # Map each subfield to values, remove empty values, join with space
474 map { join(' ', $field->subfield($_)) } split(//, $subfields_group)
478 $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document, $altscript);
485 foreach my $field (keys %{$rules->{defaults
}}) {
486 unless (defined $record_document->{$field}) {
487 $record_document->{$field} = $rules->{defaults
}->{$field};
490 foreach my $field (@
{$rules->{sum
}}) {
491 if (defined $record_document->{$field}) {
492 # TODO: validate numeric? filter?
493 # TODO: Or should only accept fields without nested values?
494 # TODO: Quick and dirty, improve if needed
495 $record_document->{$field} = sum0
(grep { !ref($_) && m/\d+(\.\d+)?/} @
{$record_document->{$field}});
498 # Index all applicable ISBN forms (ISBN-10 and ISBN-13 with and without dashes)
499 foreach my $field (@
{$rules->{isbn
}}) {
500 if (defined $record_document->{$field}) {
502 foreach my $input_isbn (@
{$record_document->{$field}}) {
503 my $isbn = Business
::ISBN
->new($input_isbn);
504 if (defined $isbn && $isbn->is_valid) {
505 my $isbn13 = $isbn->as_isbn13->as_string;
506 push @isbns, $isbn13;
508 push @isbns, $isbn13;
510 my $isbn10 = $isbn->as_isbn10;
512 $isbn10 = $isbn10->as_string;
513 push @isbns, $isbn10;
515 push @isbns, $isbn10;
518 push @isbns, $input_isbn;
521 $record_document->{$field} = \
@isbns;
525 # Remove duplicate values and collapse sort fields
526 foreach my $field (keys %{$record_document}) {
527 if (ref($record_document->{$field}) eq 'ARRAY') {
528 @
{$record_document->{$field}} = do {
530 grep { !$seen{ref($_) eq 'HASH' && defined $_->{input
} ?
$_->{input
} : $_}++ } @
{$record_document->{$field}};
532 if ($field =~ /__sort$/) {
533 # Make sure to keep the sort field length sensible. 255 was chosen as a nice round value.
534 $record_document->{$field} = [substr(join(' ', @
{$record_document->{$field}}), 0, 255)];
539 # TODO: Perhaps should check if $records_document non empty, but really should never be the case
540 $record->encoding('UTF-8');
543 # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
544 local $SIG{__WARN__
} = sub {
545 push @warnings, $_[0];
547 $record_document->{'marc_data'} = encode_base64
(encode
('UTF-8', $record->as_usmarc()));
550 # Suppress warnings if record length exceeded
551 unless (substr($record->leader(), 0, 5) eq '99999') {
552 foreach my $warning (@warnings) {
556 $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
557 $record_document->{'marc_format'} = 'MARCXML';
560 $record_document->{'marc_format'} = 'base64ISO2709';
562 my $id = $record->subfield('999', 'c');
563 push @record_documents, [$id, $record_document];
565 return \
@record_documents;
568 =head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
570 my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
572 Get mappings, an internal data structure later used by
573 L<_process_mappings($mappings, $data, $record_document, $altscript)> to process MARC target
574 data for a MARC mapping.
576 The returned C<$mappings> is not to to be confused with mappings provided by
577 C<_foreach_mapping>, rather this sub accepts properties from a mapping as
578 provided by C<_foreach_mapping> and expands it to this internal data structure.
579 In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings>
580 is then applied to each MARC target (leader, control field data, subfield or
581 joined subfields) and integrated into the mapping rules data structure used in
582 C<marc_records_to_documents> to transform MARC records into Elasticsearch
589 Boolean indicating whether to create a facet field for this mapping.
591 =item C<$suggestible>
593 Boolean indicating whether to create a suggestion field for this mapping.
597 Boolean indicating whether to create a sort field for this mapping.
599 =item C<$target_name>
601 Elasticsearch document target field name.
603 =item C<$target_type>
605 Elasticsearch document target field type.
609 An optional range as a string in the format "<START>-<END>" or "<START>",
610 where "<START>" and "<END>" are integers specifying a range that will be used
611 for extracting a substring from MARC data as Elasticsearch field target value.
613 The first character position is "0", and the range is inclusive,
614 so "0-2" means the first three characters of MARC data.
616 If only "<START>" is provided only one character at position "<START>" will
623 sub _field_mappings
{
624 my ($_self, $facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
625 my %mapping_defaults = ();
628 my $substr_args = undef;
629 if (defined $range) {
630 # TODO: use value_callback instead?
631 my ($start, $end) = map(int, split /-/, $range, 2);
632 $substr_args = [$start];
633 push @
{$substr_args}, (defined $end ?
$end - $start + 1 : 1);
635 my $default_options = {};
637 $default_options->{substr} = $substr_args;
640 # TODO: Should probably have per type value callback/hook
641 # but hard code for now
642 if ($target_type eq 'boolean') {
643 $default_options->{value_callbacks
} //= [];
644 push @
{$default_options->{value_callbacks
}}, sub {
646 # Trim whitespace at both ends
647 $value =~ s/^\s+|\s+$//g;
648 return $value ?
'true' : 'false';
652 my $mapping = [$target_name, $default_options];
653 push @mappings, $mapping;
656 push @suffixes, 'facet' if $facet;
657 push @suffixes, 'suggestion' if $suggestible;
658 push @suffixes, 'sort' if !defined $sort || $sort;
660 foreach my $suffix (@suffixes) {
661 my $mapping = ["${target_name}__$suffix"];
662 # TODO: Hack, fix later in less hideous manner
663 if ($suffix eq 'suggestion') {
664 push @
{$mapping}, {%{$default_options}, property
=> 'input'};
667 push @
{$mapping}, $default_options;
669 push @mappings, $mapping;
674 =head2 _get_marc_mapping_rules
676 my $mapping_rules = $self->_get_marc_mapping_rules()
678 Generates rules from mappings stored in database for MARC records to Elasticsearch JSON document conversion.
680 Since field retrieval is slow in C<MARC::Records> (all fields are itereted through for
681 each call to C<MARC::Record>->field) we create an optimized structure of mapping
682 rules keyed by MARC field tags holding all the mapping rules for that particular tag.
684 We can then iterate through all MARC fields for each record and apply all relevant
685 rules once per fields instead of retreiving fields multiple times for each mapping rule
686 which is terribly slow.
690 # TODO: This structure can be used for processing multiple MARC::Records so is currently
691 # rebuilt for each batch. Since it is cacheable it could also be stored in an in
692 # memory cache which it is currently not. The performance gain of caching
693 # would probably be marginal, but to do this could be a further improvement.
695 sub _get_marc_mapping_rules
{
697 my $marcflavour = lc C4
::Context
->preference('marcflavour');
698 my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d
+(?
:-\d
+)?
))?
$/;
699 my $leader_regexp = qr/^leader(?:_\/(\d
+(?
:-\d
+)?
))?
$/;
702 'control_fields' => {},
709 $self->_foreach_mapping(sub {
710 my ($name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field) = @_;
711 return if $marc_type ne $marcflavour;
713 if ($type eq 'sum') {
714 push @
{$rules->{sum
}}, $name;
716 elsif ($type eq 'isbn') {
717 push @
{$rules->{isbn
}}, $name;
719 elsif ($type eq 'boolean') {
720 # boolean gets special handling, if value doesn't exist for a field,
722 $rules->{defaults
}->{$name} = 'false';
725 if ($marc_field =~ $field_spec_regexp) {
730 # Parse and separate subfields form subfield groups
732 my $subfield_group = '';
735 foreach my $token (split //, $2) {
738 Koha
::Exceptions
::Elasticsearch
::MARCFieldExprParseError
->throw(
739 "Unmatched opening parenthesis for $marc_field"
746 elsif ($token eq ")") {
748 if ($subfield_group) {
749 push @subfield_groups, $subfield_group;
750 $subfield_group = '';
755 Koha
::Exceptions
::Elasticsearch
::MARCFieldExprParseError
->throw(
756 "Unmatched closing parenthesis for $marc_field"
760 elsif ($open_group) {
761 $subfield_group .= $token;
764 push @subfields, $token;
769 push @subfields, '*';
772 my $range = defined $3 ?
$3 : undef;
773 my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range);
775 if ($field_tag < 10) {
776 $rules->{control_fields
}->{$field_tag} //= [];
777 push @
{$rules->{control_fields
}->{$field_tag}}, @mappings;
780 $rules->{data_fields
}->{$field_tag} //= {};
781 foreach my $subfield (@subfields) {
782 $rules->{data_fields
}->{$field_tag}->{subfields
}->{$subfield} //= [];
783 push @
{$rules->{data_fields
}->{$field_tag}->{subfields
}->{$subfield}}, @mappings;
785 foreach my $subfield_group (@subfield_groups) {
786 $rules->{data_fields
}->{$field_tag}->{subfields_join
}->{$subfield_group} //= [];
787 push @
{$rules->{data_fields
}->{$field_tag}->{subfields_join
}->{$subfield_group}}, @mappings;
791 elsif ($marc_field =~ $leader_regexp) {
792 my $range = defined $1 ?
$1 : undef;
793 my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range);
794 push @
{$rules->{leader
}}, @mappings;
797 Koha
::Exceptions
::Elasticsearch
::MARCFieldExprParseError
->throw(
798 "Invalid MARC field expression: $marc_field"
805 =head2 _foreach_mapping
807 $self->_foreach_mapping(
809 my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
812 return unless $marc_type eq 'marc21';
813 print "Data comes from: " . $marc_field . "\n";
817 This allows you to apply a function to each entry in the elasticsearch mappings
818 table, in order to build the mappings for whatever is needed.
820 In the provided function, the files are:
826 The field name for elasticsearch (corresponds to the 'mapping' column in the
831 The type for this value, e.g. 'string'.
835 True if this value should be facetised. This only really makes sense if the
836 field is understood by the facet processing code anyway.
840 True if this is a field that a) needs special sort handling, and b) if it
841 should be sorted on. False if a) but not b). Undef if not a). This allows,
842 for example, author to be sorted on but not everything marked with "author"
843 to be included in that sort.
847 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
848 'unimarc', 'normarc'.
852 A string that describes the MARC field that contains the data to extract.
853 These are of a form suited to Catmandu's MARC fixers.
859 sub _foreach_mapping
{
860 my ( $self, $sub ) = @_;
862 # TODO use a caching framework here
863 my $search_fields = Koha
::Database
->schema->resultset('SearchField')->search(
865 'search_marc_map.index_name' => $self->index,
867 { join => { search_marc_to_fields
=> 'search_marc_map' },
869 'search_marc_to_fields.facet',
870 'search_marc_to_fields.suggestible',
871 'search_marc_to_fields.sort',
872 'search_marc_map.marc_type',
873 'search_marc_map.marc_field',
885 while ( my $search_field = $search_fields->next ) {
887 # Force lower case on indexed field names for case insensitive
888 # field name searches
889 lc($search_field->name),
891 $search_field->get_column('facet'),
892 $search_field->get_column('suggestible'),
893 $search_field->get_column('sort'),
894 $search_field->get_column('marc_type'),
895 $search_field->get_column('marc_field'),
902 die process_error($@);
904 This parses an Elasticsearch error message and produces a human-readable
905 result from it. This result is probably missing all the useful information
906 that you might want in diagnosing an issue, so the warning is also logged.
908 Note that currently the resulting message is not internationalised. This
909 will happen eventually by some method or other.
914 my ($self, $msg) = @_;
916 warn $msg; # simple logging
918 # This is super-primitive
919 return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
921 return "Unable to perform your search. Please try again.\n";
924 =head2 _read_configuration
926 my $conf = _read_configuration();
928 Reads the I<configuration file> and returns a hash structure with the
929 configuration information. It raises an exception if mandatory entries
932 The hashref structure has the following form:
935 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
936 'index_name' => 'koha_instance',
939 This is configured by the following in the C<config> block in koha-conf.xml:
942 <server>127.0.0.1:9200</server>
943 <server>anotherserver:9200</server>
944 <index_name>koha_instance</index_name>
949 sub _read_configuration
{
953 my $conf = C4
::Context
->config('elasticsearch');
954 Koha
::Exceptions
::Config
::MissingEntry
->throw(
955 "Missing 'elasticsearch' block in config file")
956 unless defined $conf;
958 if ( $conf && $conf->{server
} ) {
959 my $nodes = $conf->{server
};
960 if ( ref($nodes) eq 'ARRAY' ) {
961 $configuration->{nodes
} = $nodes;
964 $configuration->{nodes
} = [$nodes];
968 Koha
::Exceptions
::Config
::MissingEntry
->throw(
969 "Missing 'server' entry in config file for elasticsearch");
972 if ( defined $conf->{index_name
} ) {
973 $configuration->{index_name
} = $conf->{index_name
};
976 Koha
::Exceptions
::Config
::MissingEntry
->throw(
977 "Missing 'index_name' entry in config file for elasticsearch");
980 return $configuration;
983 =head2 get_facetable_fields
985 my @facetable_fields = Koha::SearchEngine::Elasticsearch->get_facetable_fields();
987 Returns the list of Koha::SearchFields marked to be faceted in the ES configuration
991 sub get_facetable_fields
{
994 # These should correspond to the ES field names, as opposed to the CCL
995 # things that zebra uses.
996 my @search_field_names = qw( author itype location su-geo title-series subject ccode holdingbranch homebranch ln );
997 my @faceted_fields = Koha
::SearchFields
->search(
998 { name
=> { -in => \
@search_field_names }, facet_order
=> { '!=' => undef } }, { order_by
=> ['facet_order'] }
1000 my @not_faceted_fields = Koha
::SearchFields
->search(
1001 { name
=> { -in => \
@search_field_names }, facet_order
=> undef }, { order_by
=> ['facet_order'] }
1003 # This could certainly be improved
1004 return ( @faceted_fields, @not_faceted_fields );
1015 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
1017 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
1019 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>