1 package Koha
::SearchEngine
::Elasticsearch
::Search
;
3 # Copyright 2014 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 Koha::SearchEngine::Elasticsearch::Search - search functions for Elasticsearch
27 Koha::SearchEngine::Elasticsearch::Search->new( { index => $index } );
28 my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new(
29 { index => $index } );
30 my $query = $builder->build_query('perl');
31 my $results = $searcher->search($query);
32 print "There were " . $results->total . " results.\n";
43 use base
qw(Koha::SearchEngine::Elasticsearch);
46 use Koha
::AuthorisedValues
;
47 use Koha
::SearchEngine
::QueryBuilder
;
48 use Koha
::SearchEngine
::Search
;
50 use Catmandu
::Store
::ElasticSearch
;
52 use Data
::Dumper
; #TODO remove
55 Koha
::SearchEngine
::Elasticsearch
::Search
->mk_accessors(qw( store ));
59 my $results = $searcher->search($query, $page, $count, %options);
61 Run a search using the query. It'll return C<$count> results, starting at page
62 C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)
63 C<$count> is also the number of entries on a page.
65 C<%options> is a hash containing extra options:
71 If provided, this overrides the C<$page> value, and specifies the record as
72 an offset (i.e. the number of the record to start with), rather than a page.
81 my ($self, $query, $page, $count, %options) = @_;
83 my $params = $self->get_elasticsearch_params();
85 # 20 is the default number of results per page
86 $paging{limit
} = $count || 20;
87 # ES/Catmandu doesn't want pages, it wants a record to start from.
88 if (exists $options{offset
}) {
89 $paging{start
} = $options{offset
};
91 $page = (!defined($page) || ($page <= 0)) ?
0 : $page - 1;
92 $paging{start
} = $page * $paging{limit
};
95 Catmandu
::Store
::ElasticSearch
->new(
98 ) unless $self->store;
100 $self->store->bag->search( %$query, %paging );
103 die $self->process_error($@
);
110 my $count = $searcher->count($query);
112 This mimics a search request, but just gets the result count instead. That's
113 faster than pulling all the data in, usually.
118 my ( $self, $query ) = @_;
120 my $params = $self->get_elasticsearch_params();
122 Catmandu
::Store
::ElasticSearch
->new( %$params, trace_calls
=> 0, ) )
125 my $search = $self->store->bag->search( %$query);
126 my $count = $search->total() || 0;
132 my ( $error, $results, $facets ) = $search->search_compat(
133 $query, $simple_query, \@sort_by, \@servers,
134 $results_per_page, $offset, $expanded_facet, $branches,
138 A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
139 that is returned in the query created by build_query_compat will probably
140 get ignored here, along with some other things (like C<@servers>.)
146 $self, $query, $simple_query, $sort_by,
147 $servers, $results_per_page, $offset, $expanded_facet,
148 $branches, $query_type, $scan
151 if ( !defined $offset or $offset < 0 ) {
154 $options{offset
} = $offset;
155 $options{expanded_facet
} = $expanded_facet;
156 my $results = $self->search($query, undef, $results_per_page, %options);
158 # Convert each result into a MARC::Record
159 my (@records, $index);
160 $index = $offset; # opac-search expects results to be put in the
161 # right place in the array, according to $offset
163 # The results come in an array for some reason
164 my $marc_json = $_[0]->{record
};
165 my $marc = $self->json2marc($marc_json);
166 $records[$index++] = $marc;
168 # consumers of this expect a name-spaced result, we provide the default
171 $result{biblioserver
}{hits
} = $results->total;
172 $result{biblioserver
}{RECORDS
} = \
@records;
173 return (undef, \
%result, $self->_convert_facets($results->{aggregations
}, $expanded_facet));
176 =head2 search_auth_compat
178 my ( $results, $total ) =
179 $searcher->search_auth_compat( $query, $page, $count, %options );
181 This has a similar calling convention to L<search>, however it returns its
182 results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
186 sub search_auth_compat
{
190 my $database = Koha
::Database
->new();
191 my $schema = $database->schema();
192 my $res = $self->search(@_);
193 my $bib_searcher = Koha
::SearchEngine
::Elasticsearch
::Search
->new({index => 'biblios'});
199 my $marc_json = $record->{record
};
201 # I wonder if these should be real values defined in the mapping
202 # rather than hard-coded conversions.
203 # Handle legacy nested arrays indexed with splitting enabled.
204 my $authid = $record->{ 'Local-number' }[0];
205 $authid = @
$authid[0] if (ref $authid eq 'ARRAY');
206 $result{authid
} = $authid;
208 # TODO put all this info into the record at index time so we
209 # don't have to go and sort it all out now.
210 my $authtypecode = $record->{authtype
};
211 my $rs = $schema->resultset('AuthType')
212 ->search( { authtypecode
=> $authtypecode } );
214 # FIXME there's an assumption here that we will get a result.
215 # the original code also makes an assumption that some provided
216 # authtypecode may sometimes be used instead of the one stored
217 # with the record. It's not documented why this is the case, so
218 # it's not reproduced here yet.
219 my $authtype = $rs->single;
220 my $auth_tag_to_report = $authtype ?
$authtype->auth_tag_to_report : "";
221 my $marc = $self->json2marc($marc_json);
222 my $mainentry = $marc->field($auth_tag_to_report);
225 foreach ( $mainentry->subfields() ) {
226 $reported_tag .= '$' . $_->[0] . $_->[1];
229 # Turn the resultset into a hash
230 $result{authtype
} = $authtype ?
$authtype->authtypetext : $authtypecode;
231 $result{reported_tag
} = $reported_tag;
233 # Reimplementing BuildSummary is out of scope because it'll be hard
235 C4
::AuthoritiesMarc
::BuildSummary
( $marc, $result{authid
},
237 $result{used
} = $self->count_auth_use($bib_searcher, $authid);
238 push @records, \
%result;
241 return ( \
@records, $res->total );
244 =head2 count_auth_use
246 my $count = $auth_searcher->count_auth_use($bib_searcher, $authid);
248 This runs a search to determine the number of records that reference the
249 specified authid. C<$bib_searcher> must be something compatible with
250 elasticsearch, as the query is built in this function.
255 my ($self, $bib_searcher, $authid) = @_;
260 # query => { match_all => {} },
261 filter
=> { term
=> { an
=> $authid } }
265 $bib_searcher->count($query);
268 =head2 simple_search_compat
270 my ( $error, $marcresults, $total_hits ) =
271 $searcher->simple_search( $query, $offset, $max_results, %options );
273 This is a simpler interface to the searching, intended to be similar enough to
274 L<C4::Search::SimpleSearch>.
282 A thing to search for. It could be a simple string, or something constructed
283 with the appropriate QueryBuilder module.
287 How many results to skip from the start of the results.
289 =item C<$max_results>
291 The max number of results to return. The default is 100 (because unlimited
292 is a pretty terrible thing to do.)
296 These options are unused by Elasticsearch
306 if something went wrong, this'll contain some kind of error
309 =item C<$marcresults>
311 an arrayref of MARC::Records (note that this is different from the
312 L<C4::Search> version which will return plain XML, but too bad.)
316 the total number of results that this search could have returned.
322 sub simple_search_compat
{
323 my ($self, $query, $offset, $max_results) = @_;
325 return ('No query entered', undef, undef) unless $query;
328 $offset = 0 if not defined $offset or $offset < 0;
329 $options{offset
} = $offset;
330 $max_results //= 100;
332 unless (ref $query) {
333 # We'll push it through the query builder to sanitise everything.
334 my $qb = Koha
::SearchEngine
::QueryBuilder
->new({index => $self->index});
335 (undef,$query) = $qb->build_query_compat(undef, [$query]);
337 my $results = $self->search($query, undef, $max_results, %options);
340 # The results come in an array for some reason
341 my $marc_json = $_[0]->{record
};
342 my $marc = $self->json2marc($marc_json);
343 push @records, $marc;
345 return (undef, \
@records, $results->total);
348 =head2 extract_biblionumber
350 my $biblionumber = $searcher->extract_biblionumber( $searchresult );
352 $searchresult comes from simple_search_compat.
354 Returns the biblionumber from the search result record.
358 sub extract_biblionumber
{
359 my ( $self, $searchresultrecord ) = @_;
360 return Koha
::SearchEngine
::Search
::extract_biblionumber
( $searchresultrecord );
365 my $marc = $self->json2marc($marc_json);
367 Converts the form of marc (based on its JSON, but as a Perl structure) that
368 Catmandu stores into a MARC::Record object.
373 my ( $self, $marcjson ) = @_;
375 my $marc = MARC
::Record
->new();
376 $marc->encoding('UTF-8');
379 # [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
381 # [ '001', undef, undef, '_', 'a value' ]
382 # conveniently, this is the form that MARC::Field->new() likes
383 foreach my $field (@
$marcjson) {
385 if ( $field->[0] eq 'LDR' ) {
386 $marc->leader( $field->[4] );
389 my $tag = $field->[0];
391 if ( MARC
::Field
->is_controlfield_tag( $field->[0] ) ) {
392 $marc_field = MARC
::Field
->new($field->[0], $field->[4]);
394 $marc_field = MARC
::Field
->new(@
$field);
396 $marc->append_fields($marc_field);
402 =head2 max_result_window
404 Returns the maximum number of results that can be fetched
406 This directly requests Elasticsearch for the setting index.max_result_window (or
407 the default value for this setting in case it is not set)
411 sub max_result_window
{
415 Catmandu
::Store
::ElasticSearch
->new(%{ $self->get_elasticsearch_params })
416 ) unless $self->store;
418 my $index_name = $self->store->index_name;
419 my $settings = $self->store->es->indices->get_settings(
420 index => $index_name,
421 params
=> { include_defaults
=> 1, flat_settings
=> 1 },
424 my $max_result_window = $settings->{$index_name}->{settings
}->{'index.max_result_window'};
425 $max_result_window //= $settings->{$index_name}->{defaults
}->{'index.max_result_window'};
427 return $max_result_window;
430 =head2 _convert_facets
432 my $koha_facets = _convert_facets($es_facets, $expanded_facet);
434 Converts elasticsearch facets types to the form that Koha expects.
435 It expects the ES facet name to match the Koha type, for example C<itype>,
436 C<au>, C<su-to>, etc.
438 C<$expanded_facet> is the facet that we want to show FacetMaxCount entries for, rather
439 than just 5 like normal.
443 sub _convert_facets
{
444 my ( $self, $es, $exp_facet ) = @_;
448 # These should correspond to the ES field names, as opposed to the CCL
449 # things that zebra uses.
450 # TODO let the library define the order using the interface.
451 my %type_to_label = (
452 author
=> { order
=> 1, label
=> 'Authors', },
453 itype
=> { order
=> 2, label
=> 'ItemTypes', },
454 location
=> { order
=> 3, label
=> 'Location', },
455 'su-geo' => { order
=> 4, label
=> 'Places', },
456 se
=> { order
=> 5, label
=> 'Series', },
457 subject
=> { order
=> 6, label
=> 'Topics', },
458 ccode
=> { order
=> 7, label
=> 'CollectionCodes',},
459 holdingbranch
=> { order
=> 8, label
=> 'HoldingLibrary' },
460 homebranch
=> { order
=> 9, label
=> 'HomeLibrary' }
463 # We also have some special cases, e.g. itypes that need to show the
464 # value rather than the code.
465 my @itypes = Koha
::ItemTypes
->search;
466 my @libraries = Koha
::Libraries
->search;
467 my $library_names = { map { $_->branchcode => $_->branchname } @libraries };
468 my @locations = Koha
::AuthorisedValues
->search( { category
=> 'LOC' } );
469 my $opac = C4
::Context
->interface eq 'opac' ;
471 itype
=> { map { $_->itemtype => $_->description } @itypes },
472 location
=> { map { $_->authorised_value => ( $opac ?
( $_->lib_opac || $_->lib ) : $_->lib ) } @locations },
473 holdingbranch
=> $library_names,
474 homebranch
=> $library_names
478 while ( my ( $type, $data ) = each %$es ) {
479 next if !exists( $type_to_label{$type} );
481 # We restrict to the most popular $limit !results
482 my $limit = ( $type eq $exp_facet ) ? C4
::Context
->preference('FacetMaxCount') : 5;
484 type_id
=> $type . '_id',
486 expandable
=> ( $type ne $exp_facet )
487 && ( @
{ $data->{buckets
} } > $limit ),
488 "type_label_$type_to_label{$type}{label}" => 1,
489 type_link_value
=> $type,
490 order
=> $type_to_label{$type}{order
},
492 $limit = @
{ $data->{buckets
} } if ( $limit > @
{ $data->{buckets
} } );
493 foreach my $term ( @
{ $data->{buckets
} }[ 0 .. $limit - 1 ] ) {
494 my $t = $term->{key
};
495 my $c = $term->{doc_count
};
497 if ( exists( $special{$type} ) ) {
498 $label = $special{$type}->{$t} // $t;
503 push @
{ $facet->{facets
} }, {
505 facet_link_value
=> $t,
506 facet_title_value
=> $t . " ($c)",
507 facet_label_value
=> $label, # TODO either truncate this,
508 # or make the template do it like it should anyway
509 type_link_value
=> $type,
512 push @facets, $facet if exists $facet->{facets
};
515 @facets = sort { $a->{order
} cmp $b->{order
} } @facets;