3 # This file is part of Koha.
5 # Koha is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # Koha is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
20 use Test
::More tests
=> 6;
29 use List
::Util
qw( any );
31 use Koha
::SearchEngine
::Elasticsearch
;
32 use Koha
::SearchEngine
::Elasticsearch
::Search
;
34 subtest
'_read_configuration() tests' => sub {
39 t
::lib
::Mocks
::mock_config
( 'elasticsearch', undef );
41 # 'elasticsearch' missing in configuration
43 $configuration = Koha
::SearchEngine
::Elasticsearch
::_read_configuration
;
45 'Koha::Exceptions::Config::MissingEntry',
46 'Configuration problem, exception thrown';
49 "Missing <elasticsearch> entry in koha-conf.xml",
50 'Exception message is correct'
53 # 'elasticsearch' present but no 'server' entry
54 t
::lib
::Mocks
::mock_config
( 'elasticsearch', {} );
56 $configuration = Koha
::SearchEngine
::Elasticsearch
::_read_configuration
;
58 'Koha::Exceptions::Config::MissingEntry',
59 'Configuration problem, exception thrown';
62 "Missing <elasticsearch>/<server> entry in koha-conf.xml",
63 'Exception message is correct'
66 # 'elasticsearch' and 'server' entries present, but no 'index_name'
67 t
::lib
::Mocks
::mock_config
( 'elasticsearch', { server
=> 'a_server' } );
69 $configuration = Koha
::SearchEngine
::Elasticsearch
::_read_configuration
;
71 'Koha::Exceptions::Config::MissingEntry',
72 'Configuration problem, exception thrown';
75 "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
76 'Exception message is correct'
79 # Correct configuration, only one server
80 t
::lib
::Mocks
::mock_config
( 'elasticsearch', { server
=> 'a_server', index_name
=> 'index' } );
82 $configuration = Koha
::SearchEngine
::Elasticsearch
::_read_configuration
;
83 is
( $configuration->{index_name
}, 'index', 'Index configuration parsed correctly' );
84 is_deeply
( $configuration->{nodes
}, ['a_server'], 'Server configuration parsed correctly' );
86 # Correct configuration, two servers
87 my @servers = ('a_server', 'another_server');
88 t
::lib
::Mocks
::mock_config
( 'elasticsearch', { server
=> \
@servers, index_name
=> 'index' } );
90 $configuration = Koha
::SearchEngine
::Elasticsearch
::_read_configuration
;
91 is
( $configuration->{index_name
}, 'index', 'Index configuration parsed correctly' );
92 is_deeply
( $configuration->{nodes
}, \
@servers , 'Server configuration parsed correctly' );
95 subtest
'get_elasticsearch_settings() tests' => sub {
101 # test reading index settings
102 my $es = Koha
::SearchEngine
::Elasticsearch
->new( {index => $Koha::SearchEngine
::Elasticsearch
::BIBLIOS_INDEX
} );
103 $settings = $es->get_elasticsearch_settings();
104 is
( $settings->{index}{analysis
}{analyzer
}{analyzer_phrase
}{tokenizer
}, 'keyword', 'Index settings parsed correctly' );
107 subtest
'get_elasticsearch_mappings() tests' => sub {
113 # test reading mappings
114 my $es = Koha
::SearchEngine
::Elasticsearch
->new( {index => $Koha::SearchEngine
::Elasticsearch
::BIBLIOS_INDEX
} );
115 $mappings = $es->get_elasticsearch_mappings();
116 is
( $mappings->{data
}{properties
}{isbn__sort
}{index}, 'false', 'Field mappings parsed correctly' );
119 subtest
'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
123 t
::lib
::Mocks
::mock_preference
('marcflavour', 'MARC21');
124 t
::lib
::Mocks
::mock_preference
('ElasticsearchMARCFormat', 'ISO2709');
128 name
=> 'control_number',
134 marc_type
=> 'marc21',
144 marc_type
=> 'marc21',
145 marc_field
=> '020a',
154 marc_type
=> 'marc21',
155 marc_field
=> '100a',
164 marc_type
=> 'marc21',
165 marc_field
=> '110a',
174 marc_type
=> 'marc21',
175 marc_field
=> '245(ab)ab',
178 name
=> 'unimarc_title',
184 marc_type
=> 'unimarc',
185 marc_field
=> '245a',
191 suggestible
=> undef,
194 marc_type
=> 'marc21',
198 name
=> 'uniform_title',
204 marc_type
=> 'marc21',
205 marc_field
=> '240a',
208 name
=> 'title_wildcard',
214 marc_type
=> 'marc21',
218 name
=> 'sum_item_price',
224 marc_type
=> 'marc21',
225 marc_field
=> '952g',
228 name
=> 'items_withdrawn_status',
234 marc_type
=> 'marc21',
235 marc_field
=> '9520',
238 name
=> 'local_classification',
244 marc_type
=> 'marc21',
245 marc_field
=> '952o',
248 name
=> 'type_of_record',
254 marc_type
=> 'marc21',
255 marc_field
=> 'leader_/6',
258 name
=> 'type_of_record_and_bib_level',
264 marc_type
=> 'marc21',
265 marc_field
=> 'leader_/6-7',
274 marc_type
=> 'marc21',
275 marc_field
=> '007_/0',
284 marc_type
=> 'marc21',
285 marc_field
=> '952l',
289 my $se = Test
::MockModule
->new('Koha::SearchEngine::Elasticsearch');
290 $se->mock('_foreach_mapping', sub {
291 my ($self, $sub) = @_;
293 foreach my $map (@mappings) {
307 my $see = Koha
::SearchEngine
::Elasticsearch
::Search
->new({ index => $Koha::SearchEngine
::Elasticsearch
::BIBLIOS_INDEX
});
309 my $callno = 'ABC123';
310 my $callno2 = 'ABC456';
311 my $long_callno = '1234567890' x
30;
313 my $marc_record_1 = MARC
::Record
->new();
314 $marc_record_1->leader(' cam 22 a 4500');
315 $marc_record_1->append_fields(
316 MARC
::Field
->new('001', '123'),
317 MARC
::Field
->new('007', 'ku'),
318 MARC
::Field
->new('020', '', '', a
=> '1-56619-909-3'),
319 MARC
::Field
->new('100', '', '', a
=> 'Author 1'),
320 MARC
::Field
->new('110', '', '', a
=> 'Corp Author'),
321 MARC
::Field
->new('210', '', '', a
=> 'Title 1'),
322 MARC
::Field
->new('240', '', '4', a
=> 'The uniform title with nonfiling indicator'),
323 MARC
::Field
->new('245', '', '', a
=> 'Title:', b
=> 'first record'),
324 MARC
::Field
->new('999', '', '', c
=> '1234567'),
325 # ' ' for testing trimming of white space in boolean value callback:
326 MARC
::Field
->new('952', '', '', 0 => ' ', g
=> '123.30', o
=> $callno, l
=> 3),
327 MARC
::Field
->new('952', '', '', 0 => 0, g
=> '127.20', o
=> $callno2, l
=> 2),
328 MARC
::Field
->new('952', '', '', 0 => 1, g
=> '0.00', o
=> $long_callno, l
=> 1),
330 my $marc_record_2 = MARC
::Record
->new();
331 $marc_record_2->leader(' cam 22 a 4500');
332 $marc_record_2->append_fields(
333 MARC
::Field
->new('100', '', '', a
=> 'Author 2'),
334 # MARC::Field->new('210', '', '', a => 'Title 2'),
335 # MARC::Field->new('245', '', '', a => 'Title: second record'),
336 MARC
::Field
->new('999', '', '', c
=> '1234568'),
337 MARC
::Field
->new('952', '', '', 0 => 1, g
=> 'string where should be numeric', o
=> $long_callno),
339 my $records = [$marc_record_1, $marc_record_2];
341 $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
343 my $docs = $see->marc_records_to_documents($records);
346 is
(scalar @
{$docs}, 2, 'Two records converted to documents');
348 is_deeply
($docs->[0]->{control_number
}, ['123'], 'First record control number should be set correctly');
350 is_deeply
($docs->[0]->{'ff7-00'}, ['k'], 'First record ff7-00 should be set correctly');
352 is
(scalar @
{$docs->[0]->{author
}}, 2, 'First document author field should contain two values');
353 is_deeply
($docs->[0]->{author
}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
355 is
(scalar @
{$docs->[0]->{author__sort
}}, 1, 'First document author__sort field should have a single value');
356 is_deeply
($docs->[0]->{author__sort
}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly');
358 is
(scalar @
{$docs->[0]->{title__sort
}}, 1, 'First document title__sort field should have a single');
359 is_deeply
($docs->[0]->{title__sort
}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
361 is
($docs->[0]->{issues
}, 6, 'Issues field should be sum of the issues for each item');
362 is
($docs->[0]->{issues__sort
}, 6, 'Issues sort field should also be a sum of the issues');
364 is
(scalar @
{$docs->[0]->{title_wildcard
}}, 2, 'First document title_wildcard field should have two values');
365 is_deeply
($docs->[0]->{title_wildcard
}, ['Title:', 'first record'], 'First document title_wildcard field should be set correctly');
368 is
(scalar @
{$docs->[0]->{author__suggestion
}}, 2, 'First document author__suggestion field should contain two values');
370 $docs->[0]->{author__suggestion
},
373 'input' => 'Author 1'
376 'input' => 'Corp Author'
379 'First document author__suggestion field should be set correctly'
382 is
(scalar @
{$docs->[0]->{title__suggestion
}}, 3, 'First document title__suggestion field should contain three values');
384 $docs->[0]->{title__suggestion
},
386 { 'input' => 'Title:' },
387 { 'input' => 'first record' },
388 { 'input' => 'Title: first record' }
390 'First document title__suggestion field should be set correctly'
393 ok
(!(defined $docs->[0]->{title__facet
}), 'First document should have no title__facet field');
395 is
(scalar @
{$docs->[0]->{author__facet
}}, 2, 'First document author__facet field should have two values');
397 $docs->[0]->{author__facet
},
398 ['Author 1', 'Corp Author'],
399 'First document author__facet field should be set correctly'
402 is
(scalar @
{$docs->[0]->{items_withdrawn_status
}}, 2, 'First document items_withdrawn_status field should have two values');
404 $docs->[0]->{items_withdrawn_status
},
406 'First document items_withdrawn_status field should be set correctly'
410 $docs->[0]->{sum_item_price
},
412 'First document sum_item_price field should be set correctly'
415 ok
(defined $docs->[0]->{marc_data
}, 'First document marc_data field should be set');
416 ok
(defined $docs->[0]->{marc_format
}, 'First document marc_format field should be set');
417 is
($docs->[0]->{marc_format
}, 'base64ISO2709', 'First document marc_format should be set correctly');
419 my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
421 ok
($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
422 is
($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
424 is
(scalar @
{$docs->[0]->{type_of_record
}}, 1, 'First document type_of_record field should have one value');
426 $docs->[0]->{type_of_record
},
428 'First document type_of_record field should be set correctly'
431 is
(scalar @
{$docs->[0]->{type_of_record_and_bib_level
}}, 1, 'First document type_of_record_and_bib_level field should have one value');
433 $docs->[0]->{type_of_record_and_bib_level
},
435 'First document type_of_record_and_bib_level field should be set correctly'
438 is
(scalar @
{$docs->[0]->{isbn
}}, 4, 'First document isbn field should contain four values');
439 is_deeply
($docs->[0]->{isbn
}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
442 $docs->[0]->{'local_classification'},
443 [$callno, $callno2, $long_callno],
444 'First document local_classification field should be set correctly'
447 # Nonfiling characters for sort fields
449 $docs->[0]->{uniform_title
},
450 ['The uniform title with nonfiling indicator'],
451 'First document uniform_title field should contain the title verbatim'
454 $docs->[0]->{uniform_title__sort
},
455 ['uniform title with nonfiling indicator'],
456 'First document uniform_title__sort field should contain the title with the first four initial characters removed'
461 is
(scalar @
{$docs->[1]->{author
}}, 1, 'Second document author field should contain one value');
462 is_deeply
($docs->[1]->{author
}, ['Author 2'], 'Second document author field should be set correctly');
464 is
(scalar @
{$docs->[1]->{items_withdrawn_status
}}, 1, 'Second document items_withdrawn_status field should have one value');
466 $docs->[1]->{items_withdrawn_status
},
468 'Second document items_withdrawn_status field should be set correctly'
472 $docs->[1]->{sum_item_price
},
474 'Second document sum_item_price field should be set correctly'
478 $docs->[1]->{local_classification__sort
},
479 [substr($long_callno, 0, 255)],
480 'Second document local_classification__sort field should be set correctly'
483 # Mappings marc_type:
485 ok
(!(defined $docs->[0]->{unimarc_title
}), "No mapping when marc_type doesn't match marc flavour");
487 # Marc serialization format fallback for records exceeding ISO2709 max record size
489 my $large_marc_record = MARC
::Record
->new();
490 $large_marc_record->leader(' cam 22 a 4500');
492 $large_marc_record->append_fields(
493 MARC
::Field
->new('100', '', '', a
=> 'Author 1'),
494 MARC
::Field
->new('110', '', '', a
=> 'Corp Author'),
495 MARC
::Field
->new('210', '', '', a
=> 'Title 1'),
496 MARC
::Field
->new('245', '', '', a
=> 'Title:', b
=> 'large record'),
497 MARC
::Field
->new('999', '', '', c
=> '1234567'),
500 my $item_field = MARC
::Field
->new('952', '', '', o
=> '123456789123456789123456789', p
=> '123456789', z
=> 'test');
501 my $items_count = 1638;
502 while(--$items_count) {
503 $large_marc_record->append_fields($item_field);
506 $docs = $see->marc_records_to_documents([$large_marc_record]);
508 is
($docs->[0]->{marc_format
}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
510 $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
512 ok
($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
513 is
($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
521 marc_type
=> 'marc21',
522 marc_field
=> '245((ab)ab',
525 my $exception = try
{
526 $see->marc_records_to_documents($records);
532 ok
(defined $exception, "Exception has been thrown when processing mapping with unmatched opening parenthesis");
533 ok
($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
534 ok
($exception->message =~ /Unmatched opening parenthesis/, "Exception has the correct message");
543 marc_type
=> 'marc21',
544 marc_field
=> '245(ab))ab',
548 $see->marc_records_to_documents($records);
554 ok
(defined $exception, "Exception has been thrown when processing mapping with unmatched closing parenthesis");
555 ok
($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
556 ok
($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
559 subtest
'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
563 t
::lib
::Mocks
::mock_preference
('marcflavour', 'MARC21');
564 t
::lib
::Mocks
::mock_preference
('ElasticsearchMARCFormat', 'ARRAY');
568 name
=> 'control_number',
574 marc_type
=> 'marc21',
579 my $se = Test
::MockModule
->new('Koha::SearchEngine::Elasticsearch');
580 $se->mock('_foreach_mapping', sub {
581 my ($self, $sub) = @_;
583 foreach my $map (@mappings) {
597 my $see = Koha
::SearchEngine
::Elasticsearch
::Search
->new({ index => $Koha::SearchEngine
::Elasticsearch
::BIBLIOS_INDEX
});
599 my $marc_record_1 = MARC
::Record
->new();
600 $marc_record_1->leader(' cam 22 a 4500');
601 $marc_record_1->append_fields(
602 MARC
::Field
->new('001', '123'),
603 MARC
::Field
->new('020', '', '', a
=> '1-56619-909-3'),
604 MARC
::Field
->new('100', '', '', a
=> 'Author 1'),
605 MARC
::Field
->new('110', '', '', a
=> 'Corp Author'),
606 MARC
::Field
->new('210', '', '', a
=> 'Title 1'),
607 MARC
::Field
->new('245', '', '', a
=> 'Title:', b
=> 'first record'),
608 MARC
::Field
->new('999', '', '', c
=> '1234567'),
610 my $marc_record_2 = MARC
::Record
->new();
611 $marc_record_2->leader(' cam 22 a 4500');
612 $marc_record_2->append_fields(
613 MARC
::Field
->new('100', '', '', a
=> 'Author 2'),
614 # MARC::Field->new('210', '', '', a => 'Title 2'),
615 # MARC::Field->new('245', '', '', a => 'Title: second record'),
616 MARC
::Field
->new('999', '', '', c
=> '1234568'),
617 MARC
::Field
->new('952', '', '', 0 => 1, g
=> 'string where should be numeric'),
619 my $records = [$marc_record_1, $marc_record_2];
621 $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
623 my $docs = $see->marc_records_to_documents($records);
626 is
(scalar @
{$docs}, 2, 'Two records converted to documents');
628 is_deeply
($docs->[0]->{control_number
}, ['123'], 'First record control number should be set correctly');
630 is
($docs->[0]->{marc_format
}, 'ARRAY', 'First document marc_format should be set correctly');
632 my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
634 ok
($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
635 is
($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
638 subtest
'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authority tests' => sub {
642 t
::lib
::Mocks
::mock_preference
('marcflavour', 'MARC21');
643 t
::lib
::Mocks
::mock_preference
('ElasticsearchMARCFormat', 'ISO2709');
653 marc_type
=> 'marc21',
654 marc_field
=> '150(ae)',
663 marc_type
=> 'marc21',
664 marc_field
=> '150a',
673 marc_type
=> 'marc21',
674 marc_field
=> '150(ae)',
677 name
=> 'heading-main',
683 marc_type
=> 'marc21',
684 marc_field
=> '150a',
693 marc_type
=> 'marc21',
697 name
=> 'match-heading',
703 marc_type
=> 'marc21',
708 my $se = Test
::MockModule
->new('Koha::SearchEngine::Elasticsearch');
709 $se->mock('_foreach_mapping', sub {
710 my ($self, $sub) = @_;
712 foreach my $map (@mappings) {
726 my $see = Koha
::SearchEngine
::Elasticsearch
::Search
->new({ index => $Koha::SearchEngine
::Elasticsearch
::AUTHORITIES_INDEX
});
727 my $marc_record_1 = MARC
::Record
->new();
728 $marc_record_1->append_fields(
729 MARC
::Field
->new('001', '123'),
730 MARC
::Field
->new('007', 'ku'),
731 MARC
::Field
->new('020', '', '', a
=> '1-56619-909-3'),
732 MARC
::Field
->new('150', '', '', a
=> 'Subject', v
=> 'Genresubdiv', x
=> 'Generalsubdiv', z
=> 'Geosubdiv'),
734 my $marc_record_2 = MARC
::Record
->new();
735 $marc_record_2->append_fields(
736 MARC
::Field
->new('150', '', '', a
=> 'Subject', v
=> 'Genresubdiv', z
=> 'Geosubdiv', x
=> 'Generalsubdiv', e
=> 'wrongsubdiv' ),
738 my $records = [$marc_record_1, $marc_record_2];
740 $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
742 my $docs = $see->marc_records_to_documents($records);
745 any
{ $_ eq "Subject formsubdiv Genresubdiv generalsubdiv Generalsubdiv geographicsubdiv Geosubdiv" }
746 @
{$docs->[0]->{'match-heading'}},
747 "First record match-heading should contain the correctly formatted heading"
750 any
{ $_ eq "Subject formsubdiv Genresubdiv geographicsubdiv Geosubdiv generalsubdiv Generalsubdiv" }
751 @
{$docs->[1]->{'match-heading'}},
752 "Second record match-heading should contain the correctly formatted heading without wrong subfield"