t/Koha/SearchEngine/Elasticsearch.t

   1 #!/usr/bin/perl
   2 #
   3 # This file is part of Koha.
   4 #
   5 # Koha is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # Koha is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  17
  18 use Modern::Perl;
  19
  20 use Test::More tests => 5;
  21 use Test::Exception;
  22
  23 use t::lib::Mocks;
  24
  25 use Test::MockModule;
  26
  27 use MARC::Record;
  28 use Try::Tiny;
  29
  30 use Koha::SearchEngine::Elasticsearch;
  31 use Koha::SearchEngine::Elasticsearch::Search;
  32
  33 subtest '_read_configuration() tests' => sub {
  34
  35     plan tests => 10;
  36
  37     my $configuration;
  38     t::lib::Mocks::mock_config( 'elasticsearch', undef );
  39
  40     # 'elasticsearch' missing in configuration
  41     throws_ok {
  42         $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  43     }
  44     'Koha::Exceptions::Config::MissingEntry',
  45       'Configuration problem, exception thrown';
  46     is(
  47         $@->message,
  48         "Missing 'elasticsearch' block in config file",
  49         'Exception message is correct'
  50     );
  51
  52     # 'elasticsearch' present but no 'server' entry
  53     t::lib::Mocks::mock_config( 'elasticsearch', {} );
  54     throws_ok {
  55         $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  56     }
  57     'Koha::Exceptions::Config::MissingEntry',
  58       'Configuration problem, exception thrown';
  59     is(
  60         $@->message,
  61         "Missing 'server' entry in config file for elasticsearch",
  62         'Exception message is correct'
  63     );
  64
  65     # 'elasticsearch' and 'server' entries present, but no 'index_name'
  66     t::lib::Mocks::mock_config( 'elasticsearch', { server => 'a_server' } );
  67     throws_ok {
  68         $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  69     }
  70     'Koha::Exceptions::Config::MissingEntry',
  71       'Configuration problem, exception thrown';
  72     is(
  73         $@->message,
  74         "Missing 'index_name' entry in config file for elasticsearch",
  75         'Exception message is correct'
  76     );
  77
  78     # Correct configuration, only one server
  79     t::lib::Mocks::mock_config( 'elasticsearch',  { server => 'a_server', index_name => 'index' } );
  80
  81     $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  82     is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
  83     is_deeply( $configuration->{nodes}, ['a_server'], 'Server configuration parsed correctly' );
  84
  85     # Correct configuration, two servers
  86     my @servers = ('a_server', 'another_server');
  87     t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index' } );
  88
  89     $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  90     is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
  91     is_deeply( $configuration->{nodes}, \@servers , 'Server configuration parsed correctly' );
  92 };
  93
  94 subtest 'get_elasticsearch_settings() tests' => sub {
  95
  96     plan tests => 1;
  97
  98     my $settings;
  99
 100     # test reading index settings
 101     my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
 102     $settings = $es->get_elasticsearch_settings();
 103     is( $settings->{index}{analysis}{analyzer}{analyzer_phrase}{tokenizer}, 'keyword', 'Index settings parsed correctly' );
 104 };
 105
 106 subtest 'get_elasticsearch_mappings() tests' => sub {
 107
 108     plan tests => 1;
 109
 110     my $mappings;
 111
 112     # test reading mappings
 113     my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
 114     $mappings = $es->get_elasticsearch_mappings();
 115     is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
 116 };
 117
 118 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
 119
 120     plan tests => 51;
 121
 122     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
 123     t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
 124
 125     my @mappings = (
 126         {
 127             name => 'control_number',
 128             type => 'string',
 129             facet => 0,
 130             suggestible => 0,
 131             searchable => 1,
 132             sort => undef,
 133             marc_type => 'marc21',
 134             marc_field => '001',
 135         },
 136         {
 137             name => 'isbn',
 138             type => 'isbn',
 139             facet => 0,
 140             suggestible => 0,
 141             searchable => 1,
 142             sort => 0,
 143             marc_type => 'marc21',
 144             marc_field => '020a',
 145         },
 146         {
 147             name => 'author',
 148             type => 'string',
 149             facet => 1,
 150             suggestible => 1,
 151             searchable => 1,
 152             sort => undef,
 153             marc_type => 'marc21',
 154             marc_field => '100a',
 155         },
 156         {
 157             name => 'author',
 158             type => 'string',
 159             facet => 1,
 160             suggestible => 1,
 161             searchable => 1,
 162             sort => 1,
 163             marc_type => 'marc21',
 164             marc_field => '110a',
 165         },
 166         {
 167             name => 'title',
 168             type => 'string',
 169             facet => 0,
 170             suggestible => 1,
 171             searchable => 1,
 172             sort => 1,
 173             marc_type => 'marc21',
 174             marc_field => '245(ab)ab',
 175         },
 176         {
 177             name => 'unimarc_title',
 178             type => 'string',
 179             facet => 0,
 180             suggestible => 1,
 181             searchable => 1,
 182             sort => 1,
 183             marc_type => 'unimarc',
 184             marc_field => '245a',
 185         },
 186         {
 187             name => 'title',
 188             type => 'string',
 189             facet => 0,
 190             suggestible => undef,
 191             searchable => 1,
 192             sort => 0,
 193             marc_type => 'marc21',
 194             marc_field => '220',
 195         },
 196         {
 197             name => 'title_wildcard',
 198             type => 'string',
 199             facet => 0,
 200             suggestible => 0,
 201             searchable => 1,
 202             sort => undef,
 203             marc_type => 'marc21',
 204             marc_field => '245',
 205         },
 206         {
 207             name => 'sum_item_price',
 208             type => 'sum',
 209             facet => 0,
 210             suggestible => 0,
 211             searchable => 1,
 212             sort => 0,
 213             marc_type => 'marc21',
 214             marc_field => '952g',
 215         },
 216         {
 217             name => 'items_withdrawn_status',
 218             type => 'boolean',
 219             facet => 0,
 220             suggestible => 0,
 221             searchable => 1,
 222             sort => 0,
 223             marc_type => 'marc21',
 224             marc_field => '9520',
 225         },
 226         {
 227             name => 'local_classification',
 228             type => 'string',
 229             facet => 0,
 230             suggestible => 0,
 231             searchable => 1,
 232             sort => 1,
 233             marc_type => 'marc21',
 234             marc_field => '952o',
 235         },
 236         {
 237             name => 'type_of_record',
 238             type => 'string',
 239             facet => 0,
 240             suggestible => 0,
 241             searchable => 1,
 242             sort => 0,
 243             marc_type => 'marc21',
 244             marc_field => 'leader_/6',
 245         },
 246         {
 247             name => 'type_of_record_and_bib_level',
 248             type => 'string',
 249             facet => 0,
 250             suggestible => 0,
 251             searchable => 1,
 252             sort => 0,
 253             marc_type => 'marc21',
 254             marc_field => 'leader_/6-7',
 255         },
 256         {
 257             name => 'ff7-00',
 258             type => 'string',
 259             facet => 0,
 260             suggestible => 0,
 261             searchable => 1,
 262             sort => 0,
 263             marc_type => 'marc21',
 264             marc_field => '007_/0',
 265         },
 266         {
 267             name => 'issues',
 268             type => 'sum',
 269             facet => 0,
 270             suggestible => 0,
 271             searchable => 1,
 272             sort => 1,
 273             marc_type => 'marc21',
 274             marc_field => '952l',
 275         },
 276     );
 277
 278     my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
 279     $se->mock('_foreach_mapping', sub {
 280         my ($self, $sub) = @_;
 281
 282         foreach my $map (@mappings) {
 283             $sub->(
 284                 $map->{name},
 285                 $map->{type},
 286                 $map->{facet},
 287                 $map->{suggestible},
 288                 $map->{sort},
 289                 $map->{searchable},
 290                 $map->{marc_type},
 291                 $map->{marc_field}
 292             );
 293         }
 294     });
 295
 296     my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
 297
 298     my $callno = 'ABC123';
 299     my $callno2 = 'ABC456';
 300     my $long_callno = '1234567890' x 30;
 301
 302     my $marc_record_1 = MARC::Record->new();
 303     $marc_record_1->leader('     cam  22      a 4500');
 304     $marc_record_1->append_fields(
 305         MARC::Field->new('001', '123'),
 306         MARC::Field->new('007', 'ku'),
 307         MARC::Field->new('020', '', '', a => '1-56619-909-3'),
 308         MARC::Field->new('100', '', '', a => 'Author 1'),
 309         MARC::Field->new('110', '', '', a => 'Corp Author'),
 310         MARC::Field->new('210', '', '', a => 'Title 1'),
 311         MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
 312         MARC::Field->new('999', '', '', c => '1234567'),
 313         # '  ' for testing trimming of white space in boolean value callback:
 314         MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno, l => 3),
 315         MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
 316         MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
 317     );
 318     my $marc_record_2 = MARC::Record->new();
 319     $marc_record_2->leader('     cam  22      a 4500');
 320     $marc_record_2->append_fields(
 321         MARC::Field->new('100', '', '', a => 'Author 2'),
 322         # MARC::Field->new('210', '', '', a => 'Title 2'),
 323         # MARC::Field->new('245', '', '', a => 'Title: second record'),
 324         MARC::Field->new('999', '', '', c => '1234568'),
 325         MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
 326     );
 327     my $records = [$marc_record_1, $marc_record_2];
 328
 329     $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
 330
 331     my $docs = $see->marc_records_to_documents($records);
 332
 333     # First record:
 334     is(scalar @{$docs}, 2, 'Two records converted to documents');
 335
 336     is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
 337
 338     is_deeply($docs->[0]->{'ff7-00'}, ['k'], 'First record ff7-00 should be set correctly');
 339
 340     is(scalar @{$docs->[0]->{author}}, 2, 'First document author field should contain two values');
 341     is_deeply($docs->[0]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
 342
 343     is(scalar @{$docs->[0]->{author__sort}}, 1, 'First document author__sort field should have a single value');
 344     is_deeply($docs->[0]->{author__sort}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly');
 345
 346     is(scalar @{$docs->[0]->{title__sort}}, 1, 'First document title__sort field should have a single');
 347     is_deeply($docs->[0]->{title__sort}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
 348
 349     is($docs->[0]->{issues}, 6, 'Issues field should be sum of the issues for each item');
 350     is($docs->[0]->{issues__sort}, 6, 'Issues sort field should also be a sum of the issues');
 351
 352     is(scalar @{$docs->[0]->{title_wildcard}}, 2, 'First document title_wildcard field should have two values');
 353     is_deeply($docs->[0]->{title_wildcard}, ['Title:', 'first record'], 'First document title_wildcard field should be set correctly');
 354
 355
 356     is(scalar @{$docs->[0]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
 357     is_deeply(
 358         $docs->[0]->{author__suggestion},
 359         [
 360             {
 361                 'input' => 'Author 1'
 362             },
 363             {
 364                 'input' => 'Corp Author'
 365             }
 366         ],
 367         'First document author__suggestion field should be set correctly'
 368     );
 369
 370     is(scalar @{$docs->[0]->{title__suggestion}}, 3, 'First document title__suggestion field should contain three values');
 371     is_deeply(
 372         $docs->[0]->{title__suggestion},
 373         [
 374             { 'input' => 'Title:' },
 375             { 'input' => 'first record' },
 376             { 'input' => 'Title: first record' }
 377         ],
 378         'First document title__suggestion field should be set correctly'
 379     );
 380
 381     ok(!(defined $docs->[0]->{title__facet}), 'First document should have no title__facet field');
 382
 383     is(scalar @{$docs->[0]->{author__facet}}, 2, 'First document author__facet field should have two values');
 384     is_deeply(
 385         $docs->[0]->{author__facet},
 386         ['Author 1', 'Corp Author'],
 387         'First document author__facet field should be set correctly'
 388     );
 389
 390     is(scalar @{$docs->[0]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
 391     is_deeply(
 392         $docs->[0]->{items_withdrawn_status},
 393         ['false', 'true'],
 394         'First document items_withdrawn_status field should be set correctly'
 395     );
 396
 397     is(
 398         $docs->[0]->{sum_item_price},
 399         '250.5',
 400         'First document sum_item_price field should be set correctly'
 401     );
 402
 403     ok(defined $docs->[0]->{marc_data}, 'First document marc_data field should be set');
 404     ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
 405     is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
 406
 407     my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
 408
 409     ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
 410     is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
 411
 412     is(scalar @{$docs->[0]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
 413     is_deeply(
 414         $docs->[0]->{type_of_record},
 415         ['a'],
 416         'First document type_of_record field should be set correctly'
 417     );
 418
 419     is(scalar @{$docs->[0]->{type_of_record_and_bib_level}}, 1, 'First document type_of_record_and_bib_level field should have one value');
 420     is_deeply(
 421         $docs->[0]->{type_of_record_and_bib_level},
 422         ['am'],
 423         'First document type_of_record_and_bib_level field should be set correctly'
 424     );
 425
 426     is(scalar @{$docs->[0]->{isbn}}, 4, 'First document isbn field should contain four values');
 427     is_deeply($docs->[0]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
 428
 429     is_deeply(
 430         $docs->[0]->{'local_classification'},
 431         [$callno, $callno2, $long_callno],
 432         'First document local_classification field should be set correctly'
 433     );
 434
 435     # Second record:
 436
 437     is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
 438     is_deeply($docs->[1]->{author}, ['Author 2'], 'Second document author field should be set correctly');
 439
 440     is(scalar @{$docs->[1]->{items_withdrawn_status}}, 1, 'Second document items_withdrawn_status field should have one value');
 441     is_deeply(
 442         $docs->[1]->{items_withdrawn_status},
 443         ['true'],
 444         'Second document items_withdrawn_status field should be set correctly'
 445     );
 446
 447     is(
 448         $docs->[1]->{sum_item_price},
 449         0,
 450         'Second document sum_item_price field should be set correctly'
 451     );
 452
 453     is_deeply(
 454         $docs->[1]->{local_classification__sort},
 455         [substr($long_callno, 0, 255)],
 456         'Second document local_classification__sort field should be set correctly'
 457     );
 458
 459     # Mappings marc_type:
 460
 461     ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
 462
 463     # Marc serialization format fallback for records exceeding ISO2709 max record size
 464
 465     my $large_marc_record = MARC::Record->new();
 466     $large_marc_record->leader('     cam  22      a 4500');
 467
 468     $large_marc_record->append_fields(
 469         MARC::Field->new('100', '', '', a => 'Author 1'),
 470         MARC::Field->new('110', '', '', a => 'Corp Author'),
 471         MARC::Field->new('210', '', '', a => 'Title 1'),
 472         MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
 473         MARC::Field->new('999', '', '', c => '1234567'),
 474     );
 475
 476     my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
 477     my $items_count = 1638;
 478     while(--$items_count) {
 479         $large_marc_record->append_fields($item_field);
 480     }
 481
 482     $docs = $see->marc_records_to_documents([$large_marc_record]);
 483
 484     is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
 485
 486     $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
 487
 488     ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
 489     is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
 490
 491     push @mappings, {
 492         name => 'title',
 493         type => 'string',
 494         facet => 0,
 495         suggestible => 1,
 496         sort => 1,
 497         marc_type => 'marc21',
 498         marc_field => '245((ab)ab',
 499     };
 500
 501     my $exception = try {
 502         $see->marc_records_to_documents($records);
 503     }
 504     catch {
 505         return $_;
 506     };
 507
 508     ok(defined $exception, "Exception has been thrown when processing mapping with unmatched opening parenthesis");
 509     ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
 510     ok($exception->message =~ /Unmatched opening parenthesis/, "Exception has the correct message");
 511
 512     pop @mappings;
 513     push @mappings, {
 514         name => 'title',
 515         type => 'string',
 516         facet => 0,
 517         suggestible => 1,
 518         sort => 1,
 519         marc_type => 'marc21',
 520         marc_field => '245(ab))ab',
 521     };
 522
 523     $exception = try {
 524         $see->marc_records_to_documents($records);
 525     }
 526     catch {
 527         return $_;
 528     };
 529
 530     ok(defined $exception, "Exception has been thrown when processing mapping with unmatched closing parenthesis");
 531     ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
 532     ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
 533 };
 534
 535 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
 536
 537     plan tests => 5;
 538
 539     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
 540     t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
 541
 542     my @mappings = (
 543         {
 544             name => 'control_number',
 545             type => 'string',
 546             facet => 0,
 547             suggestible => 0,
 548             sort => undef,
 549             searchable => 1,
 550             marc_type => 'marc21',
 551             marc_field => '001',
 552         }
 553     );
 554
 555     my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
 556     $se->mock('_foreach_mapping', sub {
 557         my ($self, $sub) = @_;
 558
 559         foreach my $map (@mappings) {
 560             $sub->(
 561                 $map->{name},
 562                 $map->{type},
 563                 $map->{facet},
 564                 $map->{suggestible},
 565                 $map->{sort},
 566                 $map->{searchable},
 567                 $map->{marc_type},
 568                 $map->{marc_field}
 569             );
 570         }
 571     });
 572
 573     my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
 574
 575     my $marc_record_1 = MARC::Record->new();
 576     $marc_record_1->leader('     cam  22      a 4500');
 577     $marc_record_1->append_fields(
 578         MARC::Field->new('001', '123'),
 579         MARC::Field->new('020', '', '', a => '1-56619-909-3'),
 580         MARC::Field->new('100', '', '', a => 'Author 1'),
 581         MARC::Field->new('110', '', '', a => 'Corp Author'),
 582         MARC::Field->new('210', '', '', a => 'Title 1'),
 583         MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
 584         MARC::Field->new('999', '', '', c => '1234567'),
 585     );
 586     my $marc_record_2 = MARC::Record->new();
 587     $marc_record_2->leader('     cam  22      a 4500');
 588     $marc_record_2->append_fields(
 589         MARC::Field->new('100', '', '', a => 'Author 2'),
 590         # MARC::Field->new('210', '', '', a => 'Title 2'),
 591         # MARC::Field->new('245', '', '', a => 'Title: second record'),
 592         MARC::Field->new('999', '', '', c => '1234568'),
 593         MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
 594     );
 595     my $records = [$marc_record_1, $marc_record_2];
 596
 597     $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
 598
 599     my $docs = $see->marc_records_to_documents($records);
 600
 601     # First record:
 602     is(scalar @{$docs}, 2, 'Two records converted to documents');
 603
 604     is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
 605
 606     is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
 607
 608     my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
 609
 610     ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
 611     is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
 612 };