t/db_dependent/Koha/SearchEngine/Elasticsearch.t

   1 #!/usr/bin/perl
   2 #
   3 # This file is part of Koha.
   4 #
   5 # Koha is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # Koha is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  17
  18 use Modern::Perl;
  19
  20 use Test::More tests => 6;
  21 use Test::Exception;
  22
  23 use t::lib::Mocks;
  24 use t::lib::TestBuilder;
  25
  26 use Test::MockModule;
  27
  28 use MARC::Record;
  29 use Try::Tiny;
  30 use List::Util qw( any );
  31
  32 use Koha::SearchEngine::Elasticsearch;
  33 use Koha::SearchEngine::Elasticsearch::Search;
  34
  35 my $schema = Koha::Database->new->schema;
  36 $schema->storage->txn_begin;
  37
  38 subtest '_read_configuration() tests' => sub {
  39
  40     plan tests => 13;
  41
  42     my $configuration;
  43     t::lib::Mocks::mock_config( 'elasticsearch', undef );
  44
  45     # 'elasticsearch' missing in configuration
  46     throws_ok {
  47         $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  48     }
  49     'Koha::Exceptions::Config::MissingEntry',
  50       'Configuration problem, exception thrown';
  51     is(
  52         $@->message,
  53         "Missing <elasticsearch> entry in koha-conf.xml",
  54         'Exception message is correct'
  55     );
  56
  57     # 'elasticsearch' present but no 'server' entry
  58     t::lib::Mocks::mock_config( 'elasticsearch', {} );
  59     throws_ok {
  60         $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  61     }
  62     'Koha::Exceptions::Config::MissingEntry',
  63       'Configuration problem, exception thrown';
  64     is(
  65         $@->message,
  66         "Missing <elasticsearch>/<server> entry in koha-conf.xml",
  67         'Exception message is correct'
  68     );
  69
  70     # 'elasticsearch' and 'server' entries present, but no 'index_name'
  71     t::lib::Mocks::mock_config( 'elasticsearch', { server => 'a_server' } );
  72     throws_ok {
  73         $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  74     }
  75     'Koha::Exceptions::Config::MissingEntry',
  76       'Configuration problem, exception thrown';
  77     is(
  78         $@->message,
  79         "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
  80         'Exception message is correct'
  81     );
  82
  83     # Correct configuration, only one server
  84     t::lib::Mocks::mock_config( 'elasticsearch',  { server => 'a_server', index_name => 'index' } );
  85
  86     $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  87     is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
  88     is_deeply( $configuration->{nodes}, ['a_server'], 'Server configuration parsed correctly' );
  89
  90     # Correct configuration, two servers
  91     my @servers = ('a_server', 'another_server');
  92     t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index' } );
  93
  94     $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
  95     is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
  96     is( $configuration->{cxn_pool}, 'Static', 'cxn_pool configuration set correctly to Static if not specified' );
  97     is_deeply( $configuration->{nodes}, \@servers , 'Server configuration parsed correctly' );
  98
  99     t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index', cxn_pool => 'Fluid' } );
 100
 101     $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
 102     is( $configuration->{cxn_pool}, 'Fluid', 'cxn_pool configuration parsed correctly' );
 103
 104     my $params = Koha::SearchEngine::Elasticsearch::get_elasticsearch_params;
 105     is_deeply( $configuration->{nodes}, \@servers , 'get_elasticsearch_params is just a wrapper for _read_configuration' );
 106
 107 };
 108
 109 subtest 'get_elasticsearch_settings() tests' => sub {
 110
 111     plan tests => 1;
 112
 113     my $settings;
 114
 115     # test reading index settings
 116     my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
 117     $settings = $es->get_elasticsearch_settings();
 118     is( $settings->{index}{analysis}{analyzer}{analyzer_phrase}{tokenizer}, 'keyword', 'Index settings parsed correctly' );
 119 };
 120
 121 subtest 'get_elasticsearch_mappings() tests' => sub {
 122
 123     plan tests => 1;
 124
 125     my $mappings;
 126
 127     # test reading mappings
 128     my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
 129     $mappings = $es->get_elasticsearch_mappings();
 130     is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
 131 };
 132
 133 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
 134
 135     plan tests => 61;
 136
 137     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
 138     t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
 139
 140     my @mappings = (
 141         {
 142             name => 'control_number',
 143             type => 'string',
 144             facet => 0,
 145             suggestible => 0,
 146             searchable => 1,
 147             sort => undef,
 148             marc_type => 'marc21',
 149             marc_field => '001',
 150         },
 151         {
 152             name => 'isbn',
 153             type => 'isbn',
 154             facet => 0,
 155             suggestible => 0,
 156             searchable => 1,
 157             sort => 0,
 158             marc_type => 'marc21',
 159             marc_field => '020a',
 160         },
 161         {
 162             name => 'author',
 163             type => 'string',
 164             facet => 1,
 165             suggestible => 1,
 166             searchable => 1,
 167             sort => undef,
 168             marc_type => 'marc21',
 169             marc_field => '100a',
 170         },
 171         {
 172             name => 'author',
 173             type => 'string',
 174             facet => 1,
 175             suggestible => 1,
 176             searchable => 1,
 177             sort => 1,
 178             marc_type => 'marc21',
 179             marc_field => '110a',
 180         },
 181         {
 182             name => 'title',
 183             type => 'string',
 184             facet => 0,
 185             suggestible => 1,
 186             searchable => 1,
 187             sort => 1,
 188             marc_type => 'marc21',
 189             marc_field => '245(ab)ab',
 190         },
 191         {
 192             name => 'unimarc_title',
 193             type => 'string',
 194             facet => 0,
 195             suggestible => 1,
 196             searchable => 1,
 197             sort => 1,
 198             marc_type => 'unimarc',
 199             marc_field => '245a',
 200         },
 201         {
 202             name => 'title',
 203             type => 'string',
 204             facet => 0,
 205             suggestible => undef,
 206             searchable => 1,
 207             sort => 0,
 208             marc_type => 'marc21',
 209             marc_field => '220',
 210         },
 211         {
 212             name => 'uniform_title',
 213             type => 'string',
 214             facet => 0,
 215             suggestible => 0,
 216             searchable => 1,
 217             sort => 1,
 218             marc_type => 'marc21',
 219             marc_field => '240a',
 220         },
 221         {
 222             name => 'title_wildcard',
 223             type => 'string',
 224             facet => 0,
 225             suggestible => 0,
 226             searchable => 1,
 227             sort => undef,
 228             marc_type => 'marc21',
 229             marc_field => '245',
 230         },
 231         {
 232             name => 'sum_item_price',
 233             type => 'sum',
 234             facet => 0,
 235             suggestible => 0,
 236             searchable => 1,
 237             sort => 0,
 238             marc_type => 'marc21',
 239             marc_field => '952g',
 240         },
 241         {
 242             name => 'items_withdrawn_status',
 243             type => 'boolean',
 244             facet => 0,
 245             suggestible => 0,
 246             searchable => 1,
 247             sort => 0,
 248             marc_type => 'marc21',
 249             marc_field => '9520',
 250         },
 251         {
 252             name => 'local_classification',
 253             type => 'string',
 254             facet => 0,
 255             suggestible => 0,
 256             searchable => 1,
 257             sort => 1,
 258             marc_type => 'marc21',
 259             marc_field => '952o',
 260         },
 261         {
 262             name => 'type_of_record',
 263             type => 'string',
 264             facet => 0,
 265             suggestible => 0,
 266             searchable => 1,
 267             sort => 0,
 268             marc_type => 'marc21',
 269             marc_field => 'leader_/6',
 270         },
 271         {
 272             name => 'type_of_record_and_bib_level',
 273             type => 'string',
 274             facet => 0,
 275             suggestible => 0,
 276             searchable => 1,
 277             sort => 0,
 278             marc_type => 'marc21',
 279             marc_field => 'leader_/6-7',
 280         },
 281         {
 282             name => 'ff7-00',
 283             type => 'string',
 284             facet => 0,
 285             suggestible => 0,
 286             searchable => 1,
 287             sort => 0,
 288             marc_type => 'marc21',
 289             marc_field => '007_/0',
 290         },
 291         {
 292             name => 'issues',
 293             type => 'sum',
 294             facet => 0,
 295             suggestible => 0,
 296             searchable => 1,
 297             sort => 1,
 298             marc_type => 'marc21',
 299             marc_field => '952l',
 300           },
 301           {
 302             name => 'copydate',
 303             type => 'year',
 304             facet => 0,
 305             suggestible => 0,
 306             searchable => 1,
 307             sort => 1,
 308             marc_type => 'marc21',
 309             marc_field => '260c',
 310           },
 311           {
 312             name => 'date-of-publication',
 313             type => 'year',
 314             facet => 0,
 315             suggestible => 0,
 316             searchable => 1,
 317             sort => 1,
 318             marc_type => 'marc21',
 319             marc_field => '008_/7-10',
 320         },
 321
 322     );
 323
 324     my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
 325     $se->mock('_foreach_mapping', sub {
 326         my ($self, $sub) = @_;
 327
 328         foreach my $map (@mappings) {
 329             $sub->(
 330                 $map->{name},
 331                 $map->{type},
 332                 $map->{facet},
 333                 $map->{suggestible},
 334                 $map->{sort},
 335                 $map->{searchable},
 336                 $map->{marc_type},
 337                 $map->{marc_field}
 338             );
 339         }
 340     });
 341
 342     my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
 343
 344     my $callno = 'ABC123';
 345     my $callno2 = 'ABC456';
 346     my $long_callno = '1234567890' x 30;
 347
 348     my $marc_record_1 = MARC::Record->new();
 349     $marc_record_1->leader('     cam  22      a 4500');
 350     $marc_record_1->append_fields(
 351         MARC::Field->new('001', '123'),
 352         MARC::Field->new('007', 'ku'),
 353         MARC::Field->new('008', '901111s1962 xxk|||| |00| ||eng c'),
 354         MARC::Field->new('020', '', '', a => '1-56619-909-3'),
 355         MARC::Field->new('100', '', '', a => 'Author 1'),
 356         MARC::Field->new('110', '', '', a => 'Corp Author'),
 357         MARC::Field->new('210', '', '', a => 'Title 1'),
 358         MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
 359         MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
 360         MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => 'c1962'),
 361         MARC::Field->new('999', '', '', c => '1234567'),
 362         # '  ' for testing trimming of white space in boolean value callback:
 363         MARC::Field->new('952', '', '', 0 => '  ', g => '123.30', o => $callno, l => 3),
 364         MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
 365         MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
 366     );
 367     my $marc_record_2 = MARC::Record->new();
 368     $marc_record_2->leader('     cam  22      a 4500');
 369     $marc_record_2->append_fields(
 370         MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
 371         MARC::Field->new('100', '', '', a => 'Author 2'),
 372         # MARC::Field->new('210', '', '', a => 'Title 2'),
 373         # MARC::Field->new('245', '', '', a => 'Title: second record'),
 374         MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => '1963-2003'),
 375         MARC::Field->new('999', '', '', c => '1234568'),
 376         MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
 377     );
 378
 379     my $marc_record_3 = MARC::Record->new();
 380     $marc_record_3->leader('     cam  22      a 4500');
 381     $marc_record_3->append_fields(
 382         MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
 383         MARC::Field->new('100', '', '', a => 'Author 2'),
 384         # MARC::Field->new('210', '', '', a => 'Title 3'),
 385         # MARC::Field->new('245', '', '', a => 'Title: third record'),
 386         MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => ' 89 '),
 387         MARC::Field->new('999', '', '', c => '1234568'),
 388         MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
 389     );
 390     my $records = [$marc_record_1, $marc_record_2, $marc_record_3];
 391
 392     $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
 393
 394     my $docs = $see->marc_records_to_documents($records);
 395
 396     # First record:
 397     is(scalar @{$docs}, 3, 'Two records converted to documents');
 398
 399     is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
 400
 401     is_deeply($docs->[0]->{'ff7-00'}, ['k'], 'First record ff7-00 should be set correctly');
 402
 403     is(scalar @{$docs->[0]->{author}}, 2, 'First document author field should contain two values');
 404     is_deeply($docs->[0]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
 405
 406     is(scalar @{$docs->[0]->{author__sort}}, 1, 'First document author__sort field should have a single value');
 407     is_deeply($docs->[0]->{author__sort}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly');
 408
 409     is(scalar @{$docs->[0]->{title__sort}}, 1, 'First document title__sort field should have a single');
 410     is_deeply($docs->[0]->{title__sort}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
 411
 412     is($docs->[0]->{issues}, 6, 'Issues field should be sum of the issues for each item');
 413     is($docs->[0]->{issues__sort}, 6, 'Issues sort field should also be a sum of the issues');
 414
 415     is(scalar @{$docs->[0]->{title_wildcard}}, 2, 'First document title_wildcard field should have two values');
 416     is_deeply($docs->[0]->{title_wildcard}, ['Title:', 'first record'], 'First document title_wildcard field should be set correctly');
 417
 418
 419     is(scalar @{$docs->[0]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
 420     is_deeply(
 421         $docs->[0]->{author__suggestion},
 422         [
 423             {
 424                 'input' => 'Author 1'
 425             },
 426             {
 427                 'input' => 'Corp Author'
 428             }
 429         ],
 430         'First document author__suggestion field should be set correctly'
 431     );
 432
 433     is(scalar @{$docs->[0]->{title__suggestion}}, 3, 'First document title__suggestion field should contain three values');
 434     is_deeply(
 435         $docs->[0]->{title__suggestion},
 436         [
 437             { 'input' => 'Title:' },
 438             { 'input' => 'first record' },
 439             { 'input' => 'Title: first record' }
 440         ],
 441         'First document title__suggestion field should be set correctly'
 442     );
 443
 444     ok(!(defined $docs->[0]->{title__facet}), 'First document should have no title__facet field');
 445
 446     is(scalar @{$docs->[0]->{author__facet}}, 2, 'First document author__facet field should have two values');
 447     is_deeply(
 448         $docs->[0]->{author__facet},
 449         ['Author 1', 'Corp Author'],
 450         'First document author__facet field should be set correctly'
 451     );
 452
 453     is(scalar @{$docs->[0]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
 454     is_deeply(
 455         $docs->[0]->{items_withdrawn_status},
 456         ['false', 'true'],
 457         'First document items_withdrawn_status field should be set correctly'
 458     );
 459
 460     is(
 461         $docs->[0]->{sum_item_price},
 462         '250.5',
 463         'First document sum_item_price field should be set correctly'
 464     );
 465
 466     ok(defined $docs->[0]->{marc_data}, 'First document marc_data field should be set');
 467     ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
 468     is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
 469
 470     my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
 471
 472     ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
 473     is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
 474
 475     is(scalar @{$docs->[0]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
 476     is_deeply(
 477         $docs->[0]->{type_of_record},
 478         ['a'],
 479         'First document type_of_record field should be set correctly'
 480     );
 481
 482     is(scalar @{$docs->[0]->{type_of_record_and_bib_level}}, 1, 'First document type_of_record_and_bib_level field should have one value');
 483     is_deeply(
 484         $docs->[0]->{type_of_record_and_bib_level},
 485         ['am'],
 486         'First document type_of_record_and_bib_level field should be set correctly'
 487     );
 488
 489     is(scalar @{$docs->[0]->{isbn}}, 4, 'First document isbn field should contain four values');
 490     is_deeply($docs->[0]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
 491
 492     is_deeply(
 493         $docs->[0]->{'local_classification'},
 494         [$callno, $callno2, $long_callno],
 495         'First document local_classification field should be set correctly'
 496     );
 497
 498     # Nonfiling characters for sort fields
 499     is_deeply(
 500         $docs->[0]->{uniform_title},
 501         ['The uniform title with nonfiling indicator'],
 502         'First document uniform_title field should contain the title verbatim'
 503     );
 504     is_deeply(
 505         $docs->[0]->{uniform_title__sort},
 506         ['uniform title with nonfiling indicator'],
 507         'First document uniform_title__sort field should contain the title with the first four initial characters removed'
 508     );
 509
 510     # Tests for 'year' type
 511     is(scalar @{$docs->[0]->{'date-of-publication'}}, 1, 'First document date-of-publication field should contain one value');
 512     is_deeply($docs->[0]->{'date-of-publication'}, ['1962'], 'First document date-of-publication field should be set correctly');
 513
 514     is_deeply(
 515       $docs->[0]->{'copydate'},
 516       ['1962'],
 517       'First document copydate field should be set correctly'
 518     );
 519
 520     # Second record:
 521
 522     is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
 523     is_deeply($docs->[1]->{author}, ['Author 2'], 'Second document author field should be set correctly');
 524
 525     is(scalar @{$docs->[1]->{items_withdrawn_status}}, 1, 'Second document items_withdrawn_status field should have one value');
 526     is_deeply(
 527         $docs->[1]->{items_withdrawn_status},
 528         ['true'],
 529         'Second document items_withdrawn_status field should be set correctly'
 530     );
 531
 532     is(
 533         $docs->[1]->{sum_item_price},
 534         0,
 535         'Second document sum_item_price field should be set correctly'
 536     );
 537
 538     is_deeply(
 539         $docs->[1]->{local_classification__sort},
 540         [substr($long_callno, 0, 255)],
 541         'Second document local_classification__sort field should be set correctly'
 542     );
 543
 544     # Tests for 'year' type
 545     is_deeply(
 546       $docs->[1]->{'copydate'},
 547       ['1963', '2003'],
 548       'Second document copydate field should be set correctly'
 549     );
 550     is_deeply(
 551       $docs->[1]->{'date-of-publication'},
 552       ['1900'],
 553       'Second document date-of-publication field should be set correctly'
 554     );
 555
 556     # Third record:
 557
 558     is_deeply(
 559       $docs->[2]->{'copydate'},
 560       ['0890'],
 561       'Third document copydate field should be set correctly'
 562     );
 563
 564     # Mappings marc_type:
 565
 566     ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
 567
 568     # Marc serialization format fallback for records exceeding ISO2709 max record size
 569
 570     my $large_marc_record = MARC::Record->new();
 571     $large_marc_record->leader('     cam  22      a 4500');
 572
 573     $large_marc_record->append_fields(
 574         MARC::Field->new('100', '', '', a => 'Author 1'),
 575         MARC::Field->new('110', '', '', a => 'Corp Author'),
 576         MARC::Field->new('210', '', '', a => 'Title 1'),
 577         MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
 578         MARC::Field->new('999', '', '', c => '1234567'),
 579     );
 580
 581     my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
 582     my $items_count = 1638;
 583     while(--$items_count) {
 584         $large_marc_record->append_fields($item_field);
 585     }
 586
 587     $docs = $see->marc_records_to_documents([$large_marc_record]);
 588
 589     is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
 590
 591     $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
 592
 593     ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
 594     is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
 595
 596     push @mappings, {
 597         name => 'title',
 598         type => 'string',
 599         facet => 0,
 600         suggestible => 1,
 601         sort => 1,
 602         marc_type => 'marc21',
 603         marc_field => '245((ab)ab',
 604     };
 605
 606     my $exception = try {
 607         $see->marc_records_to_documents($records);
 608     }
 609     catch {
 610         return $_;
 611     };
 612
 613     ok(defined $exception, "Exception has been thrown when processing mapping with unmatched opening parenthesis");
 614     ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
 615     ok($exception->message =~ /Unmatched opening parenthesis/, "Exception has the correct message");
 616
 617     pop @mappings;
 618     push @mappings, {
 619         name => 'title',
 620         type => 'string',
 621         facet => 0,
 622         suggestible => 1,
 623         sort => 1,
 624         marc_type => 'marc21',
 625         marc_field => '245(ab))ab',
 626     };
 627
 628     $exception = try {
 629         $see->marc_records_to_documents($records);
 630     }
 631     catch {
 632         return $_;
 633     };
 634
 635     ok(defined $exception, "Exception has been thrown when processing mapping with unmatched closing parenthesis");
 636     ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
 637     ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
 638
 639     pop @mappings;
 640     my $marc_record_with_blank_field = MARC::Record->new();
 641     $marc_record_with_blank_field->leader('     cam  22      a 4500');
 642
 643     $marc_record_with_blank_field->append_fields(
 644         MARC::Field->new('100', '', '', a => ''),
 645         MARC::Field->new('210', '', '', a => 'Title 1'),
 646         MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
 647         MARC::Field->new('999', '', '', c => '1234567'),
 648     );
 649     $docs = $see->marc_records_to_documents([$marc_record_with_blank_field]);
 650     is_deeply( $docs->[0]->{author},[],'No value placed into field if mapped marc field is blank');
 651     is_deeply( $docs->[0]->{author__suggestion},[],'No value placed into suggestion if mapped marc field is blank');
 652
 653 };
 654
 655 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
 656
 657     plan tests => 5;
 658
 659     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
 660     t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
 661
 662     my @mappings = (
 663         {
 664             name => 'control_number',
 665             type => 'string',
 666             facet => 0,
 667             suggestible => 0,
 668             sort => undef,
 669             searchable => 1,
 670             marc_type => 'marc21',
 671             marc_field => '001',
 672         }
 673     );
 674
 675     my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
 676     $se->mock('_foreach_mapping', sub {
 677         my ($self, $sub) = @_;
 678
 679         foreach my $map (@mappings) {
 680             $sub->(
 681                 $map->{name},
 682                 $map->{type},
 683                 $map->{facet},
 684                 $map->{suggestible},
 685                 $map->{sort},
 686                 $map->{searchable},
 687                 $map->{marc_type},
 688                 $map->{marc_field}
 689             );
 690         }
 691     });
 692
 693     my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
 694
 695     my $marc_record_1 = MARC::Record->new();
 696     $marc_record_1->leader('     cam  22      a 4500');
 697     $marc_record_1->append_fields(
 698         MARC::Field->new('001', '123'),
 699         MARC::Field->new('020', '', '', a => '1-56619-909-3'),
 700         MARC::Field->new('100', '', '', a => 'Author 1'),
 701         MARC::Field->new('110', '', '', a => 'Corp Author'),
 702         MARC::Field->new('210', '', '', a => 'Title 1'),
 703         MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
 704         MARC::Field->new('999', '', '', c => '1234567'),
 705     );
 706     my $marc_record_2 = MARC::Record->new();
 707     $marc_record_2->leader('     cam  22      a 4500');
 708     $marc_record_2->append_fields(
 709         MARC::Field->new('100', '', '', a => 'Author 2'),
 710         # MARC::Field->new('210', '', '', a => 'Title 2'),
 711         # MARC::Field->new('245', '', '', a => 'Title: second record'),
 712         MARC::Field->new('999', '', '', c => '1234568'),
 713         MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
 714     );
 715     my $records = [ $marc_record_1, $marc_record_2 ];
 716
 717     $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
 718
 719     my $docs = $see->marc_records_to_documents($records);
 720
 721     # First record:
 722     is(scalar @{$docs}, 2, 'Two records converted to documents');
 723
 724     is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
 725
 726     is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
 727
 728     my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
 729
 730     ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
 731     is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
 732 };
 733
 734 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authority tests' => sub {
 735
 736     plan tests => 2;
 737
 738     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
 739     t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
 740
 741     my $builder = t::lib::TestBuilder->new;
 742     my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
 743             auth_tag_to_report => '150'
 744         }
 745     });
 746
 747     my @mappings = (
 748         {
 749             name => 'match',
 750             type => 'string',
 751             facet => 0,
 752             suggestible => 0,
 753             searchable => 1,
 754             sort => 0,
 755             marc_type => 'marc21',
 756             marc_field => '150(ae)',
 757         }
 758     );
 759
 760     my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
 761     $se->mock('_foreach_mapping', sub {
 762         my ($self, $sub) = @_;
 763
 764         foreach my $map (@mappings) {
 765             $sub->(
 766                 $map->{name},
 767                 $map->{type},
 768                 $map->{facet},
 769                 $map->{suggestible},
 770                 $map->{sort},
 771                 $map->{searchable},
 772                 $map->{marc_type},
 773                 $map->{marc_field}
 774             );
 775         }
 776     });
 777
 778     my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX });
 779     my $marc_record_1 = MARC::Record->new();
 780     $marc_record_1->append_fields(
 781         MARC::Field->new('001', '123'),
 782         MARC::Field->new('007', 'ku'),
 783         MARC::Field->new('020', '', '', a => '1-56619-909-3'),
 784         MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', x => 'Generalsubdiv', z => 'Geosubdiv'),
 785     );
 786     my $marc_record_2 = MARC::Record->new();
 787     $marc_record_2->append_fields(
 788         MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
 789     );
 790     my $records = [ $marc_record_1, $marc_record_2 ];
 791
 792     $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
 793
 794     my $docs = $see->marc_records_to_documents($records);
 795
 796     ok(
 797         any { $_ eq "Subject formsubdiv Genresubdiv generalsubdiv Generalsubdiv geographicsubdiv Geosubdiv" }
 798         @{$docs->[0]->{'match-heading'}},
 799         "First record match-heading should contain the correctly formatted heading"
 800     );
 801     ok(
 802         any { $_ eq "Subject formsubdiv Genresubdiv geographicsubdiv Geosubdiv generalsubdiv Generalsubdiv" }
 803         @{$docs->[1]->{'match-heading'}},
 804         "Second record match-heading should contain the correctly formatted heading without wrong subfield"
 805     );
 806 };
 807
 808 $schema->storage->txn_rollback;