t/db_dependent/Koha/SearchEngine/Elasticsearch/QueryBuilder.t

   1 #!/usr/bin/perl
   2 #
   3 # This file is part of Koha.
   4 #
   5 # Koha is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # Koha is distributed in the hope that it will be useful, but
  11 # WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  17
  18 use Modern::Perl;
  19
  20 use C4::Context;
  21 use Test::Exception;
  22 use t::lib::Mocks;
  23 use t::lib::TestBuilder;
  24 use Test::More tests => 6;
  25
  26 use Koha::Database;
  27 use Koha::SearchEngine::Elasticsearch::QueryBuilder;
  28
  29 my $schema = Koha::Database->new->schema;
  30 $schema->storage->txn_begin;
  31
  32 my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
  33 $se->mock( 'get_elasticsearch_mappings', sub {
  34     my ($self) = @_;
  35
  36     my %all_mappings;
  37
  38     my $mappings = {
  39         data => {
  40             properties => {
  41                 title => {
  42                     type => 'text'
  43                 },
  44                 title__sort => {
  45                     type => 'text'
  46                 },
  47                 subject => {
  48                     type => 'text'
  49                 },
  50                 itemnumber => {
  51                     type => 'integer'
  52                 },
  53                 sortablenumber => {
  54                     type => 'integer'
  55                 },
  56                 sortablenumber__sort => {
  57                     type => 'integer'
  58                 },
  59                 Heading => {
  60                     type => 'text'
  61                 },
  62                 Heading__sort => {
  63                     type => 'text'
  64                 }
  65             }
  66         }
  67     };
  68     $all_mappings{$self->index} = $mappings;
  69
  70     my $sort_fields = {
  71         $self->index => {
  72             title => 1,
  73             subject => 0,
  74             itemnumber => 0,
  75             sortablenumber => 1,
  76             mainentry => 1
  77         }
  78     };
  79     $self->sort_fields($sort_fields->{$self->index});
  80
  81     return $all_mappings{$self->index};
  82 });
  83
  84 subtest 'build_authorities_query_compat() tests' => sub {
  85     plan tests => 37;
  86
  87     my $qb;
  88
  89     ok(
  90         $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new({ 'index' => 'authorities' }),
  91         'Creating new query builder object for authorities'
  92     );
  93
  94     my $koha_to_index_name = $Koha::SearchEngine::Elasticsearch::QueryBuilder::koha_to_index_name;
  95     my $search_term = 'a';
  96     foreach my $koha_name ( keys %{ $koha_to_index_name } ) {
  97         my $query = $qb->build_authorities_query_compat( [ $koha_name ],  undef, undef, ['contains'], [$search_term], 'AUTH_TYPE', 'asc' );
  98         if ( $koha_name eq 'all' || $koha_name eq 'any' ) {
  99             is( $query->{query}->{bool}->{must}[0]->{query_string}->{query},
 100                 "a*");
 101         } else {
 102             is( $query->{query}->{bool}->{must}[0]->{query_string}->{query},
 103                 "a*");
 104         }
 105     }
 106
 107     $search_term = 'Donald Duck';
 108     foreach my $koha_name ( keys %{ $koha_to_index_name } ) {
 109         my $query = $qb->build_authorities_query_compat( [ $koha_name ],  undef, undef, ['contains'], [$search_term], 'AUTH_TYPE', 'asc' );
 110         if ( $koha_name eq 'all' || $koha_name eq 'any' ) {
 111             is( $query->{query}->{bool}->{must}[0]->{query_string}->{query},
 112                 "(Donald*) AND (Duck*)");
 113         } else {
 114             is( $query->{query}->{bool}->{must}[0]->{query_string}->{query},
 115                 "(Donald*) AND (Duck*)");
 116         }
 117     }
 118
 119     foreach my $koha_name ( keys %{ $koha_to_index_name } ) {
 120         my $query = $qb->build_authorities_query_compat( [ $koha_name ],  undef, undef, ['is'], [$search_term], 'AUTH_TYPE', 'asc' );
 121         if ( $koha_name eq 'all' || $koha_name eq 'any' ) {
 122             is( $query->{query}->{bool}->{must}[0]->{match_phrase}->{"_all.phrase"},
 123                 "donald duck");
 124         } else {
 125             is( $query->{query}->{bool}->{must}[0]->{match_phrase}->{$koha_to_index_name->{$koha_name}.".phrase"},
 126                 "donald duck");
 127         }
 128     }
 129
 130     foreach my $koha_name ( keys %{ $koha_to_index_name } ) {
 131         my $query = $qb->build_authorities_query_compat( [ $koha_name ],  undef, undef, ['start'], [$search_term], 'AUTH_TYPE', 'asc' );
 132         if ( $koha_name eq 'all' || $koha_name eq 'any' ) {
 133             is( $query->{query}->{bool}->{must}[0]->{match_phrase_prefix}->{"_all.phrase"},
 134                 "donald duck");
 135         } else {
 136             is( $query->{query}->{bool}->{must}[0]->{match_phrase_prefix}->{$koha_to_index_name->{$koha_name}.".phrase"},
 137                 "donald duck");
 138         }
 139     }
 140
 141     # Sorting
 142     my $query = $qb->build_authorities_query_compat( [ 'mainentry' ],  undef, undef, ['start'], [$search_term], 'AUTH_TYPE', 'HeadingAsc' );
 143     is_deeply(
 144         $query->{sort},
 145         [
 146             {
 147                 'heading__sort' => 'asc'
 148             }
 149         ],
 150         "ascending sort parameter properly formed"
 151     );
 152     $query = $qb->build_authorities_query_compat( [ 'mainentry' ],  undef, undef, ['start'], [$search_term], 'AUTH_TYPE', 'HeadingDsc' );
 153     is_deeply(
 154         $query->{sort},
 155         [
 156             {
 157                 'heading__sort' => 'desc'
 158             }
 159         ],
 160         "descending sort parameter properly formed"
 161     );
 162
 163     # Authorities type
 164     $query = $qb->build_authorities_query_compat( [ 'mainentry' ],  undef, undef, ['contains'], [$search_term], 'AUTH_TYPE', 'asc' );
 165     is_deeply(
 166         $query->{query}->{bool}->{filter},
 167         { term => { 'authtype' => 'auth_type' } },
 168         "authorities type code is used as filter"
 169     );
 170
 171     # Failing case
 172     throws_ok {
 173         $qb->build_authorities_query_compat( [ 'tomas' ],  undef, undef, ['contains'], [$search_term], 'AUTH_TYPE', 'asc' );
 174     }
 175     'Koha::Exceptions::WrongParameter',
 176         'Exception thrown on invalid value in the marclist param';
 177 };
 178
 179 subtest 'build_query tests' => sub {
 180     plan tests => 40;
 181
 182     my $qb;
 183
 184     ok(
 185         $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new({ 'index' => 'biblios' }),
 186         'Creating new query builder object for biblios'
 187     );
 188
 189     my @sort_by = 'title_asc';
 190     my @sort_params = $qb->_convert_sort_fields(@sort_by);
 191     my %options;
 192     $options{sort} = \@sort_params;
 193     my $query = $qb->build_query('test', %options);
 194
 195     is_deeply(
 196         $query->{sort},
 197         [
 198             {
 199             'title__sort.phrase' => {
 200                     'order' => 'asc'
 201                 }
 202             }
 203         ],
 204         "sort parameter properly formed"
 205     );
 206
 207     t::lib::Mocks::mock_preference('FacetMaxCount','37');
 208     $query = $qb->build_query('test', %options);
 209     ok( defined $query->{aggregations}{ccode}{terms}{size},'we need to ask for a size or we get only 5 facet' );
 210     is( $query->{aggregations}{ccode}{terms}{size}, 37,'we ask for the size as defined by the syspref FacetMaxCount');
 211
 212     t::lib::Mocks::mock_preference('DisplayLibraryFacets','both');
 213     $query = $qb->build_query();
 214     ok( defined $query->{aggregations}{homebranch},
 215         'homebranch added to facets if DisplayLibraryFacets=both' );
 216     ok( defined $query->{aggregations}{holdingbranch},
 217         'holdingbranch added to facets if DisplayLibraryFacets=both' );
 218     t::lib::Mocks::mock_preference('DisplayLibraryFacets','holding');
 219     $query = $qb->build_query();
 220     ok( !defined $query->{aggregations}{homebranch},
 221         'homebranch not added to facets if DisplayLibraryFacets=holding' );
 222     ok( defined $query->{aggregations}{holdingbranch},
 223         'holdingbranch added to facets if DisplayLibraryFacets=holding' );
 224     t::lib::Mocks::mock_preference('DisplayLibraryFacets','home');
 225     $query = $qb->build_query();
 226     ok( defined $query->{aggregations}{homebranch},
 227         'homebranch added to facets if DisplayLibraryFacets=home' );
 228     ok( !defined $query->{aggregations}{holdingbranch},
 229         'holdingbranch not added to facets if DisplayLibraryFacets=home' );
 230
 231     t::lib::Mocks::mock_preference( 'QueryAutoTruncate', '' );
 232
 233     ( undef, $query ) = $qb->build_query_compat( undef, ['donald duck'] );
 234     is(
 235         $query->{query}{query_string}{query},
 236         "(donald duck)",
 237         "query not altered if QueryAutoTruncate disabled"
 238     );
 239
 240     ( undef, $query ) = $qb->build_query_compat( undef, ['donald duck'], ['title'] );
 241     is(
 242         $query->{query}{query_string}{query},
 243         '(title:(donald duck))',
 244         'multiple words in a query term are enclosed in parenthesis'
 245     );
 246
 247     ( undef, $query ) = $qb->build_query_compat( ['AND'], ['donald duck', 'disney'], ['title', 'author'] );
 248     is(
 249         $query->{query}{query_string}{query},
 250         '(title:(donald duck)) AND (author:disney)',
 251         'multiple query terms are enclosed in parenthesis while a single one is not'
 252     );
 253
 254     my ($simple_query, $query_cgi, $query_desc);
 255     ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['"donald duck"', 'walt disney'], ['ti', 'au'] );
 256     is($query_cgi, 'idx=ti&q=%22donald%20duck%22&idx=au&q=walt%20disney', 'query cgi ok for multiterm query');
 257     is($query_desc, '(title:("donald duck")) (author:(walt disney))', 'query desc ok for multiterm query');
 258
 259     ( undef, $query ) = $qb->build_query_compat( undef, ['2019'], ['yr,st-year'] );
 260     is(
 261         $query->{query}{query_string}{query},
 262         '(date-of-publication:2019)',
 263         'Year in an st-year search is handled properly'
 264     );
 265
 266     ( undef, $query ) = $qb->build_query_compat( undef, ['2018-2019'], ['yr,st-year'] );
 267     is(
 268         $query->{query}{query_string}{query},
 269         '(date-of-publication:[2018 TO 2019])',
 270         'Year range in an st-year search is handled properly'
 271     );
 272
 273     ( undef, $query ) = $qb->build_query_compat( undef, ['-2019'], ['yr,st-year'] );
 274     is(
 275         $query->{query}{query_string}{query},
 276         '(date-of-publication:[* TO 2019])',
 277         'Open start year in year range of an st-year search is handled properly'
 278     );
 279
 280     ( undef, $query ) = $qb->build_query_compat( undef, ['2019-'], ['yr,st-year'] );
 281     is(
 282         $query->{query}{query_string}{query},
 283         '(date-of-publication:[2019 TO *])',
 284         'Open end year in year range of an st-year search is handled properly'
 285     );
 286
 287     ( undef, $query ) = $qb->build_query_compat( undef, ['2019-'], ['yr,st-year'], ['yr,st-numeric=-2019'] );
 288     is(
 289         $query->{query}{query_string}{query},
 290         '(date-of-publication:[2019 TO *]) AND copydate:[* TO 2019]',
 291         'Open end year in year range of an st-year search is handled properly'
 292     );
 293
 294     # Enable auto-truncation
 295     t::lib::Mocks::mock_preference( 'QueryAutoTruncate', '1' );
 296
 297     ( undef, $query ) = $qb->build_query_compat( undef, ['donald duck'] );
 298     is(
 299         $query->{query}{query_string}{query},
 300         "(donald* duck*)",
 301         "simple query is auto truncated when QueryAutoTruncate enabled"
 302     );
 303
 304     # Ensure reserved words are not truncated
 305     ( undef, $query ) = $qb->build_query_compat( undef,
 306         ['donald or duck and mickey not mouse'] );
 307     is(
 308         $query->{query}{query_string}{query},
 309         "(donald* or duck* and mickey* not mouse*)",
 310         "reserved words are not affected by QueryAutoTruncate"
 311     );
 312
 313     ( undef, $query ) = $qb->build_query_compat( undef, ['donald* duck*'] );
 314     is(
 315         $query->{query}{query_string}{query},
 316         "(donald* duck*)",
 317         "query with '*' is unaltered when QueryAutoTruncate is enabled"
 318     );
 319
 320     ( undef, $query ) = $qb->build_query_compat( undef, ['donald duck and the mouse'] );
 321     is(
 322         $query->{query}{query_string}{query},
 323         "(donald* duck* and the* mouse*)",
 324         "individual words are all truncated and stopwords ignored"
 325     );
 326
 327     ( undef, $query ) = $qb->build_query_compat( undef, ['*'] );
 328     is(
 329         $query->{query}{query_string}{query},
 330         "(*)",
 331         "query of just '*' is unaltered when QueryAutoTruncate is enabled"
 332     );
 333
 334     ( undef, $query ) = $qb->build_query_compat( undef, ['"donald duck"'] );
 335     is(
 336         $query->{query}{query_string}{query},
 337         '("donald duck")',
 338         "query with quotes is unaltered when QueryAutoTruncate is enabled"
 339     );
 340
 341
 342     ( undef, $query ) = $qb->build_query_compat( undef, ['"donald duck" and "the mouse"'] );
 343     is(
 344         $query->{query}{query_string}{query},
 345         '("donald duck" and "the mouse")',
 346         "all quoted strings are unaltered if more than one in query"
 347     );
 348
 349     ( undef, $query ) = $qb->build_query_compat( undef, ['barcode:123456'] );
 350     is(
 351         $query->{query}{query_string}{query},
 352         '(barcode:123456*)',
 353         "query of specific field is truncated"
 354     );
 355
 356     ( undef, $query ) = $qb->build_query_compat( undef, ['Local-number:"123456"'] );
 357     is(
 358         $query->{query}{query_string}{query},
 359         '(local-number:"123456")',
 360         "query of specific field including hyphen and quoted is not truncated, field name is converted to lower case"
 361     );
 362
 363     ( undef, $query ) = $qb->build_query_compat( undef, ['Local-number:123456'] );
 364     is(
 365         $query->{query}{query_string}{query},
 366         '(local-number:123456*)',
 367         "query of specific field including hyphen and not quoted is truncated, field name is converted to lower case"
 368     );
 369
 370     ( undef, $query ) = $qb->build_query_compat( undef, ['Local-number.raw:123456'] );
 371     is(
 372         $query->{query}{query_string}{query},
 373         '(local-number.raw:123456*)',
 374         "query of specific field including period and not quoted is truncated, field name is converted to lower case"
 375     );
 376
 377     ( undef, $query ) = $qb->build_query_compat( undef, ['Local-number.raw:"123456"'] );
 378     is(
 379         $query->{query}{query_string}{query},
 380         '(local-number.raw:"123456")',
 381         "query of specific field including period and quoted is not truncated, field name is converted to lower case"
 382     );
 383
 384     ( undef, $query ) = $qb->build_query_compat( undef, ['J.R.R'] );
 385     is(
 386         $query->{query}{query_string}{query},
 387         '(J.R.R*)',
 388         "query including period is truncated but not split at periods"
 389     );
 390
 391     ( undef, $query ) = $qb->build_query_compat( undef, ['title:"donald duck"'] );
 392     is(
 393         $query->{query}{query_string}{query},
 394         '(title:"donald duck")',
 395         "query of specific field is not truncated when surrounded by quotes"
 396     );
 397
 398     ( undef, $query ) = $qb->build_query_compat( undef, ['donald duck'], ['title'] );
 399     is(
 400         $query->{query}{query_string}{query},
 401         '(title:(donald* duck*))',
 402         'words of a multi-word term are properly truncated'
 403     );
 404
 405     ( undef, $query ) = $qb->build_query_compat( ['AND'], ['donald duck', 'disney'], ['title', 'author'] );
 406     is(
 407         $query->{query}{query_string}{query},
 408         '(title:(donald* duck*)) AND (author:disney*)',
 409         'words of a multi-word term and single-word term are properly truncated'
 410     );
 411
 412     ( undef, $query ) = $qb->build_query_compat( undef, ['title:"donald duck"'], undef, undef, undef, undef, undef, { suppress => 1 } );
 413     is(
 414         $query->{query}{query_string}{query},
 415         '(title:"donald duck") AND suppress:0',
 416         "query of specific field is added AND suppress:0"
 417     );
 418
 419     ( undef, $query, $simple_query, $query_cgi, $query_desc ) = $qb->build_query_compat( undef, ['title:"donald duck"'], undef, undef, undef, undef, undef, { suppress => 0 } );
 420     is(
 421         $query->{query}{query_string}{query},
 422         '(title:"donald duck")',
 423         "query of specific field is not added AND suppress:0"
 424     );
 425     is($query_cgi, 'idx=&q=title%3A%22donald%20duck%22', 'query cgi');
 426     is($query_desc, 'title:"donald duck"', 'query desc ok');
 427 };
 428
 429
 430 subtest 'build query from form subtests' => sub {
 431     plan tests => 5;
 432
 433     my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new({ 'index' => 'authorities' }),
 434     #when searching for authorities from a record the form returns marclist with blanks for unentered terms
 435     my @marclist = ('mainmainentry','mainentry','match', 'all');
 436     my @values   = ( undef,         'Hamilton',  undef,   undef);
 437     my @operator = ( 'contains', 'contains', 'contains', 'contains');
 438
 439     my $query = $qb->build_authorities_query_compat( \@marclist, undef,
 440                     undef, \@operator , \@values, 'AUTH_TYPE', 'asc' );
 441     is($query->{query}->{bool}->{must}[0]->{query_string}->{query}, "Hamilton*","Expected search is populated");
 442     is( scalar @{ $query->{query}->{bool}->{must} }, 1,"Only defined search is populated");
 443
 444     @values[2] = 'Jefferson';
 445     $query = $qb->build_authorities_query_compat( \@marclist, undef,
 446                     undef, \@operator , \@values, 'AUTH_TYPE', 'asc' );
 447     is($query->{query}->{bool}->{must}[0]->{query_string}->{query}, "Hamilton*","First index searched as expected");
 448     is($query->{query}->{bool}->{must}[1]->{query_string}->{query}, "Jefferson*","Second index searched when populated");
 449     is( scalar @{ $query->{query}->{bool}->{must} }, 2,"Only defined searches are populated");
 450
 451
 452 };
 453
 454 subtest 'build_query with weighted fields tests' => sub {
 455     plan tests => 4;
 456
 457     my $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new( { index => 'mydb' } );
 458     my $db_builder = t::lib::TestBuilder->new();
 459
 460     Koha::SearchFields->search({})->delete;
 461
 462     $db_builder->build({
 463         source => 'SearchField',
 464         value => {
 465             name    => 'acqdate',
 466             label   => 'acqdate',
 467             weight  => undef
 468         }
 469     });
 470
 471     $db_builder->build({
 472         source => 'SearchField',
 473         value => {
 474             name    => 'title',
 475             label   => 'title',
 476             weight  => 25
 477         }
 478     });
 479
 480     $db_builder->build({
 481         source => 'SearchField',
 482         value => {
 483             name    => 'subject',
 484             label   => 'subject',
 485             weight  => 15
 486         }
 487     });
 488
 489     my ( undef, $query ) = $qb->build_query_compat( undef, ['title:"donald duck"'], undef, undef,
 490     undef, undef, undef, { weighted_fields => 1 });
 491
 492     my $fields = $query->{query}{query_string}{fields};
 493     is(scalar(@$fields), 3, 'Search is done on 3 fields');
 494     is($fields->[0], '_all', 'First search field is _all');
 495     is($fields->[1], 'title^25.00', 'Second search field is title');
 496     is($fields->[2], 'subject^15.00', 'Third search field is subject');
 497 };
 498
 499 subtest "_convert_sort_fields() tests" => sub {
 500     plan tests => 3;
 501
 502     my $qb;
 503
 504     ok(
 505         $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new({ 'index' => 'biblios' }),
 506         'Creating new query builder object for biblios'
 507     );
 508
 509     my @sort_by = $qb->_convert_sort_fields(qw( call_number_asc author_dsc ));
 510     is_deeply(
 511         \@sort_by,
 512         [
 513             { field => 'local-classification', direction => 'asc' },
 514             { field => 'author',  direction => 'desc' }
 515         ],
 516         'sort fields should have been split correctly'
 517     );
 518
 519     # We could expect this to pass, but direction is undef instead of 'desc'
 520     @sort_by = $qb->_convert_sort_fields(qw( call_number_asc author_desc ));
 521     is_deeply(
 522         \@sort_by,
 523         [
 524             { field => 'local-classification', direction => 'asc' },
 525             { field => 'author',  direction => 'desc' }
 526         ],
 527         'sort fields should have been split correctly'
 528     );
 529 };
 530
 531 subtest "_sort_field() tests" => sub {
 532     plan tests => 5;
 533
 534     my $qb;
 535
 536     ok(
 537         $qb = Koha::SearchEngine::Elasticsearch::QueryBuilder->new({ 'index' => 'biblios' }),
 538         'Creating new query builder object for biblios'
 539     );
 540
 541     my $f = $qb->_sort_field('title');
 542     is(
 543         $f,
 544         'title__sort.phrase',
 545         'title sort mapped correctly'
 546     );
 547
 548     $f = $qb->_sort_field('subject');
 549     is(
 550         $f,
 551         'subject.raw',
 552         'subject sort mapped correctly'
 553     );
 554
 555     $f = $qb->_sort_field('itemnumber');
 556     is(
 557         $f,
 558         'itemnumber',
 559         'itemnumber sort mapped correctly'
 560     );
 561
 562     $f = $qb->_sort_field('sortablenumber');
 563     is(
 564         $f,
 565         'sortablenumber__sort',
 566         'sortablenumber sort mapped correctly'
 567     );
 568 };
 569
 570 $schema->storage->txn_rollback;