C4/Matcher.pm

   1 package C4::Matcher;
   2
   3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
   4 #
   5 # This file is part of Koha.
   6 #
   7 # Koha is free software; you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # Koha is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  19
  20 use Modern::Perl;
  21
  22 use MARC::Record;
  23
  24 use Koha::SearchEngine;
  25 use Koha::SearchEngine::Search;
  26 use Koha::SearchEngine::QueryBuilder;
  27 use Koha::Util::Normalize qw/legacy_default remove_spaces upper_case lower_case ISBN/;
  28
  29 =head1 NAME
  30
  31 C4::Matcher - find MARC records matching another one
  32
  33 =head1 SYNOPSIS
  34
  35   my @matchers = C4::Matcher::GetMatcherList();
  36
  37   my $matcher = C4::Matcher->new($record_type);
  38   $matcher->threshold($threshold);
  39   $matcher->code($code);
  40   $matcher->description($description);
  41
  42   $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
  43   $matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
  44   $matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
  45
  46   $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
  47   $matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
  48                                [ { tag => '245', subfields => 'a', norms => [] } ]);
  49
  50   my @matches = $matcher->get_matches($marc_record, $max_matches);
  51
  52   foreach $match (@matches) {
  53
  54       # matches already sorted in order of
  55       # decreasing score
  56       print "record ID: $match->{'record_id'};
  57       print "score:     $match->{'score'};
  58
  59   }
  60
  61   my $matcher_description = $matcher->dump();
  62
  63 =head1 FUNCTIONS
  64
  65 =cut
  66
  67 =head2 GetMatcherList
  68
  69   my @matchers = C4::Matcher::GetMatcherList();
  70
  71 Returns an array of hashrefs list all matchers
  72 present in the database.  Each hashref includes:
  73
  74  * matcher_id
  75  * code
  76  * description
  77
  78 =cut
  79
  80 sub GetMatcherList {
  81     my $dbh = C4::Context->dbh;
  82
  83     my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
  84     $sth->execute();
  85     my @results = ();
  86     while (my $row = $sth->fetchrow_hashref) {
  87         push @results, $row;
  88     }
  89     return @results;
  90 }
  91
  92 =head2 GetMatcherId
  93
  94   my $matcher_id = C4::Matcher::GetMatcherId($code);
  95
  96 Returns the matcher_id of a code.
  97
  98 =cut
  99
 100 sub GetMatcherId {
 101     my ($code) = @_;
 102     my $dbh = C4::Context->dbh;
 103
 104     my $matcher_id = $dbh->selectrow_array("SELECT matcher_id FROM marc_matchers WHERE code = ?", undef, $code);
 105     return $matcher_id;
 106 }
 107
 108 =head1 METHODS
 109
 110 =head2 new
 111
 112   my $matcher = C4::Matcher->new($record_type, $threshold);
 113
 114 Creates a new Matcher.  C<$record_type> indicates which search
 115 database to use, e.g., 'biblio' or 'authority' and defaults to
 116 'biblio', while C<$threshold> is the minimum score required for a match
 117 and defaults to 1000.
 118
 119 =cut
 120
 121 sub new {
 122     my $class = shift;
 123     my $self = {};
 124
 125     $self->{'id'} = undef;
 126
 127     if ($#_ > -1) {
 128         $self->{'record_type'} = shift;
 129     } else {
 130         $self->{'record_type'} = 'biblio';
 131     }
 132
 133     if ($#_ > -1) {
 134         $self->{'threshold'} = shift;
 135     } else {
 136         $self->{'threshold'} = 1000;
 137     }
 138
 139     $self->{'code'} = '';
 140     $self->{'description'} = '';
 141
 142     $self->{'matchpoints'} = [];
 143     $self->{'required_checks'} = [];
 144
 145     bless $self, $class;
 146     return $self;
 147 }
 148
 149 =head2 fetch
 150
 151   my $matcher = C4::Matcher->fetch($id);
 152
 153 Creates a matcher object from the version stored
 154 in the database.  If a matcher with the given
 155 id does not exist, returns undef.
 156
 157 =cut
 158
 159 sub fetch {
 160     my $class = shift;
 161     my $id = shift;
 162     my $dbh = C4::Context->dbh();
 163
 164     my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
 165     $sth->execute($id);
 166     my $row = $sth->fetchrow_hashref;
 167     $sth->finish();
 168     return undef unless defined $row;
 169
 170     my $self = {};
 171     $self->{'id'} = $row->{'matcher_id'};
 172     $self->{'record_type'} = $row->{'record_type'};
 173     $self->{'code'} = $row->{'code'};
 174     $self->{'description'} = $row->{'description'};
 175     $self->{'threshold'} = int($row->{'threshold'});
 176     bless $self, $class;
 177
 178     # matchpoints
 179     $self->{'matchpoints'} = [];
 180     $sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
 181     $sth->execute($self->{'id'});
 182     while (my $row = $sth->fetchrow_hashref) {
 183         my $matchpoint = $self->_fetch_matchpoint($row->{'matchpoint_id'});
 184         push @{ $self->{'matchpoints'} }, $matchpoint;
 185     }
 186
 187     # required checks
 188     $self->{'required_checks'} = [];
 189     $sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
 190     $sth->execute($self->{'id'});
 191     while (my $row = $sth->fetchrow_hashref) {
 192         my $source_matchpoint = $self->_fetch_matchpoint($row->{'source_matchpoint_id'});
 193         my $target_matchpoint = $self->_fetch_matchpoint($row->{'target_matchpoint_id'});
 194         my $matchcheck = {};
 195         $matchcheck->{'source_matchpoint'} = $source_matchpoint;
 196         $matchcheck->{'target_matchpoint'} = $target_matchpoint;
 197         push @{ $self->{'required_checks'} }, $matchcheck;
 198     }
 199
 200     return $self;
 201 }
 202
 203 sub _fetch_matchpoint {
 204     my $self = shift;
 205     my $matchpoint_id = shift;
 206
 207     my $dbh = C4::Context->dbh;
 208     my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
 209     $sth->execute($matchpoint_id);
 210     my $row = $sth->fetchrow_hashref;
 211     my $matchpoint = {};
 212     $matchpoint->{'index'} = $row->{'search_index'};
 213     $matchpoint->{'score'} = int($row->{'score'});
 214     $sth->finish();
 215
 216     $matchpoint->{'components'} = [];
 217     $sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
 218     $sth->execute($matchpoint_id);
 219     while ($row = $sth->fetchrow_hashref) {
 220         my $component = {};
 221         $component->{'tag'} = $row->{'tag'};
 222         $component->{'subfields'} = { map { $_ => 1 } split(//,  $row->{'subfields'}) };
 223         $component->{'offset'} = int($row->{'offset'});
 224         $component->{'length'} = int($row->{'length'});
 225         $component->{'norms'} = [];
 226         my $sth2 = $dbh->prepare_cached("SELECT *
 227                                          FROM matchpoint_component_norms
 228                                          WHERE matchpoint_component_id = ? ORDER BY sequence");
 229         $sth2->execute($row->{'matchpoint_component_id'});
 230         while (my $row2 = $sth2->fetchrow_hashref) {
 231             push @{ $component->{'norms'} }, $row2->{'norm_routine'};
 232         }
 233         push @{ $matchpoint->{'components'} }, $component;
 234     }
 235     return $matchpoint;
 236 }
 237
 238 =head2 store
 239
 240   my $id = $matcher->store();
 241
 242 Stores matcher in database.  The return value is the ID
 243 of the marc_matchers row.  If the matcher was
 244 previously retrieved from the database via the fetch()
 245 method, the DB representation of the matcher
 246 is replaced.
 247
 248 =cut
 249
 250 sub store {
 251     my $self = shift;
 252
 253     if (defined $self->{'id'}) {
 254         # update
 255         $self->_del_matcher_components();
 256         $self->_update_marc_matchers();
 257     } else {
 258         # create new
 259         $self->_new_marc_matchers();
 260     }
 261     $self->_store_matcher_components();
 262     return $self->{'id'};
 263 }
 264
 265 sub _del_matcher_components {
 266     my $self = shift;
 267
 268     my $dbh = C4::Context->dbh();
 269     my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
 270     $sth->execute($self->{'id'});
 271     $sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
 272     $sth->execute($self->{'id'});
 273     # foreign key delete cascades take care of deleting relevant rows
 274     # from matcher_matchpoints, matchpoint_components, and
 275     # matchpoint_component_norms
 276 }
 277
 278 sub _update_marc_matchers {
 279     my $self = shift;
 280
 281     my $dbh = C4::Context->dbh();
 282     my $sth = $dbh->prepare_cached("UPDATE marc_matchers
 283                                     SET code = ?,
 284                                         description = ?,
 285                                         record_type = ?,
 286                                         threshold = ?
 287                                     WHERE matcher_id = ?");
 288     $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'}, $self->{'id'});
 289 }
 290
 291 sub _new_marc_matchers {
 292     my $self = shift;
 293
 294     my $dbh = C4::Context->dbh();
 295     my $sth = $dbh->prepare_cached("INSERT INTO marc_matchers
 296                                     (code, description, record_type, threshold)
 297                                     VALUES (?, ?, ?, ?)");
 298     $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'});
 299     $self->{'id'} = $dbh->{'mysql_insertid'};
 300 }
 301
 302 sub _store_matcher_components {
 303     my $self = shift;
 304
 305     my $dbh = C4::Context->dbh();
 306     my $sth;
 307     my $matcher_id = $self->{'id'};
 308     foreach my $matchpoint (@{ $self->{'matchpoints'}}) {
 309         my $matchpoint_id = $self->_store_matchpoint($matchpoint);
 310         $sth = $dbh->prepare_cached("INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
 311                                      VALUES (?, ?)");
 312         $sth->execute($matcher_id, $matchpoint_id);
 313     }
 314     foreach my $matchcheck (@{ $self->{'required_checks'} }) {
 315         my $source_matchpoint_id = $self->_store_matchpoint($matchcheck->{'source_matchpoint'});
 316         my $target_matchpoint_id = $self->_store_matchpoint($matchcheck->{'target_matchpoint'});
 317         $sth = $dbh->prepare_cached("INSERT INTO matchchecks
 318                                      (matcher_id, source_matchpoint_id, target_matchpoint_id)
 319                                      VALUES (?, ?, ?)");
 320         $sth->execute($matcher_id, $source_matchpoint_id,  $target_matchpoint_id);
 321     }
 322
 323 }
 324
 325 sub _store_matchpoint {
 326     my $self = shift;
 327     my $matchpoint = shift;
 328
 329     my $dbh = C4::Context->dbh();
 330     my $sth;
 331     my $matcher_id = $self->{'id'};
 332     $sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
 333                                  VALUES (?, ?, ?)");
 334     $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'}||0);
 335     my $matchpoint_id = $dbh->{'mysql_insertid'};
 336     my $seqnum = 0;
 337     foreach my $component (@{ $matchpoint->{'components'} }) {
 338         $seqnum++;
 339         $sth = $dbh->prepare_cached("INSERT INTO matchpoint_components
 340                                      (matchpoint_id, sequence, tag, subfields, offset, length)
 341                                      VALUES (?, ?, ?, ?, ?, ?)");
 342         $sth->bind_param(1, $matchpoint_id);
 343         $sth->bind_param(2, $seqnum);
 344         $sth->bind_param(3, $component->{'tag'});
 345         $sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
 346         $sth->bind_param(5, $component->{'offset'}||0);
 347         $sth->bind_param(6, $component->{'length'});
 348         $sth->execute();
 349         my $matchpoint_component_id = $dbh->{'mysql_insertid'};
 350         my $normseq = 0;
 351         foreach my $norm (@{ $component->{'norms'} }) {
 352             $normseq++;
 353             $sth = $dbh->prepare_cached("INSERT INTO matchpoint_component_norms
 354                                          (matchpoint_component_id, sequence, norm_routine)
 355                                          VALUES (?, ?, ?)");
 356             $sth->execute($matchpoint_component_id, $normseq, $norm);
 357         }
 358     }
 359     return $matchpoint_id;
 360 }
 361
 362
 363 =head2 delete
 364
 365   C4::Matcher->delete($id);
 366
 367 Deletes the matcher of the specified ID
 368 from the database.
 369
 370 =cut
 371
 372 sub delete {
 373     my $class = shift;
 374     my $matcher_id = shift;
 375
 376     my $dbh = C4::Context->dbh;
 377     my $sth = $dbh->prepare("DELETE FROM marc_matchers WHERE matcher_id = ?");
 378     $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
 379 }
 380
 381 =head2 record_type
 382
 383   $matcher->record_type('biblio');
 384   my $record_type = $matcher->record_type();
 385
 386 Accessor method.
 387
 388 =cut
 389
 390 sub record_type {
 391     my $self = shift;
 392     @_ ? $self->{'record_type'} = shift : $self->{'record_type'};
 393 }
 394
 395 =head2 threshold
 396
 397   $matcher->threshold(1000);
 398   my $threshold = $matcher->threshold();
 399
 400 Accessor method.
 401
 402 =cut
 403
 404 sub threshold {
 405     my $self = shift;
 406     @_ ? $self->{'threshold'} = shift : $self->{'threshold'};
 407 }
 408
 409 =head2 _id
 410
 411   $matcher->_id(123);
 412   my $id = $matcher->_id();
 413
 414 Accessor method.  Note that using this method
 415 to set the DB ID of the matcher should not be
 416 done outside of the editing CGI.
 417
 418 =cut
 419
 420 sub _id {
 421     my $self = shift;
 422     @_ ? $self->{'id'} = shift : $self->{'id'};
 423 }
 424
 425 =head2 code
 426
 427   $matcher->code('ISBN');
 428   my $code = $matcher->code();
 429
 430 Accessor method.
 431
 432 =cut
 433
 434 sub code {
 435     my $self = shift;
 436     @_ ? $self->{'code'} = shift : $self->{'code'};
 437 }
 438
 439 =head2 description
 440
 441   $matcher->description('match on ISBN');
 442   my $description = $matcher->description();
 443
 444 Accessor method.
 445
 446 =cut
 447
 448 sub description {
 449     my $self = shift;
 450     @_ ? $self->{'description'} = shift : $self->{'description'};
 451 }
 452
 453 =head2 add_matchpoint
 454
 455   $matcher->add_matchpoint($index, $score, $matchcomponents);
 456
 457 Adds a matchpoint that may include multiple components.  The $index
 458 parameter identifies the index that will be searched, while $score
 459 is the weight that will be added if a match is found.
 460
 461 $matchcomponents should be a reference to an array of matchpoint
 462 compoents, each of which should be a hash containing the following
 463 keys:
 464     tag
 465     subfields
 466     offset
 467     length
 468     norms
 469
 470 The normalization_rules value should in turn be a reference to an
 471 array, each element of which should be a reference to a
 472 normalization subroutine (under C4::Normalize) to be applied
 473 to the source string.
 474
 475 =cut
 476
 477 sub add_matchpoint {
 478     my $self = shift;
 479     my ($index, $score, $matchcomponents) = @_;
 480
 481     my $matchpoint = {};
 482     $matchpoint->{'index'} = $index;
 483     $matchpoint->{'score'} = $score;
 484     $matchpoint->{'components'} = [];
 485     foreach my $input_component (@{ $matchcomponents }) {
 486         push @{ $matchpoint->{'components'} }, _parse_match_component($input_component);
 487     }
 488     push @{ $self->{'matchpoints'} }, $matchpoint;
 489 }
 490
 491 =head2 add_simple_matchpoint
 492
 493   $matcher->add_simple_matchpoint($index, $score, $source_tag,
 494                             $source_subfields, $source_offset,
 495                             $source_length, $source_normalizer);
 496
 497
 498 Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
 499 normalized per the normalization fuction, search the index.  All records retrieved
 500 will receive the assigned score.
 501
 502 =cut
 503
 504 sub add_simple_matchpoint {
 505     my $self = shift;
 506     my ($index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer) = @_;
 507
 508     $self->add_matchpoint($index, $score, [
 509                           { tag => $source_tag, subfields => $source_subfields,
 510                             offset => $source_offset, 'length' => $source_length,
 511                             norms => [ $source_normalizer ]
 512                           }
 513                          ]);
 514 }
 515
 516 =head2 add_required_check
 517
 518   $match->add_required_check($source_matchpoint, $target_matchpoint);
 519
 520 Adds a required check definition.  A required check means that in
 521 order for a match to be considered valid, the key derived from the
 522 source (incoming) record must match the key derived from the target
 523 (already in DB) record.
 524
 525 Unlike a regular matchpoint, only the first repeat of each tag
 526 in the source and target match criteria are considered.
 527
 528 A typical example of a required check would be verifying that the
 529 titles and publication dates match.
 530
 531 $source_matchpoint and $target_matchpoint are each a reference to
 532 an array of hashes, where each hash follows the same definition
 533 as the matchpoint component specification in add_matchpoint, i.e.,
 534
 535     tag
 536     subfields
 537     offset
 538     length
 539     norms
 540
 541 The normalization_rules value should in turn be a reference to an
 542 array, each element of which should be a reference to a
 543 normalization subroutine (under C4::Normalize) to be applied
 544 to the source string.
 545
 546 =cut
 547
 548 sub add_required_check {
 549     my $self = shift;
 550     my ($source_matchpoint, $target_matchpoint) = @_;
 551
 552     my $matchcheck = {};
 553     $matchcheck->{'source_matchpoint'}->{'index'} = '';
 554     $matchcheck->{'source_matchpoint'}->{'score'} = 0;
 555     $matchcheck->{'source_matchpoint'}->{'components'} = [];
 556     $matchcheck->{'target_matchpoint'}->{'index'} = '';
 557     $matchcheck->{'target_matchpoint'}->{'score'} = 0;
 558     $matchcheck->{'target_matchpoint'}->{'components'} = [];
 559     foreach my $input_component (@{ $source_matchpoint }) {
 560         push @{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component($input_component);
 561     }
 562     foreach my $input_component (@{ $target_matchpoint }) {
 563         push @{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component($input_component);
 564     }
 565     push @{ $self->{'required_checks'} }, $matchcheck;
 566 }
 567
 568 =head2 add_simple_required_check
 569
 570   $matcher->add_simple_required_check($source_tag, $source_subfields,
 571                 $source_offset, $source_length, $source_normalizer,
 572                 $target_tag, $target_subfields, $target_offset,
 573                 $target_length, $target_normalizer);
 574
 575 Adds a required check, which requires that the normalized keys made from the source and targets
 576 must match for a match to be considered valid.
 577
 578 =cut
 579
 580 sub add_simple_required_check {
 581     my $self = shift;
 582     my ($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
 583         $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer) = @_;
 584
 585     $self->add_required_check(
 586       [ { tag => $source_tag, subfields => $source_subfields, offset => $source_offset, 'length' => $source_length,
 587           norms => [ $source_normalizer ] } ],
 588       [ { tag => $target_tag, subfields => $target_subfields, offset => $target_offset, 'length' => $target_length,
 589           norms => [ $target_normalizer ] } ]
 590     );
 591 }
 592
 593 =head2 get_matches
 594
 595   my @matches = $matcher->get_matches($marc_record, $max_matches);
 596   foreach $match (@matches) {
 597       # matches already sorted in order of
 598       # decreasing score
 599       print "record ID: $match->{'record_id'};
 600       print "score:     $match->{'score'};
 601   }
 602
 603 Identifies all of the records matching the given MARC record.  For a record already
 604 in the database to be considered a match, it must meet the following criteria:
 605
 606 =over 2
 607
 608 =item 1. Total score from its matching field must exceed the supplied threshold.
 609
 610 =item 2. It must pass all required checks.
 611
 612 =back
 613
 614 Only the top $max_matches matches are returned.  The returned array is sorted
 615 in order of decreasing score, i.e., the best match is first.
 616
 617 =cut
 618
 619 sub get_matches {
 620     my $self = shift;
 621     my ($source_record, $max_matches) = @_;
 622
 623     my $matches = {};
 624
 625     my $QParser;
 626     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 627     foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
 628         my @source_keys = _get_match_keys( $source_record, $matchpoint );
 629
 630         next if scalar(@source_keys) == 0;
 631
 632         # FIXME - because of a bug in QueryParser, an expression ofthe
 633         # format 'isbn:"isbn1" || isbn:"isbn2" || isbn"isbn3"...'
 634         # does not get parsed correctly, so we will not
 635         # do AggressiveMatchOnISBN if UseQueryParser is on
 636         @source_keys = C4::Koha::GetVariationsOfISBNs(@source_keys)
 637           if ( $matchpoint->{index} =~ /^isbn$/i
 638             && C4::Context->preference('AggressiveMatchOnISBN') )
 639             && !C4::Context->preference('UseQueryParser');
 640
 641         @source_keys = C4::Koha::GetVariationsOfISSNs(@source_keys)
 642           if ( $matchpoint->{index} =~ /^issn$/i
 643             && C4::Context->preference('AggressiveMatchOnISSN') )
 644             && !C4::Context->preference('UseQueryParser');
 645
 646         # build query
 647         my $query;
 648         my $error;
 649         my $searchresults;
 650         my $total_hits;
 651         if ( $self->{'record_type'} eq 'biblio' ) {
 652
 653             #NOTE: The QueryParser can't handle the CCL syntax of 'qualifier','qualifier', so fallback to non-QueryParser.
 654             #NOTE: You can see this in C4::Search::SimpleSearch() as well in a different way.
 655             if ($QParser && $matchpoint->{'index'} !~ m/\w,\w/) {
 656                 $query = join( " || ",
 657                     map { "$matchpoint->{'index'}:$_" } @source_keys );
 658             }
 659             else {
 660                 my $phr = ( C4::Context->preference('AggressiveMatchOnISBN') || C4::Context->preference('AggressiveMatchOnISSN') )  ? ',phr' : q{};
 661                 $query = join( " OR ",
 662                     map { "$matchpoint->{'index'}$phr=\"$_\"" } @source_keys );
 663                     #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
 664             }
 665
 666             # Use state variables to avoid recreating the objects every time.
 667             # With Elasticsearch this also avoids creating a massive amount of
 668             # ES connectors that would eventually run out of file descriptors.
 669             state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
 670             ( $error, $searchresults, $total_hits ) =
 671               $searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
 672
 673             if ( defined $error ) {
 674                 warn "search failed ($query) $error";
 675             }
 676             else {
 677                 foreach my $matched ( @{$searchresults} ) {
 678                     my $target_record = C4::Search::new_record_from_zebra( 'biblioserver', $matched );
 679                     my ( $biblionumber_tag, $biblionumber_subfield ) = C4::Biblio::GetMarcFromKohaField( "biblio.biblionumber" );
 680                     my $id = ( $biblionumber_tag > 10 ) ?
 681                         $target_record->field($biblionumber_tag)->subfield($biblionumber_subfield) :
 682                         $target_record->field($biblionumber_tag)->data();
 683                     $matches->{$id}->{score} += $matchpoint->{score};
 684                     $matches->{$id}->{record} = $target_record;
 685                 }
 686             }
 687
 688         }
 689         elsif ( $self->{'record_type'} eq 'authority' ) {
 690             my @marclist;
 691             my @and_or;
 692             my @excluding = [];
 693             my @operator;
 694             my @value;
 695             foreach my $key (@source_keys) {
 696                 push @marclist, $matchpoint->{'index'};
 697                 push @and_or,   'or';
 698                 push @operator, 'exact';
 699                 push @value,    $key;
 700             }
 701             # Use state variables to avoid recreating the objects every time.
 702             # With Elasticsearch this also avoids creating a massive amount of
 703             # ES connectors that would eventually run out of file descriptors.
 704             state $builder  = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
 705             state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
 706             my $search_query = $builder->build_authorities_query_compat(
 707                 \@marclist, \@and_or, \@excluding, \@operator,
 708                 \@value, undef, 'AuthidAsc'
 709             );
 710             my ( $authresults, $total ) = $searcher->search_auth_compat( $search_query, 0, 20 );
 711
 712             foreach my $result (@$authresults) {
 713                 my $id = $result->{authid};
 714                 $matches->{$id}->{score} += $matchpoint->{'score'};
 715                 $matches->{$id}->{record} = $id;
 716             }
 717         }
 718     }
 719
 720     # get rid of any that don't meet the threshold
 721     $matches = { map { ($matches->{$_}->{score} >= $self->{'threshold'}) ? ($_ => $matches->{$_}) : () } keys %$matches };
 722
 723     my @results = ();
 724     if ($self->{'record_type'} eq 'biblio') {
 725         require C4::Biblio;
 726         # get rid of any that don't meet the required checks
 727         $matches = {
 728             map {
 729                 _passes_required_checks( $source_record, $_, $self->{'required_checks'} )
 730                   ? ( $_ => $matches->{$_} )
 731                   : ()
 732             } keys %$matches
 733         };
 734
 735         foreach my $id ( keys %$matches ) {
 736             push @results, {
 737                 record_id => $id,
 738                 score     => $matches->{$id}->{score}
 739             };
 740         }
 741     } elsif ($self->{'record_type'} eq 'authority') {
 742         require C4::AuthoritiesMarc;
 743         foreach my $id (keys %$matches) {
 744             push @results, {
 745                 record_id => $id,
 746                 score     => $matches->{$id}->{score}
 747             };
 748         }
 749     }
 750     @results = sort {
 751         $b->{'score'} cmp $a->{'score'} or
 752         $b->{'record_id'} cmp $a->{'record_id'}
 753     } @results;
 754     if (scalar(@results) > $max_matches) {
 755         @results = @results[0..$max_matches-1];
 756     }
 757     return @results;
 758 }
 759
 760 =head2 dump
 761
 762   $description = $matcher->dump();
 763
 764 Returns a reference to a structure containing all of the information
 765 in the matcher object.  This is mainly a convenience method to
 766 aid setting up a HTML editing form.
 767
 768 =cut
 769
 770 sub dump {
 771     my $self = shift;
 772
 773     my $result = {};
 774
 775     $result->{'matcher_id'} = $self->{'id'};
 776     $result->{'code'} = $self->{'code'};
 777     $result->{'description'} = $self->{'description'};
 778     $result->{'record_type'} = $self->{'record_type'};
 779
 780     $result->{'matchpoints'} = [];
 781     foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
 782         push @{  $result->{'matchpoints'} }, $matchpoint;
 783     }
 784     $result->{'matchchecks'} = [];
 785     foreach my $matchcheck (@{ $self->{'required_checks'} }) {
 786         push @{  $result->{'matchchecks'} }, $matchcheck;
 787     }
 788
 789     return $result;
 790 }
 791
 792 sub _passes_required_checks {
 793     my ($source_record, $target_blob, $matchchecks) = @_;
 794     my $target_record = MARC::Record->new_from_usmarc($target_blob); # FIXME -- need to avoid parsing record twice
 795
 796     # no checks supplied == automatic pass
 797     return 1 if $#{ $matchchecks } == -1;
 798
 799     foreach my $matchcheck (@{ $matchchecks }) {
 800         my $source_key = join "", _get_match_keys($source_record, $matchcheck->{'source_matchpoint'});
 801         my $target_key = join "", _get_match_keys($target_record, $matchcheck->{'target_matchpoint'});
 802         return 0 unless $source_key eq $target_key;
 803     }
 804     return 1;
 805 }
 806
 807 sub _get_match_keys {
 808
 809     my $source_record = shift;
 810     my $matchpoint = shift;
 811     my $check_only_first_repeat = @_ ? shift : 0;
 812
 813     # If there is more than one component to the matchpoint (e.g.,
 814     # matchpoint includes both 003 and 001), any repeats
 815     # of the first component's tag are identified; repeats
 816     # of the subsequent components' tags are appended to
 817     # each parallel key dervied from the first component,
 818     # up to the number of repeats of the first component's tag.
 819     #
 820     # For example, if the record has one 003 and two 001s, only
 821     # one key is retrieved because there is only one 003.  The key
 822     # will consist of the contents of the first 003 and first 001.
 823     #
 824     # If there are two 003s and two 001s, there will be two keys:
 825     #    first 003 + first 001
 826     #    second 003 + second 001
 827
 828     my @keys = ();
 829     for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
 830         my $component = $matchpoint->{'components'}->[$i];
 831         my $j = -1;
 832         FIELD: foreach my $field ($source_record->field($component->{'tag'})) {
 833             $j++;
 834             last FIELD if $j > 0 and $check_only_first_repeat;
 835             last FIELD if $i > 0 and $j > $#keys;
 836
 837             my $string;
 838             if ( $field->is_control_field() ) {
 839                 $string = $field->data();
 840             } else {
 841                 $string = $field->as_string(
 842                     join('', keys %{ $component->{ subfields } }), ' ' # ' ' as separator
 843                 );
 844             }
 845
 846             if ($component->{'length'}>0) {
 847                 $string= substr($string, $component->{'offset'}, $component->{'length'});
 848             } elsif ($component->{'offset'}) {
 849                 $string= substr($string, $component->{'offset'});
 850             }
 851
 852             my $norms = $component->{'norms'};
 853             my $key = $string;
 854
 855             foreach my $norm ( @{ $norms } ) {
 856                 if ( grep { $norm eq $_ } valid_normalization_routines() ) {
 857                     if ( $norm eq 'remove_spaces' ) {
 858                         $key = remove_spaces($key);
 859                     }
 860                     elsif ( $norm eq 'upper_case' ) {
 861                         $key = upper_case($key);
 862                     }
 863                     elsif ( $norm eq 'lower_case' ) {
 864                         $key = lower_case($key);
 865                     }
 866                     elsif ( $norm eq 'legacy_default' ) {
 867                         $key = legacy_default($key);
 868                     }
 869                     elsif ( $norm eq 'ISBN' ) {
 870                         $key = ISBN($key);
 871                     }
 872                 } else {
 873                     warn "Invalid normalization routine required ($norm)"
 874                         unless $norm eq 'none';
 875                 }
 876             }
 877
 878             if ($i == 0) {
 879                 push @keys, $key if $key;
 880             } else {
 881                 $keys[$j] .= " $key" if $key;
 882             }
 883         }
 884     }
 885     return @keys;
 886 }
 887
 888
 889 sub _parse_match_component {
 890     my $input_component = shift;
 891
 892     my $component = {};
 893     $component->{'tag'} = $input_component->{'tag'};
 894     $component->{'subfields'} = { map { $_ => 1 } split(//, $input_component->{'subfields'}) };
 895     $component->{'offset'} = exists($input_component->{'offset'}) ? $input_component->{'offset'} : -1;
 896     $component->{'length'} = $input_component->{'length'} ? $input_component->{'length'} : 0;
 897     $component->{'norms'} =  $input_component->{'norms'} ? $input_component->{'norms'} : [];
 898
 899     return $component;
 900 }
 901
 902 sub valid_normalization_routines {
 903
 904     return (
 905         'remove_spaces',
 906         'upper_case',
 907         'lower_case',
 908         'legacy_default',
 909         'ISBN'
 910     );
 911 }
 912
 913 1;
 914 __END__
 915
 916 =head1 AUTHOR
 917
 918 Koha Development Team <http://koha-community.org/>
 919
 920 Galen Charlton <galen.charlton@liblime.com>
 921
 922 =cut