C4/Matcher.pm

   1 package C4::Matcher;
   2
   3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
   4 #
   5 # This file is part of Koha.
   6 #
   7 # Koha is free software; you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # Koha is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  19
  20 use Modern::Perl;
  21
  22 use MARC::Record;
  23
  24 use Koha::SearchEngine;
  25 use Koha::SearchEngine::Search;
  26 use Koha::Util::Normalize qw/legacy_default remove_spaces upper_case lower_case/;
  27
  28 =head1 NAME
  29
  30 C4::Matcher - find MARC records matching another one
  31
  32 =head1 SYNOPSIS
  33
  34   my @matchers = C4::Matcher::GetMatcherList();
  35
  36   my $matcher = C4::Matcher->new($record_type);
  37   $matcher->threshold($threshold);
  38   $matcher->code($code);
  39   $matcher->description($description);
  40
  41   $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
  42   $matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
  43   $matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
  44
  45   $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
  46   $matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
  47                                [ { tag => '245', subfields => 'a', norms => [] } ]);
  48
  49   my @matches = $matcher->get_matches($marc_record, $max_matches);
  50
  51   foreach $match (@matches) {
  52
  53       # matches already sorted in order of
  54       # decreasing score
  55       print "record ID: $match->{'record_id'};
  56       print "score:     $match->{'score'};
  57
  58   }
  59
  60   my $matcher_description = $matcher->dump();
  61
  62 =head1 FUNCTIONS
  63
  64 =cut
  65
  66 =head2 GetMatcherList
  67
  68   my @matchers = C4::Matcher::GetMatcherList();
  69
  70 Returns an array of hashrefs list all matchers
  71 present in the database.  Each hashref includes:
  72
  73  * matcher_id
  74  * code
  75  * description
  76
  77 =cut
  78
  79 sub GetMatcherList {
  80     my $dbh = C4::Context->dbh;
  81
  82     my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
  83     $sth->execute();
  84     my @results = ();
  85     while (my $row = $sth->fetchrow_hashref) {
  86         push @results, $row;
  87     }
  88     return @results;
  89 }
  90
  91 =head2 GetMatcherId
  92
  93   my $matcher_id = C4::Matcher::GetMatcherId($code);
  94
  95 Returns the matcher_id of a code.
  96
  97 =cut
  98
  99 sub GetMatcherId {
 100     my ($code) = @_;
 101     my $dbh = C4::Context->dbh;
 102
 103     my $matcher_id = $dbh->selectrow_array("SELECT matcher_id FROM marc_matchers WHERE code = ?", undef, $code);
 104     return $matcher_id;
 105 }
 106
 107 =head1 METHODS
 108
 109 =head2 new
 110
 111   my $matcher = C4::Matcher->new($record_type, $threshold);
 112
 113 Creates a new Matcher.  C<$record_type> indicates which search
 114 database to use, e.g., 'biblio' or 'authority' and defaults to
 115 'biblio', while C<$threshold> is the minimum score required for a match
 116 and defaults to 1000.
 117
 118 =cut
 119
 120 sub new {
 121     my $class = shift;
 122     my $self = {};
 123
 124     $self->{'id'} = undef;
 125
 126     if ($#_ > -1) {
 127         $self->{'record_type'} = shift;
 128     } else {
 129         $self->{'record_type'} = 'biblio';
 130     }
 131
 132     if ($#_ > -1) {
 133         $self->{'threshold'} = shift;
 134     } else {
 135         $self->{'threshold'} = 1000;
 136     }
 137
 138     $self->{'code'} = '';
 139     $self->{'description'} = '';
 140
 141     $self->{'matchpoints'} = [];
 142     $self->{'required_checks'} = [];
 143
 144     bless $self, $class;
 145     return $self;
 146 }
 147
 148 =head2 fetch
 149
 150   my $matcher = C4::Matcher->fetch($id);
 151
 152 Creates a matcher object from the version stored
 153 in the database.  If a matcher with the given
 154 id does not exist, returns undef.
 155
 156 =cut
 157
 158 sub fetch {
 159     my $class = shift;
 160     my $id = shift;
 161     my $dbh = C4::Context->dbh();
 162
 163     my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
 164     $sth->execute($id);
 165     my $row = $sth->fetchrow_hashref;
 166     $sth->finish();
 167     return undef unless defined $row;
 168
 169     my $self = {};
 170     $self->{'id'} = $row->{'matcher_id'};
 171     $self->{'record_type'} = $row->{'record_type'};
 172     $self->{'code'} = $row->{'code'};
 173     $self->{'description'} = $row->{'description'};
 174     $self->{'threshold'} = int($row->{'threshold'});
 175     bless $self, $class;
 176
 177     # matchpoints
 178     $self->{'matchpoints'} = [];
 179     $sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
 180     $sth->execute($self->{'id'});
 181     while (my $row = $sth->fetchrow_hashref) {
 182         my $matchpoint = $self->_fetch_matchpoint($row->{'matchpoint_id'});
 183         push @{ $self->{'matchpoints'} }, $matchpoint;
 184     }
 185
 186     # required checks
 187     $self->{'required_checks'} = [];
 188     $sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
 189     $sth->execute($self->{'id'});
 190     while (my $row = $sth->fetchrow_hashref) {
 191         my $source_matchpoint = $self->_fetch_matchpoint($row->{'source_matchpoint_id'});
 192         my $target_matchpoint = $self->_fetch_matchpoint($row->{'target_matchpoint_id'});
 193         my $matchcheck = {};
 194         $matchcheck->{'source_matchpoint'} = $source_matchpoint;
 195         $matchcheck->{'target_matchpoint'} = $target_matchpoint;
 196         push @{ $self->{'required_checks'} }, $matchcheck;
 197     }
 198
 199     return $self;
 200 }
 201
 202 sub _fetch_matchpoint {
 203     my $self = shift;
 204     my $matchpoint_id = shift;
 205
 206     my $dbh = C4::Context->dbh;
 207     my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
 208     $sth->execute($matchpoint_id);
 209     my $row = $sth->fetchrow_hashref;
 210     my $matchpoint = {};
 211     $matchpoint->{'index'} = $row->{'search_index'};
 212     $matchpoint->{'score'} = int($row->{'score'});
 213     $sth->finish();
 214
 215     $matchpoint->{'components'} = [];
 216     $sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
 217     $sth->execute($matchpoint_id);
 218     while ($row = $sth->fetchrow_hashref) {
 219         my $component = {};
 220         $component->{'tag'} = $row->{'tag'};
 221         $component->{'subfields'} = { map { $_ => 1 } split(//,  $row->{'subfields'}) };
 222         $component->{'offset'} = int($row->{'offset'});
 223         $component->{'length'} = int($row->{'length'});
 224         $component->{'norms'} = [];
 225         my $sth2 = $dbh->prepare_cached("SELECT *
 226                                          FROM matchpoint_component_norms
 227                                          WHERE matchpoint_component_id = ? ORDER BY sequence");
 228         $sth2->execute($row->{'matchpoint_component_id'});
 229         while (my $row2 = $sth2->fetchrow_hashref) {
 230             push @{ $component->{'norms'} }, $row2->{'norm_routine'};
 231         }
 232         push @{ $matchpoint->{'components'} }, $component;
 233     }
 234     return $matchpoint;
 235 }
 236
 237 =head2 store
 238
 239   my $id = $matcher->store();
 240
 241 Stores matcher in database.  The return value is the ID
 242 of the marc_matchers row.  If the matcher was
 243 previously retrieved from the database via the fetch()
 244 method, the DB representation of the matcher
 245 is replaced.
 246
 247 =cut
 248
 249 sub store {
 250     my $self = shift;
 251
 252     if (defined $self->{'id'}) {
 253         # update
 254         $self->_del_matcher_components();
 255         $self->_update_marc_matchers();
 256     } else {
 257         # create new
 258         $self->_new_marc_matchers();
 259     }
 260     $self->_store_matcher_components();
 261     return $self->{'id'};
 262 }
 263
 264 sub _del_matcher_components {
 265     my $self = shift;
 266
 267     my $dbh = C4::Context->dbh();
 268     my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
 269     $sth->execute($self->{'id'});
 270     $sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
 271     $sth->execute($self->{'id'});
 272     # foreign key delete cascades take care of deleting relevant rows
 273     # from matcher_matchpoints, matchpoint_components, and
 274     # matchpoint_component_norms
 275 }
 276
 277 sub _update_marc_matchers {
 278     my $self = shift;
 279
 280     my $dbh = C4::Context->dbh();
 281     my $sth = $dbh->prepare_cached("UPDATE marc_matchers
 282                                     SET code = ?,
 283                                         description = ?,
 284                                         record_type = ?,
 285                                         threshold = ?
 286                                     WHERE matcher_id = ?");
 287     $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'}, $self->{'id'});
 288 }
 289
 290 sub _new_marc_matchers {
 291     my $self = shift;
 292
 293     my $dbh = C4::Context->dbh();
 294     my $sth = $dbh->prepare_cached("INSERT INTO marc_matchers
 295                                     (code, description, record_type, threshold)
 296                                     VALUES (?, ?, ?, ?)");
 297     $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'});
 298     $self->{'id'} = $dbh->{'mysql_insertid'};
 299 }
 300
 301 sub _store_matcher_components {
 302     my $self = shift;
 303
 304     my $dbh = C4::Context->dbh();
 305     my $sth;
 306     my $matcher_id = $self->{'id'};
 307     foreach my $matchpoint (@{ $self->{'matchpoints'}}) {
 308         my $matchpoint_id = $self->_store_matchpoint($matchpoint);
 309         $sth = $dbh->prepare_cached("INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
 310                                      VALUES (?, ?)");
 311         $sth->execute($matcher_id, $matchpoint_id);
 312     }
 313     foreach my $matchcheck (@{ $self->{'required_checks'} }) {
 314         my $source_matchpoint_id = $self->_store_matchpoint($matchcheck->{'source_matchpoint'});
 315         my $target_matchpoint_id = $self->_store_matchpoint($matchcheck->{'target_matchpoint'});
 316         $sth = $dbh->prepare_cached("INSERT INTO matchchecks
 317                                      (matcher_id, source_matchpoint_id, target_matchpoint_id)
 318                                      VALUES (?, ?, ?)");
 319         $sth->execute($matcher_id, $source_matchpoint_id,  $target_matchpoint_id);
 320     }
 321
 322 }
 323
 324 sub _store_matchpoint {
 325     my $self = shift;
 326     my $matchpoint = shift;
 327
 328     my $dbh = C4::Context->dbh();
 329     my $sth;
 330     my $matcher_id = $self->{'id'};
 331     $sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
 332                                  VALUES (?, ?, ?)");
 333     $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'});
 334     my $matchpoint_id = $dbh->{'mysql_insertid'};
 335     my $seqnum = 0;
 336     foreach my $component (@{ $matchpoint->{'components'} }) {
 337         $seqnum++;
 338         $sth = $dbh->prepare_cached("INSERT INTO matchpoint_components
 339                                      (matchpoint_id, sequence, tag, subfields, offset, length)
 340                                      VALUES (?, ?, ?, ?, ?, ?)");
 341         $sth->bind_param(1, $matchpoint_id);
 342         $sth->bind_param(2, $seqnum);
 343         $sth->bind_param(3, $component->{'tag'});
 344         $sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
 345         $sth->bind_param(5, $component->{'offset'});
 346         $sth->bind_param(6, $component->{'length'});
 347         $sth->execute();
 348         my $matchpoint_component_id = $dbh->{'mysql_insertid'};
 349         my $normseq = 0;
 350         foreach my $norm (@{ $component->{'norms'} }) {
 351             $normseq++;
 352             $sth = $dbh->prepare_cached("INSERT INTO matchpoint_component_norms
 353                                          (matchpoint_component_id, sequence, norm_routine)
 354                                          VALUES (?, ?, ?)");
 355             $sth->execute($matchpoint_component_id, $normseq, $norm);
 356         }
 357     }
 358     return $matchpoint_id;
 359 }
 360
 361
 362 =head2 delete
 363
 364   C4::Matcher->delete($id);
 365
 366 Deletes the matcher of the specified ID
 367 from the database.
 368
 369 =cut
 370
 371 sub delete {
 372     my $class = shift;
 373     my $matcher_id = shift;
 374
 375     my $dbh = C4::Context->dbh;
 376     my $sth = $dbh->prepare("DELETE FROM marc_matchers WHERE matcher_id = ?");
 377     $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
 378 }
 379
 380 =head2 record_type
 381
 382   $matcher->record_type('biblio');
 383   my $record_type = $matcher->record_type();
 384
 385 Accessor method.
 386
 387 =cut
 388
 389 sub record_type {
 390     my $self = shift;
 391     @_ ? $self->{'record_type'} = shift : $self->{'record_type'};
 392 }
 393
 394 =head2 threshold
 395
 396   $matcher->threshold(1000);
 397   my $threshold = $matcher->threshold();
 398
 399 Accessor method.
 400
 401 =cut
 402
 403 sub threshold {
 404     my $self = shift;
 405     @_ ? $self->{'threshold'} = shift : $self->{'threshold'};
 406 }
 407
 408 =head2 _id
 409
 410   $matcher->_id(123);
 411   my $id = $matcher->_id();
 412
 413 Accessor method.  Note that using this method
 414 to set the DB ID of the matcher should not be
 415 done outside of the editing CGI.
 416
 417 =cut
 418
 419 sub _id {
 420     my $self = shift;
 421     @_ ? $self->{'id'} = shift : $self->{'id'};
 422 }
 423
 424 =head2 code
 425
 426   $matcher->code('ISBN');
 427   my $code = $matcher->code();
 428
 429 Accessor method.
 430
 431 =cut
 432
 433 sub code {
 434     my $self = shift;
 435     @_ ? $self->{'code'} = shift : $self->{'code'};
 436 }
 437
 438 =head2 description
 439
 440   $matcher->description('match on ISBN');
 441   my $description = $matcher->description();
 442
 443 Accessor method.
 444
 445 =cut
 446
 447 sub description {
 448     my $self = shift;
 449     @_ ? $self->{'description'} = shift : $self->{'description'};
 450 }
 451
 452 =head2 add_matchpoint
 453
 454   $matcher->add_matchpoint($index, $score, $matchcomponents);
 455
 456 Adds a matchpoint that may include multiple components.  The $index
 457 parameter identifies the index that will be searched, while $score
 458 is the weight that will be added if a match is found.
 459
 460 $matchcomponents should be a reference to an array of matchpoint
 461 compoents, each of which should be a hash containing the following
 462 keys:
 463     tag
 464     subfields
 465     offset
 466     length
 467     norms
 468
 469 The normalization_rules value should in turn be a reference to an
 470 array, each element of which should be a reference to a
 471 normalization subroutine (under C4::Normalize) to be applied
 472 to the source string.
 473
 474 =cut
 475
 476 sub add_matchpoint {
 477     my $self = shift;
 478     my ($index, $score, $matchcomponents) = @_;
 479
 480     my $matchpoint = {};
 481     $matchpoint->{'index'} = $index;
 482     $matchpoint->{'score'} = $score;
 483     $matchpoint->{'components'} = [];
 484     foreach my $input_component (@{ $matchcomponents }) {
 485         push @{ $matchpoint->{'components'} }, _parse_match_component($input_component);
 486     }
 487     push @{ $self->{'matchpoints'} }, $matchpoint;
 488 }
 489
 490 =head2 add_simple_matchpoint
 491
 492   $matcher->add_simple_matchpoint($index, $score, $source_tag,
 493                             $source_subfields, $source_offset,
 494                             $source_length, $source_normalizer);
 495
 496
 497 Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
 498 normalized per the normalization fuction, search the index.  All records retrieved
 499 will receive the assigned score.
 500
 501 =cut
 502
 503 sub add_simple_matchpoint {
 504     my $self = shift;
 505     my ($index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer) = @_;
 506
 507     $self->add_matchpoint($index, $score, [
 508                           { tag => $source_tag, subfields => $source_subfields,
 509                             offset => $source_offset, 'length' => $source_length,
 510                             norms => [ $source_normalizer ]
 511                           }
 512                          ]);
 513 }
 514
 515 =head2 add_required_check
 516
 517   $match->add_required_check($source_matchpoint, $target_matchpoint);
 518
 519 Adds a required check definition.  A required check means that in
 520 order for a match to be considered valid, the key derived from the
 521 source (incoming) record must match the key derived from the target
 522 (already in DB) record.
 523
 524 Unlike a regular matchpoint, only the first repeat of each tag
 525 in the source and target match criteria are considered.
 526
 527 A typical example of a required check would be verifying that the
 528 titles and publication dates match.
 529
 530 $source_matchpoint and $target_matchpoint are each a reference to
 531 an array of hashes, where each hash follows the same definition
 532 as the matchpoint component specification in add_matchpoint, i.e.,
 533
 534     tag
 535     subfields
 536     offset
 537     length
 538     norms
 539
 540 The normalization_rules value should in turn be a reference to an
 541 array, each element of which should be a reference to a
 542 normalization subroutine (under C4::Normalize) to be applied
 543 to the source string.
 544
 545 =cut
 546
 547 sub add_required_check {
 548     my $self = shift;
 549     my ($source_matchpoint, $target_matchpoint) = @_;
 550
 551     my $matchcheck = {};
 552     $matchcheck->{'source_matchpoint'}->{'index'} = '';
 553     $matchcheck->{'source_matchpoint'}->{'score'} = 0;
 554     $matchcheck->{'source_matchpoint'}->{'components'} = [];
 555     $matchcheck->{'target_matchpoint'}->{'index'} = '';
 556     $matchcheck->{'target_matchpoint'}->{'score'} = 0;
 557     $matchcheck->{'target_matchpoint'}->{'components'} = [];
 558     foreach my $input_component (@{ $source_matchpoint }) {
 559         push @{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component($input_component);
 560     }
 561     foreach my $input_component (@{ $target_matchpoint }) {
 562         push @{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component($input_component);
 563     }
 564     push @{ $self->{'required_checks'} }, $matchcheck;
 565 }
 566
 567 =head2 add_simple_required_check
 568
 569   $matcher->add_simple_required_check($source_tag, $source_subfields,
 570                 $source_offset, $source_length, $source_normalizer,
 571                 $target_tag, $target_subfields, $target_offset,
 572                 $target_length, $target_normalizer);
 573
 574 Adds a required check, which requires that the normalized keys made from the source and targets
 575 must match for a match to be considered valid.
 576
 577 =cut
 578
 579 sub add_simple_required_check {
 580     my $self = shift;
 581     my ($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
 582         $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer) = @_;
 583
 584     $self->add_required_check(
 585       [ { tag => $source_tag, subfields => $source_subfields, offset => $source_offset, 'length' => $source_length,
 586           norms => [ $source_normalizer ] } ],
 587       [ { tag => $target_tag, subfields => $target_subfields, offset => $target_offset, 'length' => $target_length,
 588           norms => [ $target_normalizer ] } ]
 589     );
 590 }
 591
 592 =head2 get_matches
 593
 594   my @matches = $matcher->get_matches($marc_record, $max_matches);
 595   foreach $match (@matches) {
 596       # matches already sorted in order of
 597       # decreasing score
 598       print "record ID: $match->{'record_id'};
 599       print "score:     $match->{'score'};
 600   }
 601
 602 Identifies all of the records matching the given MARC record.  For a record already
 603 in the database to be considered a match, it must meet the following criteria:
 604
 605 =over 2
 606
 607 =item 1. Total score from its matching field must exceed the supplied threshold.
 608
 609 =item 2. It must pass all required checks.
 610
 611 =back
 612
 613 Only the top $max_matches matches are returned.  The returned array is sorted
 614 in order of decreasing score, i.e., the best match is first.
 615
 616 =cut
 617
 618 sub get_matches {
 619     my $self = shift;
 620     my ($source_record, $max_matches) = @_;
 621
 622     my %matches = ();
 623
 624     my $QParser;
 625     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 626     foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
 627         my @source_keys = _get_match_keys( $source_record, $matchpoint );
 628
 629         next if scalar(@source_keys) == 0;
 630
 631         # FIXME - because of a bug in QueryParser, an expression ofthe
 632         # format 'isbn:"isbn1" || isbn:"isbn2" || isbn"isbn3"...'
 633         # does not get parsed correctly, so we will not
 634         # do AggressiveMatchOnISBN if UseQueryParser is on
 635         @source_keys = C4::Koha::GetVariationsOfISBNs(@source_keys)
 636           if ( $matchpoint->{index} =~ /^isbn$/i
 637             && C4::Context->preference('AggressiveMatchOnISBN') )
 638             && !C4::Context->preference('UseQueryParser');
 639
 640         @source_keys = C4::Koha::GetVariationsOfISSNs(@source_keys)
 641           if ( $matchpoint->{index} =~ /^issn$/i
 642             && C4::Context->preference('AggressiveMatchOnISSN') )
 643             && !C4::Context->preference('UseQueryParser');
 644
 645         # build query
 646         my $query;
 647         my $error;
 648         my $searchresults;
 649         my $total_hits;
 650         if ( $self->{'record_type'} eq 'biblio' ) {
 651
 652             #NOTE: The QueryParser can't handle the CCL syntax of 'qualifier','qualifier', so fallback to non-QueryParser.
 653             #NOTE: You can see this in C4::Search::SimpleSearch() as well in a different way.
 654             if ($QParser && $matchpoint->{'index'} !~ m/\w,\w/) {
 655                 $query = join( " || ",
 656                     map { "$matchpoint->{'index'}:$_" } @source_keys );
 657             }
 658             else {
 659                 my $phr = ( C4::Context->preference('AggressiveMatchOnISBN') || C4::Context->preference('AggressiveMatchOnISSN') )  ? ',phr' : q{};
 660                 $query = join( " or ",
 661                     map { "$matchpoint->{'index'}$phr=\"$_\"" } @source_keys );
 662                     #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
 663             }
 664
 665             my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
 666             ( $error, $searchresults, $total_hits ) =
 667               $searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
 668         }
 669         elsif ( $self->{'record_type'} eq 'authority' ) {
 670             my $authresults;
 671             my @marclist;
 672             my @and_or;
 673             my @excluding = [];
 674             my @operator;
 675             my @value;
 676             foreach my $key (@source_keys) {
 677                 push @marclist, $matchpoint->{'index'};
 678                 push @and_or,   'or';
 679                 push @operator, 'exact';
 680                 push @value,    $key;
 681             }
 682             require C4::AuthoritiesMarc;
 683             ( $authresults, $total_hits ) =
 684               C4::AuthoritiesMarc::SearchAuthorities(
 685                 \@marclist,  \@and_or, \@excluding, \@operator,
 686                 \@value,     0,        20,          undef,
 687                 'AuthidAsc', 1
 688               );
 689             foreach my $result (@$authresults) {
 690                 push @$searchresults, $result->{'authid'};
 691             }
 692         }
 693
 694         if ( defined $error ) {
 695             warn "search failed ($query) $error";
 696         }
 697         else {
 698             foreach my $matched ( @{$searchresults} ) {
 699                 $matches{$matched} += $matchpoint->{'score'};
 700             }
 701         }
 702     }
 703
 704     # get rid of any that don't meet the threshold
 705     %matches = map { ($matches{$_} >= $self->{'threshold'}) ? ($_ => $matches{$_}) : () } keys %matches;
 706
 707     # get rid of any that don't meet the required checks
 708     %matches = map { _passes_required_checks($source_record, $_, $self->{'required_checks'}) ?  ($_ => $matches{$_}) : () }
 709                 keys %matches unless ($self->{'record_type'} eq 'auth');
 710
 711     my @results = ();
 712     if ($self->{'record_type'} eq 'biblio') {
 713         require C4::Biblio;
 714         foreach my $marcblob (keys %matches) {
 715             my $target_record = C4::Search::new_record_from_zebra('biblioserver',$marcblob);
 716             my $record_number;
 717             my $result = C4::Biblio::TransformMarcToKoha($target_record, '');
 718             $record_number = $result->{'biblionumber'};
 719             push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
 720         }
 721     } elsif ($self->{'record_type'} eq 'authority') {
 722         require C4::AuthoritiesMarc;
 723         foreach my $authid (keys %matches) {
 724             push @results, { 'record_id' => $authid, 'score' => $matches{$authid} };
 725         }
 726     }
 727     @results = sort {
 728         $b->{'score'} cmp $a->{'score'} or
 729         $b->{'record_id'} cmp $a->{'record_id'}
 730     } @results;
 731     if (scalar(@results) > $max_matches) {
 732         @results = @results[0..$max_matches-1];
 733     }
 734     return @results;
 735
 736 }
 737
 738 =head2 dump
 739
 740   $description = $matcher->dump();
 741
 742 Returns a reference to a structure containing all of the information
 743 in the matcher object.  This is mainly a convenience method to
 744 aid setting up a HTML editing form.
 745
 746 =cut
 747
 748 sub dump {
 749     my $self = shift;
 750
 751     my $result = {};
 752
 753     $result->{'matcher_id'} = $self->{'id'};
 754     $result->{'code'} = $self->{'code'};
 755     $result->{'description'} = $self->{'description'};
 756     $result->{'record_type'} = $self->{'record_type'};
 757
 758     $result->{'matchpoints'} = [];
 759     foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
 760         push @{  $result->{'matchpoints'} }, $matchpoint;
 761     }
 762     $result->{'matchchecks'} = [];
 763     foreach my $matchcheck (@{ $self->{'required_checks'} }) {
 764         push @{  $result->{'matchchecks'} }, $matchcheck;
 765     }
 766
 767     return $result;
 768 }
 769
 770 sub _passes_required_checks {
 771     my ($source_record, $target_blob, $matchchecks) = @_;
 772     my $target_record = MARC::Record->new_from_usmarc($target_blob); # FIXME -- need to avoid parsing record twice
 773
 774     # no checks supplied == automatic pass
 775     return 1 if $#{ $matchchecks } == -1;
 776
 777     foreach my $matchcheck (@{ $matchchecks }) {
 778         my $source_key = join "", _get_match_keys($source_record, $matchcheck->{'source_matchpoint'});
 779         my $target_key = join "", _get_match_keys($target_record, $matchcheck->{'target_matchpoint'});
 780         return 0 unless $source_key eq $target_key;
 781     }
 782     return 1;
 783 }
 784
 785 sub _get_match_keys {
 786
 787     my $source_record = shift;
 788     my $matchpoint = shift;
 789     my $check_only_first_repeat = @_ ? shift : 0;
 790
 791     # If there is more than one component to the matchpoint (e.g.,
 792     # matchpoint includes both 003 and 001), any repeats
 793     # of the first component's tag are identified; repeats
 794     # of the subsequent components' tags are appended to
 795     # each parallel key dervied from the first component,
 796     # up to the number of repeats of the first component's tag.
 797     #
 798     # For example, if the record has one 003 and two 001s, only
 799     # one key is retrieved because there is only one 003.  The key
 800     # will consist of the contents of the first 003 and first 001.
 801     #
 802     # If there are two 003s and two 001s, there will be two keys:
 803     #    first 003 + first 001
 804     #    second 003 + second 001
 805
 806     my @keys = ();
 807     for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
 808         my $component = $matchpoint->{'components'}->[$i];
 809         my $j = -1;
 810         FIELD: foreach my $field ($source_record->field($component->{'tag'})) {
 811             $j++;
 812             last FIELD if $j > 0 and $check_only_first_repeat;
 813             last FIELD if $i > 0 and $j > $#keys;
 814
 815             my $string;
 816             if ( $field->is_control_field() ) {
 817                 $string = $field->data();
 818             } else {
 819                 $string = $field->as_string(
 820                     join('', keys %{ $component->{ subfields } }), ' ' # ' ' as separator
 821                 );
 822             }
 823
 824             if ($component->{'length'}>0) {
 825                 $string= substr($string, $component->{'offset'}, $component->{'length'});
 826             } elsif ($component->{'offset'}) {
 827                 $string= substr($string, $component->{'offset'});
 828             }
 829
 830             my $norms = $component->{'norms'};
 831             my $key = $string;
 832
 833             foreach my $norm ( @{ $norms } ) {
 834                 if ( grep { $norm eq $_ } valid_normalization_routines() ) {
 835                     if ( $norm eq 'remove_spaces' ) {
 836                         $key = remove_spaces($key);
 837                     }
 838                     elsif ( $norm eq 'upper_case' ) {
 839                         $key = upper_case($key);
 840                     }
 841                     elsif ( $norm eq 'lower_case' ) {
 842                         $key = lower_case($key);
 843                     }
 844                     elsif ( $norm eq 'legacy_default' ) {
 845                         $key = legacy_default($key);
 846                     }
 847                 } else {
 848                     warn "Invalid normalization routine required ($norm)"
 849                         unless $norm eq 'none';
 850                 }
 851             }
 852
 853             if ($i == 0) {
 854                 push @keys, $key if $key;
 855             } else {
 856                 $keys[$j] .= " $key" if $key;
 857             }
 858         }
 859     }
 860     return @keys;
 861 }
 862
 863
 864 sub _parse_match_component {
 865     my $input_component = shift;
 866
 867     my $component = {};
 868     $component->{'tag'} = $input_component->{'tag'};
 869     $component->{'subfields'} = { map { $_ => 1 } split(//, $input_component->{'subfields'}) };
 870     $component->{'offset'} = exists($input_component->{'offset'}) ? $input_component->{'offset'} : -1;
 871     $component->{'length'} = $input_component->{'length'} ? $input_component->{'length'} : 0;
 872     $component->{'norms'} =  $input_component->{'norms'} ? $input_component->{'norms'} : [];
 873
 874     return $component;
 875 }
 876
 877 sub valid_normalization_routines {
 878
 879     return (
 880         'remove_spaces',
 881         'upper_case',
 882         'lower_case',
 883         'legacy_default'
 884     );
 885 }
 886
 887 1;
 888 __END__
 889
 890 =head1 AUTHOR
 891
 892 Koha Development Team <http://koha-community.org/>
 893
 894 Galen Charlton <galen.charlton@liblime.com>
 895
 896 =cut