C4/Matcher.pm

   1 package C4::Matcher;
   2
   3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
   4 #
   5 # This file is part of Koha.
   6 #
   7 # Koha is free software; you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # Koha is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
  19
  20 use strict;
  21 use warnings;
  22
  23 use C4::Context;
  24 use MARC::Record;
  25
  26 use vars qw();
  27
  28 BEGIN {
  29         # set the version for version checking
  30 }
  31
  32 =head1 NAME
  33
  34 C4::Matcher - find MARC records matching another one
  35
  36 =head1 SYNOPSIS
  37
  38   my @matchers = C4::Matcher::GetMatcherList();
  39
  40   my $matcher = C4::Matcher->new($record_type);
  41   $matcher->threshold($threshold);
  42   $matcher->code($code);
  43   $matcher->description($description);
  44
  45   $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
  46   $matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
  47   $matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
  48
  49   $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
  50   $matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
  51                                [ { tag => '245', subfields => 'a', norms => [] } ]);
  52
  53   my @matches = $matcher->get_matches($marc_record, $max_matches);
  54
  55   foreach $match (@matches) {
  56
  57       # matches already sorted in order of
  58       # decreasing score
  59       print "record ID: $match->{'record_id'};
  60       print "score:     $match->{'score'};
  61
  62   }
  63
  64   my $matcher_description = $matcher->dump();
  65
  66 =head1 FUNCTIONS
  67
  68 =cut
  69
  70 =head2 GetMatcherList
  71
  72   my @matchers = C4::Matcher::GetMatcherList();
  73
  74 Returns an array of hashrefs list all matchers
  75 present in the database.  Each hashref includes:
  76
  77  * matcher_id
  78  * code
  79  * description
  80
  81 =cut
  82
  83 sub GetMatcherList {
  84     my $dbh = C4::Context->dbh;
  85
  86     my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
  87     $sth->execute();
  88     my @results = ();
  89     while (my $row = $sth->fetchrow_hashref) {
  90         push @results, $row;
  91     }
  92     return @results;
  93 }
  94
  95 =head2 GetMatcherId
  96
  97   my $matcher_id = C4::Matcher::GetMatcherId($code);
  98
  99 Returns the matcher_id of a code.
 100
 101 =cut
 102
 103 sub GetMatcherId {
 104     my ($code) = @_;
 105     my $dbh = C4::Context->dbh;
 106
 107     my $matcher_id = $dbh->selectrow_array("SELECT matcher_id FROM marc_matchers WHERE code = ?", undef, $code);
 108     return $matcher_id;
 109 }
 110
 111 =head1 METHODS
 112
 113 =head2 new
 114
 115   my $matcher = C4::Matcher->new($record_type, $threshold);
 116
 117 Creates a new Matcher.  C<$record_type> indicates which search
 118 database to use, e.g., 'biblio' or 'authority' and defaults to
 119 'biblio', while C<$threshold> is the minimum score required for a match
 120 and defaults to 1000.
 121
 122 =cut
 123
 124 sub new {
 125     my $class = shift;
 126     my $self = {};
 127
 128     $self->{'id'} = undef;
 129
 130     if ($#_ > -1) {
 131         $self->{'record_type'} = shift;
 132     } else {
 133         $self->{'record_type'} = 'biblio';
 134     }
 135
 136     if ($#_ > -1) {
 137         $self->{'threshold'} = shift;
 138     } else {
 139         $self->{'threshold'} = 1000;
 140     }
 141
 142     $self->{'code'} = '';
 143     $self->{'description'} = '';
 144
 145     $self->{'matchpoints'} = [];
 146     $self->{'required_checks'} = [];
 147
 148     bless $self, $class;
 149     return $self;
 150 }
 151
 152 =head2 fetch
 153
 154   my $matcher = C4::Matcher->fetch($id);
 155
 156 Creates a matcher object from the version stored
 157 in the database.  If a matcher with the given
 158 id does not exist, returns undef.
 159
 160 =cut
 161
 162 sub fetch {
 163     my $class = shift;
 164     my $id = shift;
 165     my $dbh = C4::Context->dbh();
 166
 167     my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
 168     $sth->execute($id);
 169     my $row = $sth->fetchrow_hashref;
 170     $sth->finish();
 171     return undef unless defined $row;
 172
 173     my $self = {};
 174     $self->{'id'} = $row->{'matcher_id'};
 175     $self->{'record_type'} = $row->{'record_type'};
 176     $self->{'code'} = $row->{'code'};
 177     $self->{'description'} = $row->{'description'};
 178     $self->{'threshold'} = int($row->{'threshold'});
 179     bless $self, $class;
 180
 181     # matchpoints
 182     $self->{'matchpoints'} = [];
 183     $sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
 184     $sth->execute($self->{'id'});
 185     while (my $row = $sth->fetchrow_hashref) {
 186         my $matchpoint = $self->_fetch_matchpoint($row->{'matchpoint_id'});
 187         push @{ $self->{'matchpoints'} }, $matchpoint;
 188     }
 189
 190     # required checks
 191     $self->{'required_checks'} = [];
 192     $sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
 193     $sth->execute($self->{'id'});
 194     while (my $row = $sth->fetchrow_hashref) {
 195         my $source_matchpoint = $self->_fetch_matchpoint($row->{'source_matchpoint_id'});
 196         my $target_matchpoint = $self->_fetch_matchpoint($row->{'target_matchpoint_id'});
 197         my $matchcheck = {};
 198         $matchcheck->{'source_matchpoint'} = $source_matchpoint;
 199         $matchcheck->{'target_matchpoint'} = $target_matchpoint;
 200         push @{ $self->{'required_checks'} }, $matchcheck;
 201     }
 202
 203     return $self;
 204 }
 205
 206 sub _fetch_matchpoint {
 207     my $self = shift;
 208     my $matchpoint_id = shift;
 209
 210     my $dbh = C4::Context->dbh;
 211     my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
 212     $sth->execute($matchpoint_id);
 213     my $row = $sth->fetchrow_hashref;
 214     my $matchpoint = {};
 215     $matchpoint->{'index'} = $row->{'search_index'};
 216     $matchpoint->{'score'} = int($row->{'score'});
 217     $sth->finish();
 218
 219     $matchpoint->{'components'} = [];
 220     $sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
 221     $sth->execute($matchpoint_id);
 222     while ($row = $sth->fetchrow_hashref) {
 223         my $component = {};
 224         $component->{'tag'} = $row->{'tag'};
 225         $component->{'subfields'} = { map { $_ => 1 } split(//,  $row->{'subfields'}) };
 226         $component->{'offset'} = int($row->{'offset'});
 227         $component->{'length'} = int($row->{'length'});
 228         $component->{'norms'} = [];
 229         my $sth2 = $dbh->prepare_cached("SELECT *
 230                                          FROM matchpoint_component_norms
 231                                          WHERE matchpoint_component_id = ? ORDER BY sequence");
 232         $sth2->execute($row->{'matchpoint_component_id'});
 233         while (my $row2 = $sth2->fetchrow_hashref) {
 234             push @{ $component->{'norms'} }, $row2->{'norm_routine'};
 235         }
 236         push @{ $matchpoint->{'components'} }, $component;
 237     }
 238     return $matchpoint;
 239 }
 240
 241 =head2 store
 242
 243   my $id = $matcher->store();
 244
 245 Stores matcher in database.  The return value is the ID
 246 of the marc_matchers row.  If the matcher was
 247 previously retrieved from the database via the fetch()
 248 method, the DB representation of the matcher
 249 is replaced.
 250
 251 =cut
 252
 253 sub store {
 254     my $self = shift;
 255
 256     if (defined $self->{'id'}) {
 257         # update
 258         $self->_del_matcher_components();
 259         $self->_update_marc_matchers();
 260     } else {
 261         # create new
 262         $self->_new_marc_matchers();
 263     }
 264     $self->_store_matcher_components();
 265     return $self->{'id'};
 266 }
 267
 268 sub _del_matcher_components {
 269     my $self = shift;
 270
 271     my $dbh = C4::Context->dbh();
 272     my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
 273     $sth->execute($self->{'id'});
 274     $sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
 275     $sth->execute($self->{'id'});
 276     # foreign key delete cascades take care of deleting relevant rows
 277     # from matcher_matchpoints, matchpoint_components, and
 278     # matchpoint_component_norms
 279 }
 280
 281 sub _update_marc_matchers {
 282     my $self = shift;
 283
 284     my $dbh = C4::Context->dbh();
 285     my $sth = $dbh->prepare_cached("UPDATE marc_matchers
 286                                     SET code = ?,
 287                                         description = ?,
 288                                         record_type = ?,
 289                                         threshold = ?
 290                                     WHERE matcher_id = ?");
 291     $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'}, $self->{'id'});
 292 }
 293
 294 sub _new_marc_matchers {
 295     my $self = shift;
 296
 297     my $dbh = C4::Context->dbh();
 298     my $sth = $dbh->prepare_cached("INSERT INTO marc_matchers
 299                                     (code, description, record_type, threshold)
 300                                     VALUES (?, ?, ?, ?)");
 301     $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'});
 302     $self->{'id'} = $dbh->{'mysql_insertid'};
 303 }
 304
 305 sub _store_matcher_components {
 306     my $self = shift;
 307
 308     my $dbh = C4::Context->dbh();
 309     my $sth;
 310     my $matcher_id = $self->{'id'};
 311     foreach my $matchpoint (@{ $self->{'matchpoints'}}) {
 312         my $matchpoint_id = $self->_store_matchpoint($matchpoint);
 313         $sth = $dbh->prepare_cached("INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
 314                                      VALUES (?, ?)");
 315         $sth->execute($matcher_id, $matchpoint_id);
 316     }
 317     foreach my $matchcheck (@{ $self->{'required_checks'} }) {
 318         my $source_matchpoint_id = $self->_store_matchpoint($matchcheck->{'source_matchpoint'});
 319         my $target_matchpoint_id = $self->_store_matchpoint($matchcheck->{'target_matchpoint'});
 320         $sth = $dbh->prepare_cached("INSERT INTO matchchecks
 321                                      (matcher_id, source_matchpoint_id, target_matchpoint_id)
 322                                      VALUES (?, ?, ?)");
 323         $sth->execute($matcher_id, $source_matchpoint_id,  $target_matchpoint_id);
 324     }
 325
 326 }
 327
 328 sub _store_matchpoint {
 329     my $self = shift;
 330     my $matchpoint = shift;
 331
 332     my $dbh = C4::Context->dbh();
 333     my $sth;
 334     my $matcher_id = $self->{'id'};
 335     $sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
 336                                  VALUES (?, ?, ?)");
 337     $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'});
 338     my $matchpoint_id = $dbh->{'mysql_insertid'};
 339     my $seqnum = 0;
 340     foreach my $component (@{ $matchpoint->{'components'} }) {
 341         $seqnum++;
 342         $sth = $dbh->prepare_cached("INSERT INTO matchpoint_components
 343                                      (matchpoint_id, sequence, tag, subfields, offset, length)
 344                                      VALUES (?, ?, ?, ?, ?, ?)");
 345         $sth->bind_param(1, $matchpoint_id);
 346         $sth->bind_param(2, $seqnum);
 347         $sth->bind_param(3, $component->{'tag'});
 348         $sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
 349         $sth->bind_param(5, $component->{'offset'});
 350         $sth->bind_param(6, $component->{'length'});
 351         $sth->execute();
 352         my $matchpoint_component_id = $dbh->{'mysql_insertid'};
 353         my $normseq = 0;
 354         foreach my $norm (@{ $component->{'norms'} }) {
 355             $normseq++;
 356             $sth = $dbh->prepare_cached("INSERT INTO matchpoint_component_norms
 357                                          (matchpoint_component_id, sequence, norm_routine)
 358                                          VALUES (?, ?, ?)");
 359             $sth->execute($matchpoint_component_id, $normseq, $norm);
 360         }
 361     }
 362     return $matchpoint_id;
 363 }
 364
 365
 366 =head2 delete
 367
 368   C4::Matcher->delete($id);
 369
 370 Deletes the matcher of the specified ID
 371 from the database.
 372
 373 =cut
 374
 375 sub delete {
 376     my $class = shift;
 377     my $matcher_id = shift;
 378
 379     my $dbh = C4::Context->dbh;
 380     my $sth = $dbh->prepare("DELETE FROM marc_matchers WHERE matcher_id = ?");
 381     $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
 382 }
 383
 384 =head2 record_type
 385
 386   $matcher->record_type('biblio');
 387   my $record_type = $matcher->record_type();
 388
 389 Accessor method.
 390
 391 =cut
 392
 393 sub record_type {
 394     my $self = shift;
 395     @_ ? $self->{'record_type'} = shift : $self->{'record_type'};
 396 }
 397
 398 =head2 threshold
 399
 400   $matcher->threshold(1000);
 401   my $threshold = $matcher->threshold();
 402
 403 Accessor method.
 404
 405 =cut
 406
 407 sub threshold {
 408     my $self = shift;
 409     @_ ? $self->{'threshold'} = shift : $self->{'threshold'};
 410 }
 411
 412 =head2 _id
 413
 414   $matcher->_id(123);
 415   my $id = $matcher->_id();
 416
 417 Accessor method.  Note that using this method
 418 to set the DB ID of the matcher should not be
 419 done outside of the editing CGI.
 420
 421 =cut
 422
 423 sub _id {
 424     my $self = shift;
 425     @_ ? $self->{'id'} = shift : $self->{'id'};
 426 }
 427
 428 =head2 code
 429
 430   $matcher->code('ISBN');
 431   my $code = $matcher->code();
 432
 433 Accessor method.
 434
 435 =cut
 436
 437 sub code {
 438     my $self = shift;
 439     @_ ? $self->{'code'} = shift : $self->{'code'};
 440 }
 441
 442 =head2 description
 443
 444   $matcher->description('match on ISBN');
 445   my $description = $matcher->description();
 446
 447 Accessor method.
 448
 449 =cut
 450
 451 sub description {
 452     my $self = shift;
 453     @_ ? $self->{'description'} = shift : $self->{'description'};
 454 }
 455
 456 =head2 add_matchpoint
 457
 458   $matcher->add_matchpoint($index, $score, $matchcomponents);
 459
 460 Adds a matchpoint that may include multiple components.  The $index
 461 parameter identifies the index that will be searched, while $score
 462 is the weight that will be added if a match is found.
 463
 464 $matchcomponents should be a reference to an array of matchpoint
 465 compoents, each of which should be a hash containing the following
 466 keys:
 467     tag
 468     subfields
 469     offset
 470     length
 471     norms
 472
 473 The normalization_rules value should in turn be a reference to an
 474 array, each element of which should be a reference to a
 475 normalization subroutine (under C4::Normalize) to be applied
 476 to the source string.
 477
 478 =cut
 479
 480 sub add_matchpoint {
 481     my $self = shift;
 482     my ($index, $score, $matchcomponents) = @_;
 483
 484     my $matchpoint = {};
 485     $matchpoint->{'index'} = $index;
 486     $matchpoint->{'score'} = $score;
 487     $matchpoint->{'components'} = [];
 488     foreach my $input_component (@{ $matchcomponents }) {
 489         push @{ $matchpoint->{'components'} }, _parse_match_component($input_component);
 490     }
 491     push @{ $self->{'matchpoints'} }, $matchpoint;
 492 }
 493
 494 =head2 add_simple_matchpoint
 495
 496   $matcher->add_simple_matchpoint($index, $score, $source_tag,
 497                             $source_subfields, $source_offset,
 498                             $source_length, $source_normalizer);
 499
 500
 501 Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
 502 normalized per the normalization fuction, search the index.  All records retrieved
 503 will receive the assigned score.
 504
 505 =cut
 506
 507 sub add_simple_matchpoint {
 508     my $self = shift;
 509     my ($index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer) = @_;
 510
 511     $self->add_matchpoint($index, $score, [
 512                           { tag => $source_tag, subfields => $source_subfields,
 513                             offset => $source_offset, 'length' => $source_length,
 514                             norms => [ $source_normalizer ]
 515                           }
 516                          ]);
 517 }
 518
 519 =head2 add_required_check
 520
 521   $match->add_required_check($source_matchpoint, $target_matchpoint);
 522
 523 Adds a required check definition.  A required check means that in
 524 order for a match to be considered valid, the key derived from the
 525 source (incoming) record must match the key derived from the target
 526 (already in DB) record.
 527
 528 Unlike a regular matchpoint, only the first repeat of each tag
 529 in the source and target match criteria are considered.
 530
 531 A typical example of a required check would be verifying that the
 532 titles and publication dates match.
 533
 534 $source_matchpoint and $target_matchpoint are each a reference to
 535 an array of hashes, where each hash follows the same definition
 536 as the matchpoint component specification in add_matchpoint, i.e.,
 537
 538     tag
 539     subfields
 540     offset
 541     length
 542     norms
 543
 544 The normalization_rules value should in turn be a reference to an
 545 array, each element of which should be a reference to a
 546 normalization subroutine (under C4::Normalize) to be applied
 547 to the source string.
 548
 549 =cut
 550
 551 sub add_required_check {
 552     my $self = shift;
 553     my ($source_matchpoint, $target_matchpoint) = @_;
 554
 555     my $matchcheck = {};
 556     $matchcheck->{'source_matchpoint'}->{'index'} = '';
 557     $matchcheck->{'source_matchpoint'}->{'score'} = 0;
 558     $matchcheck->{'source_matchpoint'}->{'components'} = [];
 559     $matchcheck->{'target_matchpoint'}->{'index'} = '';
 560     $matchcheck->{'target_matchpoint'}->{'score'} = 0;
 561     $matchcheck->{'target_matchpoint'}->{'components'} = [];
 562     foreach my $input_component (@{ $source_matchpoint }) {
 563         push @{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component($input_component);
 564     }
 565     foreach my $input_component (@{ $target_matchpoint }) {
 566         push @{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component($input_component);
 567     }
 568     push @{ $self->{'required_checks'} }, $matchcheck;
 569 }
 570
 571 =head2 add_simple_required_check
 572
 573   $matcher->add_simple_required_check($source_tag, $source_subfields,
 574                 $source_offset, $source_length, $source_normalizer,
 575                 $target_tag, $target_subfields, $target_offset,
 576                 $target_length, $target_normalizer);
 577
 578 Adds a required check, which requires that the normalized keys made from the source and targets
 579 must match for a match to be considered valid.
 580
 581 =cut
 582
 583 sub add_simple_required_check {
 584     my $self = shift;
 585     my ($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
 586         $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer) = @_;
 587
 588     $self->add_required_check(
 589       [ { tag => $source_tag, subfields => $source_subfields, offset => $source_offset, 'length' => $source_length,
 590           norms => [ $source_normalizer ] } ],
 591       [ { tag => $target_tag, subfields => $target_subfields, offset => $target_offset, 'length' => $target_length,
 592           norms => [ $target_normalizer ] } ]
 593     );
 594 }
 595
 596 =head2 get_matches
 597
 598   my @matches = $matcher->get_matches($marc_record, $max_matches);
 599   foreach $match (@matches) {
 600       # matches already sorted in order of
 601       # decreasing score
 602       print "record ID: $match->{'record_id'};
 603       print "score:     $match->{'score'};
 604   }
 605
 606 Identifies all of the records matching the given MARC record.  For a record already
 607 in the database to be considered a match, it must meet the following criteria:
 608
 609 =over 2
 610
 611 =item 1. Total score from its matching field must exceed the supplied threshold.
 612
 613 =item 2. It must pass all required checks.
 614
 615 =back
 616
 617 Only the top $max_matches matches are returned.  The returned array is sorted
 618 in order of decreasing score, i.e., the best match is first.
 619
 620 =cut
 621
 622 sub get_matches {
 623     my $self = shift;
 624     my ($source_record, $max_matches) = @_;
 625
 626     my %matches = ();
 627
 628     my $QParser;
 629     $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
 630     foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) {
 631         my @source_keys = _get_match_keys( $source_record, $matchpoint );
 632
 633         next if scalar(@source_keys) == 0;
 634
 635         # FIXME - because of a bug in QueryParser, an expression ofthe
 636         # format 'isbn:"isbn1" || isbn:"isbn2" || isbn"isbn3"...'
 637         # does not get parsed correctly, so we will not
 638         # do AggressiveMatchOnISBN if UseQueryParser is on
 639         @source_keys = C4::Koha::GetVariationsOfISBNs(@source_keys)
 640           if ( $matchpoint->{index} =~ /^isbn$/i
 641             && C4::Context->preference('AggressiveMatchOnISBN') )
 642             && !C4::Context->preference('UseQueryParser');
 643
 644         # build query
 645         my $query;
 646         my $error;
 647         my $searchresults;
 648         my $total_hits;
 649         if ( $self->{'record_type'} eq 'biblio' ) {
 650
 651             if ($QParser) {
 652                 $query = join( " || ",
 653                     map { "$matchpoint->{'index'}:$_" } @source_keys );
 654             }
 655             else {
 656                 my $phr = C4::Context->preference('AggressiveMatchOnISBN') ? ',phr' : q{};
 657                 $query = join( " or ",
 658                     map { "$matchpoint->{'index'}$phr=$_" } @source_keys );
 659             }
 660
 661             require C4::Search;
 662
 663             ( $error, $searchresults, $total_hits ) =
 664               C4::Search::SimpleSearch( $query, 0, $max_matches );
 665         }
 666         elsif ( $self->{'record_type'} eq 'authority' ) {
 667             my $authresults;
 668             my @marclist;
 669             my @and_or;
 670             my @excluding = [];
 671             my @operator;
 672             my @value;
 673             foreach my $key (@source_keys) {
 674                 push @marclist, $matchpoint->{'index'};
 675                 push @and_or,   'or';
 676                 push @operator, 'exact';
 677                 push @value,    $key;
 678             }
 679             require C4::AuthoritiesMarc;
 680             ( $authresults, $total_hits ) =
 681               C4::AuthoritiesMarc::SearchAuthorities(
 682                 \@marclist,  \@and_or, \@excluding, \@operator,
 683                 \@value,     0,        20,          undef,
 684                 'AuthidAsc', 1
 685               );
 686             foreach my $result (@$authresults) {
 687                 push @$searchresults, $result->{'authid'};
 688             }
 689         }
 690
 691         if ( defined $error ) {
 692             warn "search failed ($query) $error";
 693         }
 694         else {
 695             foreach my $matched ( @{$searchresults} ) {
 696                 $matches{$matched} += $matchpoint->{'score'};
 697             }
 698         }
 699     }
 700
 701     # get rid of any that don't meet the threshold
 702     %matches = map { ($matches{$_} >= $self->{'threshold'}) ? ($_ => $matches{$_}) : () } keys %matches;
 703
 704     # get rid of any that don't meet the required checks
 705     %matches = map { _passes_required_checks($source_record, $_, $self->{'required_checks'}) ?  ($_ => $matches{$_}) : () }
 706                 keys %matches unless ($self->{'record_type'} eq 'auth');
 707
 708     my @results = ();
 709     if ($self->{'record_type'} eq 'biblio') {
 710         require C4::Biblio;
 711         foreach my $marcblob (keys %matches) {
 712             my $target_record = C4::Search::new_record_from_zebra('biblioserver',$marcblob);
 713             my $record_number;
 714             my $result = C4::Biblio::TransformMarcToKoha(C4::Context->dbh, $target_record, '');
 715             $record_number = $result->{'biblionumber'};
 716             push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
 717         }
 718     } elsif ($self->{'record_type'} eq 'authority') {
 719         require C4::AuthoritiesMarc;
 720         foreach my $authid (keys %matches) {
 721             push @results, { 'record_id' => $authid, 'score' => $matches{$authid} };
 722         }
 723     }
 724     @results = sort { $b->{'score'} cmp $a->{'score'} } @results;
 725     if (scalar(@results) > $max_matches) {
 726         @results = @results[0..$max_matches-1];
 727     }
 728     return @results;
 729
 730 }
 731
 732 =head2 dump
 733
 734   $description = $matcher->dump();
 735
 736 Returns a reference to a structure containing all of the information
 737 in the matcher object.  This is mainly a convenience method to
 738 aid setting up a HTML editing form.
 739
 740 =cut
 741
 742 sub dump {
 743     my $self = shift;
 744
 745     my $result = {};
 746
 747     $result->{'matcher_id'} = $self->{'id'};
 748     $result->{'code'} = $self->{'code'};
 749     $result->{'description'} = $self->{'description'};
 750     $result->{'record_type'} = $self->{'record_type'};
 751
 752     $result->{'matchpoints'} = [];
 753     foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
 754         push @{  $result->{'matchpoints'} }, $matchpoint;
 755     }
 756     $result->{'matchchecks'} = [];
 757     foreach my $matchcheck (@{ $self->{'required_checks'} }) {
 758         push @{  $result->{'matchchecks'} }, $matchcheck;
 759     }
 760
 761     return $result;
 762 }
 763
 764 sub _passes_required_checks {
 765     my ($source_record, $target_blob, $matchchecks) = @_;
 766     my $target_record = MARC::Record->new_from_usmarc($target_blob); # FIXME -- need to avoid parsing record twice
 767
 768     # no checks supplied == automatic pass
 769     return 1 if $#{ $matchchecks } == -1;
 770
 771     foreach my $matchcheck (@{ $matchchecks }) {
 772         my $source_key = join "", _get_match_keys($source_record, $matchcheck->{'source_matchpoint'});
 773         my $target_key = join "", _get_match_keys($target_record, $matchcheck->{'target_matchpoint'});
 774         return 0 unless $source_key eq $target_key;
 775     }
 776     return 1;
 777 }
 778
 779 sub _get_match_keys {
 780     my $source_record = shift;
 781     my $matchpoint = shift;
 782     my $check_only_first_repeat = @_ ? shift : 0;
 783
 784     # If there is more than one component to the matchpoint (e.g.,
 785     # matchpoint includes both 003 and 001), any repeats
 786     # of the first component's tag are identified; repeats
 787     # of the subsequent components' tags are appended to
 788     # each parallel key dervied from the first component,
 789     # up to the number of repeats of the first component's tag.
 790     #
 791     # For example, if the record has one 003 and two 001s, only
 792     # one key is retrieved because there is only one 003.  The key
 793     # will consist of the contents of the first 003 and first 001.
 794     #
 795     # If there are two 003s and two 001s, there will be two keys:
 796     #    first 003 + first 001
 797     #    second 003 + second 001
 798
 799     my @keys = ();
 800     for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
 801         my $component = $matchpoint->{'components'}->[$i];
 802         my $j = -1;
 803         FIELD: foreach my $field ($source_record->field($component->{'tag'})) {
 804             $j++;
 805             last FIELD if $j > 0 and $check_only_first_repeat;
 806             last FIELD if $i > 0 and $j > $#keys;
 807             my $key = "";
 808                         my $string;
 809             if ($field->is_control_field()) {
 810                                 $string=$field->data();
 811             } else {
 812                 foreach my $subfield ($field->subfields()) {
 813                     if (exists $component->{'subfields'}->{$subfield->[0]}) {
 814                         $string .= " " . $subfield->[1];
 815                     }
 816                 }
 817                         }
 818             if ($component->{'length'}>0) {
 819                     $string= substr($string, $component->{'offset'}, $component->{'length'});
 820                             # FIXME normalize, substr
 821             } elsif ($component->{'offset'}) {
 822                     $string= substr($string, $component->{'offset'});
 823             }
 824             $key = _normalize($string);
 825             if ($i == 0) {
 826                 push @keys, $key if $key;
 827             } else {
 828                 $keys[$j] .= " $key" if $key;
 829             }
 830         }
 831     }
 832     return @keys;
 833 }
 834
 835
 836 sub _parse_match_component {
 837     my $input_component = shift;
 838
 839     my $component = {};
 840     $component->{'tag'} = $input_component->{'tag'};
 841     $component->{'subfields'} = { map { $_ => 1 } split(//, $input_component->{'subfields'}) };
 842     $component->{'offset'} = exists($input_component->{'offset'}) ? $input_component->{'offset'} : -1;
 843     $component->{'length'} = $input_component->{'length'} ? $input_component->{'length'} : 0;
 844     $component->{'norms'} =  $input_component->{'norms'} ? $input_component->{'norms'} : [];
 845
 846     return $component;
 847 }
 848
 849 # FIXME - default normalizer
 850 sub _normalize {
 851     my $value = uc shift;
 852     $value =~ s/[.;:,\]\[\)\(\/'"]//g;
 853     $value =~ s/^\s+//;
 854     #$value =~ s/^\s+$//;
 855     $value =~ s/\s+$//;
 856     $value =~ s/\s+/ /g;
 857     #$value =~ s/[.;,\]\[\)\(\/"']//g;
 858     return $value;
 859 }
 860
 861 1;
 862 __END__
 863
 864 =head1 AUTHOR
 865
 866 Koha Development Team <http://koha-community.org/>
 867
 868 Galen Charlton <galen.charlton@liblime.com>
 869
 870 =cut