3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # Koha is distributed in the hope that it will be useful, but
13 # WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
30 C4::Matcher - find MARC records matching another one
34 my @matchers = C4::Matcher::GetMatcherList();
36 my $matcher = C4::Matcher->new($record_type);
37 $matcher->threshold($threshold);
38 $matcher->code($code);
39 $matcher->description($description);
41 $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
42 $matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
43 $matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
45 $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
46 $matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
47 [ { tag => '245', subfields => 'a', norms => [] } ]);
49 my @matches = $matcher->get_matches($marc_record, $max_matches);
51 foreach $match (@matches) {
53 # matches already sorted in order of
55 print "record ID: $match->{'record_id'};
56 print "score: $match->{'score'};
60 my $matcher_description = $matcher->dump();
68 my @matchers = C4::Matcher::GetMatcherList();
70 Returns an array of hashrefs list all matchers
71 present in the database. Each hashref includes:
80 my $dbh = C4
::Context
->dbh;
82 my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
85 while (my $row = $sth->fetchrow_hashref) {
93 my $matcher_id = C4::Matcher::GetMatcherId($code);
95 Returns the matcher_id of a code.
101 my $dbh = C4
::Context
->dbh;
103 my $matcher_id = $dbh->selectrow_array("SELECT matcher_id FROM marc_matchers WHERE code = ?", undef, $code);
111 my $matcher = C4::Matcher->new($record_type, $threshold);
113 Creates a new Matcher. C<$record_type> indicates which search
114 database to use, e.g., 'biblio' or 'authority' and defaults to
115 'biblio', while C<$threshold> is the minimum score required for a match
116 and defaults to 1000.
124 $self->{'id'} = undef;
127 $self->{'record_type'} = shift;
129 $self->{'record_type'} = 'biblio';
133 $self->{'threshold'} = shift;
135 $self->{'threshold'} = 1000;
138 $self->{'code'} = '';
139 $self->{'description'} = '';
141 $self->{'matchpoints'} = [];
142 $self->{'required_checks'} = [];
150 my $matcher = C4::Matcher->fetch($id);
152 Creates a matcher object from the version stored
153 in the database. If a matcher with the given
154 id does not exist, returns undef.
161 my $dbh = C4
::Context
->dbh();
163 my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
165 my $row = $sth->fetchrow_hashref;
167 return undef unless defined $row;
170 $self->{'id'} = $row->{'matcher_id'};
171 $self->{'record_type'} = $row->{'record_type'};
172 $self->{'code'} = $row->{'code'};
173 $self->{'description'} = $row->{'description'};
174 $self->{'threshold'} = int($row->{'threshold'});
178 $self->{'matchpoints'} = [];
179 $sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
180 $sth->execute($self->{'id'});
181 while (my $row = $sth->fetchrow_hashref) {
182 my $matchpoint = $self->_fetch_matchpoint($row->{'matchpoint_id'});
183 push @
{ $self->{'matchpoints'} }, $matchpoint;
187 $self->{'required_checks'} = [];
188 $sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
189 $sth->execute($self->{'id'});
190 while (my $row = $sth->fetchrow_hashref) {
191 my $source_matchpoint = $self->_fetch_matchpoint($row->{'source_matchpoint_id'});
192 my $target_matchpoint = $self->_fetch_matchpoint($row->{'target_matchpoint_id'});
194 $matchcheck->{'source_matchpoint'} = $source_matchpoint;
195 $matchcheck->{'target_matchpoint'} = $target_matchpoint;
196 push @
{ $self->{'required_checks'} }, $matchcheck;
202 sub _fetch_matchpoint
{
204 my $matchpoint_id = shift;
206 my $dbh = C4
::Context
->dbh;
207 my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
208 $sth->execute($matchpoint_id);
209 my $row = $sth->fetchrow_hashref;
211 $matchpoint->{'index'} = $row->{'search_index'};
212 $matchpoint->{'score'} = int($row->{'score'});
215 $matchpoint->{'components'} = [];
216 $sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
217 $sth->execute($matchpoint_id);
218 while ($row = $sth->fetchrow_hashref) {
220 $component->{'tag'} = $row->{'tag'};
221 $component->{'subfields'} = { map { $_ => 1 } split(//, $row->{'subfields'}) };
222 $component->{'offset'} = int($row->{'offset'});
223 $component->{'length'} = int($row->{'length'});
224 $component->{'norms'} = [];
225 my $sth2 = $dbh->prepare_cached("SELECT *
226 FROM matchpoint_component_norms
227 WHERE matchpoint_component_id = ? ORDER BY sequence");
228 $sth2->execute($row->{'matchpoint_component_id'});
229 while (my $row2 = $sth2->fetchrow_hashref) {
230 push @
{ $component->{'norms'} }, $row2->{'norm_routine'};
232 push @
{ $matchpoint->{'components'} }, $component;
239 my $id = $matcher->store();
241 Stores matcher in database. The return value is the ID
242 of the marc_matchers row. If the matcher was
243 previously retrieved from the database via the fetch()
244 method, the DB representation of the matcher
252 if (defined $self->{'id'}) {
254 $self->_del_matcher_components();
255 $self->_update_marc_matchers();
258 $self->_new_marc_matchers();
260 $self->_store_matcher_components();
261 return $self->{'id'};
264 sub _del_matcher_components
{
267 my $dbh = C4
::Context
->dbh();
268 my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
269 $sth->execute($self->{'id'});
270 $sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
271 $sth->execute($self->{'id'});
272 # foreign key delete cascades take care of deleting relevant rows
273 # from matcher_matchpoints, matchpoint_components, and
274 # matchpoint_component_norms
277 sub _update_marc_matchers
{
280 my $dbh = C4
::Context
->dbh();
281 my $sth = $dbh->prepare_cached("UPDATE marc_matchers
286 WHERE matcher_id = ?");
287 $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'}, $self->{'id'});
290 sub _new_marc_matchers
{
293 my $dbh = C4
::Context
->dbh();
294 my $sth = $dbh->prepare_cached("INSERT INTO marc_matchers
295 (code, description, record_type, threshold)
296 VALUES (?, ?, ?, ?)");
297 $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'});
298 $self->{'id'} = $dbh->{'mysql_insertid'};
301 sub _store_matcher_components
{
304 my $dbh = C4
::Context
->dbh();
306 my $matcher_id = $self->{'id'};
307 foreach my $matchpoint (@
{ $self->{'matchpoints'}}) {
308 my $matchpoint_id = $self->_store_matchpoint($matchpoint);
309 $sth = $dbh->prepare_cached("INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
311 $sth->execute($matcher_id, $matchpoint_id);
313 foreach my $matchcheck (@
{ $self->{'required_checks'} }) {
314 my $source_matchpoint_id = $self->_store_matchpoint($matchcheck->{'source_matchpoint'});
315 my $target_matchpoint_id = $self->_store_matchpoint($matchcheck->{'target_matchpoint'});
316 $sth = $dbh->prepare_cached("INSERT INTO matchchecks
317 (matcher_id, source_matchpoint_id, target_matchpoint_id)
319 $sth->execute($matcher_id, $source_matchpoint_id, $target_matchpoint_id);
324 sub _store_matchpoint
{
326 my $matchpoint = shift;
328 my $dbh = C4
::Context
->dbh();
330 my $matcher_id = $self->{'id'};
331 $sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
333 $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'});
334 my $matchpoint_id = $dbh->{'mysql_insertid'};
336 foreach my $component (@
{ $matchpoint->{'components'} }) {
338 $sth = $dbh->prepare_cached("INSERT INTO matchpoint_components
339 (matchpoint_id, sequence, tag, subfields, offset, length)
340 VALUES (?, ?, ?, ?, ?, ?)");
341 $sth->bind_param(1, $matchpoint_id);
342 $sth->bind_param(2, $seqnum);
343 $sth->bind_param(3, $component->{'tag'});
344 $sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
345 $sth->bind_param(5, $component->{'offset'});
346 $sth->bind_param(6, $component->{'length'});
348 my $matchpoint_component_id = $dbh->{'mysql_insertid'};
350 foreach my $norm (@
{ $component->{'norms'} }) {
352 $sth = $dbh->prepare_cached("INSERT INTO matchpoint_component_norms
353 (matchpoint_component_id, sequence, norm_routine)
355 $sth->execute($matchpoint_component_id, $normseq, $norm);
358 return $matchpoint_id;
364 C4::Matcher->delete($id);
366 Deletes the matcher of the specified ID
373 my $matcher_id = shift;
375 my $dbh = C4
::Context
->dbh;
376 my $sth = $dbh->prepare("DELETE FROM marc_matchers WHERE matcher_id = ?");
377 $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
382 $matcher->record_type('biblio');
383 my $record_type = $matcher->record_type();
391 @_ ?
$self->{'record_type'} = shift : $self->{'record_type'};
396 $matcher->threshold(1000);
397 my $threshold = $matcher->threshold();
405 @_ ?
$self->{'threshold'} = shift : $self->{'threshold'};
411 my $id = $matcher->_id();
413 Accessor method. Note that using this method
414 to set the DB ID of the matcher should not be
415 done outside of the editing CGI.
421 @_ ?
$self->{'id'} = shift : $self->{'id'};
426 $matcher->code('ISBN');
427 my $code = $matcher->code();
435 @_ ?
$self->{'code'} = shift : $self->{'code'};
440 $matcher->description('match on ISBN');
441 my $description = $matcher->description();
449 @_ ?
$self->{'description'} = shift : $self->{'description'};
452 =head2 add_matchpoint
454 $matcher->add_matchpoint($index, $score, $matchcomponents);
456 Adds a matchpoint that may include multiple components. The $index
457 parameter identifies the index that will be searched, while $score
458 is the weight that will be added if a match is found.
460 $matchcomponents should be a reference to an array of matchpoint
461 compoents, each of which should be a hash containing the following
469 The normalization_rules value should in turn be a reference to an
470 array, each element of which should be a reference to a
471 normalization subroutine (under C4::Normalize) to be applied
472 to the source string.
478 my ($index, $score, $matchcomponents) = @_;
481 $matchpoint->{'index'} = $index;
482 $matchpoint->{'score'} = $score;
483 $matchpoint->{'components'} = [];
484 foreach my $input_component (@
{ $matchcomponents }) {
485 push @
{ $matchpoint->{'components'} }, _parse_match_component
($input_component);
487 push @
{ $self->{'matchpoints'} }, $matchpoint;
490 =head2 add_simple_matchpoint
492 $matcher->add_simple_matchpoint($index, $score, $source_tag,
493 $source_subfields, $source_offset,
494 $source_length, $source_normalizer);
497 Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
498 normalized per the normalization fuction, search the index. All records retrieved
499 will receive the assigned score.
503 sub add_simple_matchpoint
{
505 my ($index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer) = @_;
507 $self->add_matchpoint($index, $score, [
508 { tag
=> $source_tag, subfields
=> $source_subfields,
509 offset
=> $source_offset, 'length' => $source_length,
510 norms
=> [ $source_normalizer ]
515 =head2 add_required_check
517 $match->add_required_check($source_matchpoint, $target_matchpoint);
519 Adds a required check definition. A required check means that in
520 order for a match to be considered valid, the key derived from the
521 source (incoming) record must match the key derived from the target
522 (already in DB) record.
524 Unlike a regular matchpoint, only the first repeat of each tag
525 in the source and target match criteria are considered.
527 A typical example of a required check would be verifying that the
528 titles and publication dates match.
530 $source_matchpoint and $target_matchpoint are each a reference to
531 an array of hashes, where each hash follows the same definition
532 as the matchpoint component specification in add_matchpoint, i.e.,
540 The normalization_rules value should in turn be a reference to an
541 array, each element of which should be a reference to a
542 normalization subroutine (under C4::Normalize) to be applied
543 to the source string.
547 sub add_required_check
{
549 my ($source_matchpoint, $target_matchpoint) = @_;
552 $matchcheck->{'source_matchpoint'}->{'index'} = '';
553 $matchcheck->{'source_matchpoint'}->{'score'} = 0;
554 $matchcheck->{'source_matchpoint'}->{'components'} = [];
555 $matchcheck->{'target_matchpoint'}->{'index'} = '';
556 $matchcheck->{'target_matchpoint'}->{'score'} = 0;
557 $matchcheck->{'target_matchpoint'}->{'components'} = [];
558 foreach my $input_component (@
{ $source_matchpoint }) {
559 push @
{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component
($input_component);
561 foreach my $input_component (@
{ $target_matchpoint }) {
562 push @
{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component
($input_component);
564 push @
{ $self->{'required_checks'} }, $matchcheck;
567 =head2 add_simple_required_check
569 $matcher->add_simple_required_check($source_tag, $source_subfields,
570 $source_offset, $source_length, $source_normalizer,
571 $target_tag, $target_subfields, $target_offset,
572 $target_length, $target_normalizer);
574 Adds a required check, which requires that the normalized keys made from the source and targets
575 must match for a match to be considered valid.
579 sub add_simple_required_check
{
581 my ($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
582 $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer) = @_;
584 $self->add_required_check(
585 [ { tag
=> $source_tag, subfields
=> $source_subfields, offset
=> $source_offset, 'length' => $source_length,
586 norms
=> [ $source_normalizer ] } ],
587 [ { tag
=> $target_tag, subfields
=> $target_subfields, offset
=> $target_offset, 'length' => $target_length,
588 norms
=> [ $target_normalizer ] } ]
594 my @matches = $matcher->get_matches($marc_record, $max_matches);
595 foreach $match (@matches) {
596 # matches already sorted in order of
598 print "record ID: $match->{'record_id'};
599 print "score: $match->{'score'};
602 Identifies all of the records matching the given MARC record. For a record already
603 in the database to be considered a match, it must meet the following criteria:
607 =item 1. Total score from its matching field must exceed the supplied threshold.
609 =item 2. It must pass all required checks.
613 Only the top $max_matches matches are returned. The returned array is sorted
614 in order of decreasing score, i.e., the best match is first.
620 my ($source_record, $max_matches) = @_;
625 $QParser = C4
::Context
->queryparser if (C4
::Context
->preference('UseQueryParser'));
626 foreach my $matchpoint ( @
{ $self->{'matchpoints'} } ) {
627 my @source_keys = _get_match_keys
( $source_record, $matchpoint );
629 next if scalar(@source_keys) == 0;
631 # FIXME - because of a bug in QueryParser, an expression ofthe
632 # format 'isbn:"isbn1" || isbn:"isbn2" || isbn"isbn3"...'
633 # does not get parsed correctly, so we will not
634 # do AggressiveMatchOnISBN if UseQueryParser is on
635 @source_keys = C4
::Koha
::GetVariationsOfISBNs
(@source_keys)
636 if ( $matchpoint->{index} =~ /^isbn$/i
637 && C4
::Context
->preference('AggressiveMatchOnISBN') )
638 && !C4
::Context
->preference('UseQueryParser');
645 if ( $self->{'record_type'} eq 'biblio' ) {
648 $query = join( " || ",
649 map { "$matchpoint->{'index'}:$_" } @source_keys );
652 my $phr = C4
::Context
->preference('AggressiveMatchOnISBN') ?
',phr' : q{};
653 $query = join( " or ",
654 map { "$matchpoint->{'index'}$phr=$_" } @source_keys );
659 ( $error, $searchresults, $total_hits ) =
660 C4
::Search
::SimpleSearch
( $query, 0, $max_matches );
662 elsif ( $self->{'record_type'} eq 'authority' ) {
669 foreach my $key (@source_keys) {
670 push @marclist, $matchpoint->{'index'};
672 push @operator, 'exact';
675 require C4
::AuthoritiesMarc
;
676 ( $authresults, $total_hits ) =
677 C4
::AuthoritiesMarc
::SearchAuthorities
(
678 \
@marclist, \
@and_or, \
@excluding, \
@operator,
679 \
@value, 0, 20, undef,
682 foreach my $result (@
$authresults) {
683 push @
$searchresults, $result->{'authid'};
687 if ( defined $error ) {
688 warn "search failed ($query) $error";
691 foreach my $matched ( @
{$searchresults} ) {
692 $matches{$matched} += $matchpoint->{'score'};
697 # get rid of any that don't meet the threshold
698 %matches = map { ($matches{$_} >= $self->{'threshold'}) ?
($_ => $matches{$_}) : () } keys %matches;
700 # get rid of any that don't meet the required checks
701 %matches = map { _passes_required_checks
($source_record, $_, $self->{'required_checks'}) ?
($_ => $matches{$_}) : () }
702 keys %matches unless ($self->{'record_type'} eq 'auth');
705 if ($self->{'record_type'} eq 'biblio') {
707 foreach my $marcblob (keys %matches) {
708 my $target_record = C4
::Search
::new_record_from_zebra
('biblioserver',$marcblob);
710 my $result = C4
::Biblio
::TransformMarcToKoha
(C4
::Context
->dbh, $target_record, '');
711 $record_number = $result->{'biblionumber'};
712 push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
714 } elsif ($self->{'record_type'} eq 'authority') {
715 require C4
::AuthoritiesMarc
;
716 foreach my $authid (keys %matches) {
717 push @results, { 'record_id' => $authid, 'score' => $matches{$authid} };
720 @results = sort { $b->{'score'} cmp $a->{'score'} } @results;
721 if (scalar(@results) > $max_matches) {
722 @results = @results[0..$max_matches-1];
730 $description = $matcher->dump();
732 Returns a reference to a structure containing all of the information
733 in the matcher object. This is mainly a convenience method to
734 aid setting up a HTML editing form.
743 $result->{'matcher_id'} = $self->{'id'};
744 $result->{'code'} = $self->{'code'};
745 $result->{'description'} = $self->{'description'};
746 $result->{'record_type'} = $self->{'record_type'};
748 $result->{'matchpoints'} = [];
749 foreach my $matchpoint (@
{ $self->{'matchpoints'} }) {
750 push @
{ $result->{'matchpoints'} }, $matchpoint;
752 $result->{'matchchecks'} = [];
753 foreach my $matchcheck (@
{ $self->{'required_checks'} }) {
754 push @
{ $result->{'matchchecks'} }, $matchcheck;
760 sub _passes_required_checks
{
761 my ($source_record, $target_blob, $matchchecks) = @_;
762 my $target_record = MARC
::Record
->new_from_usmarc($target_blob); # FIXME -- need to avoid parsing record twice
764 # no checks supplied == automatic pass
765 return 1 if $#{ $matchchecks } == -1;
767 foreach my $matchcheck (@
{ $matchchecks }) {
768 my $source_key = join "", _get_match_keys
($source_record, $matchcheck->{'source_matchpoint'});
769 my $target_key = join "", _get_match_keys
($target_record, $matchcheck->{'target_matchpoint'});
770 return 0 unless $source_key eq $target_key;
775 sub _get_match_keys
{
776 my $source_record = shift;
777 my $matchpoint = shift;
778 my $check_only_first_repeat = @_ ?
shift : 0;
780 # If there is more than one component to the matchpoint (e.g.,
781 # matchpoint includes both 003 and 001), any repeats
782 # of the first component's tag are identified; repeats
783 # of the subsequent components' tags are appended to
784 # each parallel key dervied from the first component,
785 # up to the number of repeats of the first component's tag.
787 # For example, if the record has one 003 and two 001s, only
788 # one key is retrieved because there is only one 003. The key
789 # will consist of the contents of the first 003 and first 001.
791 # If there are two 003s and two 001s, there will be two keys:
792 # first 003 + first 001
793 # second 003 + second 001
796 for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
797 my $component = $matchpoint->{'components'}->[$i];
799 FIELD
: foreach my $field ($source_record->field($component->{'tag'})) {
801 last FIELD
if $j > 0 and $check_only_first_repeat;
802 last FIELD
if $i > 0 and $j > $#keys;
805 if ($field->is_control_field()) {
806 $string=$field->data();
808 foreach my $subfield ($field->subfields()) {
809 if (exists $component->{'subfields'}->{$subfield->[0]}) {
810 $string .= " " . $subfield->[1];
814 if ($component->{'length'}>0) {
815 $string= substr($string, $component->{'offset'}, $component->{'length'});
816 # FIXME normalize, substr
817 } elsif ($component->{'offset'}) {
818 $string= substr($string, $component->{'offset'});
820 $key = _normalize
($string);
822 push @keys, $key if $key;
824 $keys[$j] .= " $key" if $key;
832 sub _parse_match_component
{
833 my $input_component = shift;
836 $component->{'tag'} = $input_component->{'tag'};
837 $component->{'subfields'} = { map { $_ => 1 } split(//, $input_component->{'subfields'}) };
838 $component->{'offset'} = exists($input_component->{'offset'}) ?
$input_component->{'offset'} : -1;
839 $component->{'length'} = $input_component->{'length'} ?
$input_component->{'length'} : 0;
840 $component->{'norms'} = $input_component->{'norms'} ?
$input_component->{'norms'} : [];
845 # FIXME - default normalizer
847 my $value = uc shift;
848 $value =~ s/[.;:,\]\[\)\(\/'"]//g
;
850 #$value =~ s/^\s+$//;
853 #$value =~ s/[.;,\]\[\)\(\/"']//g;
862 Koha Development Team <http://koha-community.org/>
864 Galen Charlton <galen.charlton@liblime.com>