4 /* Copyright (C) 2010 Richard Boulton
5 * Copyright (C) 2016 Richhiey Thomas
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License as
9 * published by the Free Software Foundation; either version 2 of the
10 * License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
25 #include "xapian/cluster.h"
27 #include "cluster/clusterinternal.h"
28 #include "xapian/error.h"
29 #include "api/termlist.h"
34 #include <unordered_map>
37 using namespace Xapian
;
40 FreqSource::~FreqSource()
42 LOGCALL_DTOR(API
, "FreqSource");
45 Similarity::~Similarity()
47 LOGCALL_DTOR(API
, "Similarity");
50 Clusterer::~Clusterer()
52 LOGCALL_DTOR(API
, "Clusterer");
55 TermListGroup::TermListGroup(const MSet
&docs
, const Stopper
*stopper
)
57 LOGCALL_CTOR(API
, "TermListGroup", docs
| stopper
);
58 for (MSetIterator it
= docs
.begin(); it
!= docs
.end(); ++it
)
59 add_document(it
.get_document(), stopper
);
60 num_of_documents
= docs
.size();
64 DummyFreqSource::get_termfreq(const string
&) const
66 LOGCALL(API
, doccount
, "DummyFreqSource::get_termfreq", NO_ARGS
);
71 DummyFreqSource::get_doccount() const
73 LOGCALL(API
, doccount
, "DummyFreqSource::get_doccount", NO_ARGS
);
78 TermListGroup::add_document(const Document
&document
, const Stopper
*stopper
)
80 LOGCALL_VOID(API
, "TermListGroup::add_document", document
| stopper
);
82 TermIterator
titer(document
.termlist_begin());
84 for (; titer
!= document
.termlist_end(); ++titer
) {
85 const string
&term
= *titer
;
87 // Remove stopwords by using the Xapian::Stopper object
88 if (stopper
&& (*stopper
)(term
))
91 // Remove unstemmed terms since document vector should
92 // contain only stemmed terms
96 unordered_map
<string
, doccount
>::iterator i
;
97 i
= termfreq
.find(term
);
98 if (i
== termfreq
.end())
106 TermListGroup::get_doccount() const
108 LOGCALL(API
, doccount
, "TermListGroup::get_doccount", NO_ARGS
);
109 return num_of_documents
;
113 TermListGroup::get_termfreq(const string
&tname
) const
115 LOGCALL(API
, doccount
, "TermListGroup::get_termfreq", tname
);
116 unordered_map
<string
, doccount
>::const_iterator it
= termfreq
.find(tname
);
117 if (it
!= termfreq
.end())
123 DocumentSet::DocumentSet(const DocumentSet
&) = default;
126 DocumentSet::operator=(const DocumentSet
&) = default;
128 DocumentSet::DocumentSet(DocumentSet
&&) = default;
131 DocumentSet::operator=(DocumentSet
&&) = default;
133 DocumentSet::DocumentSet()
134 : internal(new Xapian::DocumentSet::Internal
)
139 DocumentSet::size() const
141 LOGCALL(API
, doccount
, "DocumentSet::size", NO_ARGS
);
142 return internal
->size();
146 DocumentSet::Internal::size() const
148 return documents
.size();
152 DocumentSet::add_document(const Document
&document
)
154 LOGCALL_VOID(API
, "DocumentSet::add_document", document
);
155 internal
->add_document(document
);
158 DocumentSet::Internal::add_document(const Document
&document
)
160 documents
.push_back(document
);
164 DocumentSet::operator[](doccount i
)
166 return internal
->get_document(i
);
170 DocumentSet::Internal::get_document(doccount i
)
176 DocumentSet::operator[](doccount i
) const
178 return internal
->get_document(i
);
182 DocumentSet::Internal::get_document(doccount i
) const
187 DocumentSet::~DocumentSet()
189 LOGCALL_DTOR(API
, "DocumentSet");
192 class PointTermIterator
: public TermIterator::Internal
{
193 unordered_map
<string
, double>::const_iterator i
;
194 unordered_map
<string
, double>::const_iterator end
;
198 PointTermIterator(const unordered_map
<string
, double> &termlist
)
199 : i(termlist
.begin()), end(termlist
.end()),
200 size(termlist
.size()), started(false)
202 termcount
get_approx_size() const { return size
; }
203 termcount
get_wdf() const { throw UnimplementedError("PointIterator doesn't support get_wdf()"); }
204 string
get_termname() const { return i
->first
; }
205 doccount
get_termfreq() const { throw UnimplementedError("PointIterator doesn't support get_termfreq()"); }
207 termcount
positionlist_count() const {
208 throw UnimplementedError("PointTermIterator doesn't support positionlist_count()");
211 PositionList
* positionlist_begin() const {
212 throw UnimplementedError("PointTermIterator doesn't support positionlist_begin()");
214 Internal
* skip_to(const string
&) {
215 throw UnimplementedError("PointTermIterator doesn't support skip_to()");
219 TermIterator::Internal
*
220 PointTermIterator::next()
232 PointTermIterator::at_end() const
234 if (!started
) return false;
239 PointType::termlist_begin() const
241 LOGCALL(API
, TermIterator
, "PointType::termlist_begin", NO_ARGS
);
242 return TermIterator(new PointTermIterator(weights
));
246 PointType::contains(const string
&term
) const
248 LOGCALL(API
, bool, "PointType::contains", term
);
249 return weights
.find(term
) != weights
.end();
253 PointType::get_weight(const string
&term
) const
255 LOGCALL(API
, double, "PointType::get_weight", term
);
256 unordered_map
<string
, double>::const_iterator it
= weights
.find(term
);
257 return (it
== weights
.end()) ? 0.0 : it
->second
;
261 PointType::get_magnitude() const {
262 LOGCALL(API
, double, "PointType::get_magnitude", NO_ARGS
);
267 PointType::add_weight(const string
&term
, double weight
)
269 LOGCALL_VOID(API
, "PointType::add_weight", term
| weight
);
270 unordered_map
<string
, double>::iterator it
;
271 it
= weights
.find(term
);
272 if (it
!= weights
.end())
273 it
->second
+= weight
;
275 weights
[term
] = weight
;
279 PointType::set_weight(const string
&term
, double weight
)
281 LOGCALL_VOID(API
, "PointType::set_weight", term
| weight
);
282 weights
[term
] = weight
;
286 PointType::termlist_size() const
288 LOGCALL(API
, termcount
, "PointType::termlist_size", NO_ARGS
);
289 return weights
.size();
293 Point::get_document() const
295 LOGCALL(API
, Document
, "Point::get_document", NO_ARGS
);
299 Point::Point(const TermListGroup
&tlg
, const Document
&document_
)
301 LOGCALL_CTOR(API
, "Point::initialize", tlg
| document_
);
302 doccount size
= tlg
.get_doccount();
303 document
= document_
;
304 for (TermIterator it
= document
.termlist_begin(); it
!= document
.termlist_end(); ++it
) {
305 doccount wdf
= it
.get_wdf();
307 double termfreq
= tlg
.get_termfreq(term
);
309 // If the term exists in only one document, or if it exists in
310 // every document within the MSet, or if it is a filter term, then
311 // these terms are not used for document vector calculations
312 if (wdf
< 1 || termfreq
<= 1 || size
== termfreq
)
315 double tf
= 1 + log((double)wdf
);
316 double idf
= log(size
/ termfreq
);
317 double wt
= tf
* idf
;
320 magnitude
+= wt
* wt
;
324 Centroid::Centroid(const Point
&point
) {
325 LOGCALL_CTOR(API
, "Centroid", point
);
326 for (TermIterator it
= point
.termlist_begin(); it
!= point
.termlist_end(); ++it
)
327 weights
[*it
] = point
.get_weight(*it
);
328 magnitude
= point
.get_magnitude();
332 Centroid::divide(double cluster_size
)
334 LOGCALL_VOID(API
, "Centroid::divide", cluster_size
);
336 unordered_map
<string
, double>::iterator it
;
337 for (it
= weights
.begin(); it
!= weights
.end(); ++it
) {
338 double new_weight
= it
->second
/ cluster_size
;
339 it
->second
= new_weight
;
340 magnitude
+= new_weight
* new_weight
;
347 LOGCALL_VOID(API
, "Centroid::clear", NO_ARGS
);
352 Cluster::operator=(const Cluster
&) = default;
354 Cluster::Cluster(const Cluster
&) = default;
356 Cluster::Cluster(Cluster
&&) = default;
359 Cluster::operator=(Cluster
&&) = default;
361 Cluster::Cluster() : internal(new Xapian::Cluster::Internal
)
363 LOGCALL_CTOR(API
, "Cluster", NO_ARGS
);
366 Cluster::Cluster(const Centroid
¢roid
)
367 : internal(new Xapian::Cluster::Internal(centroid
))
369 LOGCALL_CTOR(API
, "Cluster", centroid
);
374 LOGCALL_DTOR(API
, "Cluster");
379 LOGCALL_CTOR(API
, "Centroid", NO_ARGS
);
383 Cluster::get_documents() const
385 LOGCALL(API
, DocumentSet
, "Cluster::get_documents", NO_ARGS
);
386 return internal
->get_documents();
390 Cluster::Internal::get_documents() const
393 for (auto&& point
: cluster_docs
)
394 docs
.add_document(point
.get_document());
399 Cluster::operator[](Xapian::doccount i
)
401 return internal
->get_point(i
);
405 Cluster::Internal::get_point(Xapian::doccount i
)
407 return cluster_docs
[i
];
411 Cluster::operator[](Xapian::doccount i
) const
413 return internal
->get_point(i
);
417 Cluster::Internal::get_point(Xapian::doccount i
) const
419 return cluster_docs
[i
];
423 ClusterSet::operator=(const ClusterSet
&) = default;
425 ClusterSet::ClusterSet(const ClusterSet
&) = default;
428 ClusterSet::operator=(ClusterSet
&&) = default;
430 ClusterSet::ClusterSet(ClusterSet
&&) = default;
432 ClusterSet::ClusterSet() : internal(new Xapian::ClusterSet::Internal
)
436 ClusterSet::~ClusterSet()
441 ClusterSet::Internal::size() const
443 return clusters
.size();
447 ClusterSet::size() const
449 LOGCALL(API
, doccount
, "ClusterSet::size", NO_ARGS
);
450 return internal
->size();
454 ClusterSet::Internal::add_cluster(const Cluster
&cluster
)
456 clusters
.push_back(cluster
);
460 ClusterSet::add_cluster(const Cluster
&cluster
)
462 LOGCALL_VOID(API
, "ClusterSet::add_cluster", cluster
);
463 internal
->add_cluster(cluster
);
467 ClusterSet::Internal::get_cluster(doccount i
)
473 ClusterSet::operator[](doccount i
)
475 return internal
->get_cluster(i
);
479 ClusterSet::Internal::get_cluster(doccount i
) const
485 ClusterSet::operator[](doccount i
) const
487 return internal
->get_cluster(i
);
491 ClusterSet::Internal::add_to_cluster(const Point
&point
, unsigned int index
)
493 clusters
[index
].add_point(point
);
497 ClusterSet::add_to_cluster(const Point
&point
, unsigned int index
)
499 LOGCALL_VOID(API
, "ClusterSet::add_to_cluster", point
| index
);
500 internal
->add_to_cluster(point
, index
);
504 ClusterSet::Internal::recalculate_centroids()
506 for (auto&& cluster
: clusters
)
507 cluster
.recalculate();
511 ClusterSet::recalculate_centroids()
513 LOGCALL_VOID(API
, "ClusterSet::recalculate_centroids", NO_ARGS
);
514 internal
->recalculate_centroids();
518 ClusterSet::clear_clusters()
520 LOGCALL_VOID(API
, "ClusterSet::clear_clusters", NO_ARGS
);
521 internal
->clear_clusters();
525 ClusterSet::Internal::clear_clusters()
527 for (auto&& cluster
: clusters
)
532 Cluster::size() const
534 LOGCALL(API
, doccount
, "Cluster::size", NO_ARGS
);
535 return internal
->size();
539 Cluster::Internal::size() const
541 return (cluster_docs
.size());
545 Cluster::add_point(const Point
&point
)
547 LOGCALL_VOID(API
, "Cluster::add_point", point
);
548 internal
->add_point(point
);
552 Cluster::Internal::add_point(const Point
&point
)
554 cluster_docs
.push_back(point
);
560 LOGCALL_VOID(API
, "Cluster::clear", NO_ARGS
);
565 Cluster::Internal::clear()
567 cluster_docs
.clear();
571 Cluster::get_centroid() const
573 LOGCALL(API
, Centroid
, "Cluster::get_centroid", NO_ARGS
);
574 return internal
->get_centroid();
578 Cluster::Internal::get_centroid() const
584 Cluster::set_centroid(const Centroid
¢roid_
)
586 LOGCALL_VOID(API
, "Cluster::set_centroid", centroid_
);
587 internal
->set_centroid(centroid_
);
591 Cluster::Internal::set_centroid(const Centroid
¢roid_
)
593 centroid
= centroid_
;
597 Cluster::recalculate()
599 LOGCALL_VOID(API
, "Cluster::recalculate", NO_ARGS
);
600 internal
->recalculate();
604 Cluster::Internal::recalculate()
607 for (const Point
& temp
: cluster_docs
) {
608 for (TermIterator titer
= temp
.termlist_begin(); titer
!= temp
.termlist_end(); ++titer
)
609 centroid
.add_weight(*titer
, temp
.get_weight(*titer
));
611 centroid
.divide(size());
614 StemStopper::StemStopper(const Stem
&stemmer_
, stem_strategy strategy
)
615 : stem_action(strategy
), stemmer(stemmer_
)
617 LOGCALL_CTOR(API
, "StemStopper", stemmer_
| strategy
);
621 StemStopper::get_description() const
623 string
desc("Xapian::StemStopper(");
624 unordered_set
<string
>::const_iterator i
;
625 for (i
= stop_words
.begin(); i
!= stop_words
.end(); ++i
) {
626 if (i
!= stop_words
.begin()) desc
+= ' ';
634 StemStopper::add(const string
&term
)
636 LOGCALL_VOID(API
, "StemStopper::add", term
);
637 switch (stem_action
) {
639 stop_words
.insert(term
);
642 stop_words
.insert('Z' + stemmer(term
));
645 stop_words
.insert(stemmer(term
));
648 case STEM_SOME_FULL_POS
:
649 stop_words
.insert(term
);
650 stop_words
.insert('Z' + stemmer(term
));