2 * @brief Test stemming algorithms
4 /* Copyright 1999,2000,2001 BrightStation PLC
5 * Copyright 2002 Ananova Ltd
6 * Copyright 2002,2003,2004,2007,2008,2009,2012,2015 Olly Betts
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License as
10 * published by the Free Software Foundation; either version 2 of the
11 * License, or (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
34 #include "testsuite.h"
38 static const int JUNKSIZE
= 2 * 1048576;
40 static string language
;
42 static Xapian::Stem stemmer
;
48 // run stemmers on random text
52 static const char wordchars
[] =
53 "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz0123456789^\0";
55 tout
<< "Stemming random text... (seed " << seed
<< ")\n";
60 for (int c
= JUNKSIZE
; c
; --c
) {
61 char ch
= wordchars
[(rand() >> 8) % sizeof wordchars
];
66 stemmed_size
+= stemmer(word
).length();
69 stemmed_size
+= stemmer(word
).length();
70 tout
<< "Input size " << JUNKSIZE
<< ", stemmed size " << stemmed_size
73 if (stemmed_size
> JUNKSIZE
* 101 / 100) {
74 FAIL_TEST("Stemmed data is significantly bigger than input: "
75 << stemmed_size
<< " vs. " << JUNKSIZE
);
77 if (stemmed_size
< JUNKSIZE
/ 2) {
78 FAIL_TEST("Stemmed data is significantly smaller than input: "
79 << stemmed_size
<< " vs. " << JUNKSIZE
);
83 // run stemmers on random junk
87 tout
<< "Stemming random junk... (seed " << seed
<< ")\n";
92 for (int c
= JUNKSIZE
; c
; --c
) {
93 char ch
= char(rand() >> 8);
98 stemmed_size
+= stemmer(word
).length();
101 stemmed_size
+= stemmer(word
).length();
102 tout
<< "Input size " << JUNKSIZE
<< ", stemmed size " << stemmed_size
105 if (stemmed_size
> JUNKSIZE
* 101 / 100) {
106 FAIL_TEST("Stemmed data is significantly bigger than input ("
107 << stemmed_size
<< " vs. " << JUNKSIZE
);
109 if (stemmed_size
< JUNKSIZE
/ 2) {
110 FAIL_TEST("Stemmed data is significantly smaller than input ("
111 << stemmed_size
<< " vs. " << JUNKSIZE
);
118 string dir
= srcdir
+ "/../../xapian-data/stemming/";
120 ifstream
voc((dir
+ language
+ "/voc.txt").c_str());
121 if (!voc
.is_open()) {
122 SKIP_TEST(language
<< "/voc.txt not found");
125 ifstream
st((dir
+ language
+ "/output.txt").c_str());
128 FAIL_TEST(language
<< "/output.txt not found");
131 tout
<< "Testing " << language
<< " with Snowball dictionary...\n";
135 string word
, stem
, expect
;
136 while (!voc
.eof() && !st
.eof()) {
140 stem
= stemmer(word
);
142 TEST_EQUAL(stem
, expect
);
147 if (pass
== 2) break;
149 voc
.open((dir
+ language
+ "/voc2.txt").c_str());
150 if (!voc
.is_open()) break;
152 st
.open((dir
+ language
+ "/output2.txt").c_str());
155 FAIL_TEST(language
<< "/output2.txt not found");
157 tout
<< "Testing " << language
<< " with supplemental dictionary...\n";
162 // ##################################################################
163 // # End of actual tests #
164 // ##################################################################
166 /// The lists of tests to perform
167 static const test_desc tests
[] = {
168 {"stemrandom", test_stemrandom
},
169 {"stemjunk", test_stemjunk
},
170 {"stemdict", test_stemdict
},
174 int main(int argc
, char **argv
)
176 string langs
= Xapian::Stem::get_available_languages();
177 test_driver::add_command_line_option("languages", 'l', &langs
);
181 test_driver::add_command_line_option("seed", 's', &seed_str
);
183 test_driver::parse_command_line(argc
, argv
);
184 srcdir
= test_driver::get_srcdir();
187 if (!seed_str
.empty()) {
188 if (!parse_signed(seed_str
.c_str(), seed
)) {
189 throw "seed must be an integer";
192 cout
<< "The random seed is " << seed
<< '\n';
193 cout
<< "Please report the seed when reporting a test failure.\n";
195 string::size_type b
= 0;
196 while (b
!= langs
.size()) {
197 string::size_type a
= b
;
198 while (b
< langs
.size() && langs
[b
] != ' ') ++b
;
199 language
.assign(langs
, a
, b
- a
);
200 while (b
< langs
.size() && langs
[b
] == ' ') ++b
;
201 cout
<< "Running tests with " << language
<< " stemmer...\n";
202 stemmer
= Xapian::Stem(language
);
203 result
= max(result
, test_driver::run(tests
));
206 } catch (const char * e
) {