xapian-core/tests/stemtest.cc

   1 /** @file
   2  * @brief Test stemming algorithms
   3  */
   4 /* Copyright 1999,2000,2001 BrightStation PLC
   5  * Copyright 2002 Ananova Ltd
   6  * Copyright 2002,2003,2004,2007,2008,2009,2012,2015 Olly Betts
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include <cstdlib>
  27
  28 #include <string>
  29 #include <fstream>
  30 #include <iostream>
  31
  32 #include <xapian.h>
  33 #include "parseint.h"
  34 #include "testsuite.h"
  35
  36 using namespace std;
  37
  38 static const int JUNKSIZE = 2 * 1048576;
  39
  40 static string language;
  41
  42 static Xapian::Stem stemmer;
  43
  44 static string srcdir;
  45
  46 static int seed;
  47
  48 // run stemmers on random text
  49 static void
  50 test_stemrandom()
  51 {
  52     static const char wordchars[] =
  53         "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz0123456789^\0";
  54
  55     tout << "Stemming random text... (seed " << seed << ")\n";
  56     srand(seed);
  57
  58     string word;
  59     int stemmed_size = 0;
  60     for (int c = JUNKSIZE; c; --c) {
  61         char ch = wordchars[(rand() >> 8) % sizeof wordchars];
  62         if (ch) {
  63             word += ch;
  64             continue;
  65         }
  66         stemmed_size += stemmer(word).length();
  67         word.resize(0);
  68     }
  69     stemmed_size += stemmer(word).length();
  70     tout << "Input size " << JUNKSIZE << ", stemmed size " << stemmed_size
  71          << '\n';
  72
  73     if (stemmed_size > JUNKSIZE * 101 / 100) {
  74         FAIL_TEST("Stemmed data is significantly bigger than input: "
  75                   << stemmed_size << " vs. " << JUNKSIZE);
  76     }
  77     if (stemmed_size < JUNKSIZE / 2) {
  78         FAIL_TEST("Stemmed data is significantly smaller than input: "
  79                   << stemmed_size << " vs. " << JUNKSIZE);
  80     }
  81 }
  82
  83 // run stemmers on random junk
  84 static void
  85 test_stemjunk()
  86 {
  87     tout << "Stemming random junk... (seed " << seed << ")\n";
  88     srand(seed);
  89
  90     string word;
  91     int stemmed_size = 0;
  92     for (int c = JUNKSIZE; c; --c) {
  93         char ch = char(rand() >> 8);
  94         if (ch) {
  95             word += ch;
  96             continue;
  97         }
  98         stemmed_size += stemmer(word).length();
  99         word.resize(0);
 100     }
 101     stemmed_size += stemmer(word).length();
 102     tout << "Input size " << JUNKSIZE << ", stemmed size " << stemmed_size
 103          << '\n';
 104
 105     if (stemmed_size > JUNKSIZE * 101 / 100) {
 106         FAIL_TEST("Stemmed data is significantly bigger than input ("
 107                   << stemmed_size << " vs. " << JUNKSIZE);
 108     }
 109     if (stemmed_size < JUNKSIZE / 2) {
 110         FAIL_TEST("Stemmed data is significantly smaller than input ("
 111                   << stemmed_size << " vs. " << JUNKSIZE);
 112     }
 113 }
 114
 115 static void
 116 test_stemdict()
 117 {
 118     string dir = srcdir + "/../../xapian-data/stemming/";
 119
 120     ifstream voc((dir + language + "/voc.txt").c_str());
 121     if (!voc.is_open()) {
 122         SKIP_TEST(language << "/voc.txt not found");
 123     }
 124
 125     ifstream st((dir + language + "/output.txt").c_str());
 126     if (!st.is_open()) {
 127         voc.close();
 128         FAIL_TEST(language << "/output.txt not found");
 129     }
 130
 131     tout << "Testing " << language << " with Snowball dictionary...\n";
 132
 133     int pass = 1;
 134     while (true) {
 135         string word, stem, expect;
 136         while (!voc.eof() && !st.eof()) {
 137             getline(voc, word);
 138             getline(st, expect);
 139
 140             stem = stemmer(word);
 141
 142             TEST_EQUAL(stem, expect);
 143         }
 144         voc.close();
 145         st.close();
 146
 147         if (pass == 2) break;
 148
 149         voc.open((dir + language + "/voc2.txt").c_str());
 150         if (!voc.is_open()) break;
 151
 152         st.open((dir + language + "/output2.txt").c_str());
 153         if (!st.is_open()) {
 154             voc.close();
 155             FAIL_TEST(language << "/output2.txt not found");
 156         }
 157         tout << "Testing " << language << " with supplemental dictionary...\n";
 158         ++pass;
 159     }
 160 }
 161
 162 // ##################################################################
 163 // # End of actual tests                                            #
 164 // ##################################################################
 165
 166 /// The lists of tests to perform
 167 static const test_desc tests[] = {
 168     {"stemrandom",              test_stemrandom},
 169     {"stemjunk",                test_stemjunk},
 170     {"stemdict",                test_stemdict},
 171     {0, 0}
 172 };
 173
 174 int main(int argc, char **argv)
 175 try {
 176     string langs = Xapian::Stem::get_available_languages();
 177     test_driver::add_command_line_option("languages", 'l', &langs);
 178
 179     seed = 42;
 180     string seed_str;
 181     test_driver::add_command_line_option("seed", 's', &seed_str);
 182
 183     test_driver::parse_command_line(argc, argv);
 184     srcdir = test_driver::get_srcdir();
 185     int result = 0;
 186
 187     if (!seed_str.empty()) {
 188         if (!parse_signed(seed_str.c_str(), seed)) {
 189             throw "seed must be an integer";
 190         }
 191     }
 192     cout << "The random seed is " << seed << '\n';
 193     cout << "Please report the seed when reporting a test failure.\n";
 194
 195     string::size_type b = 0;
 196     while (b != langs.size()) {
 197         string::size_type a = b;
 198         while (b < langs.size() && langs[b] != ' ') ++b;
 199         language.assign(langs, a, b - a);
 200         while (b < langs.size() && langs[b] == ' ') ++b;
 201         cout << "Running tests with " << language << " stemmer...\n";
 202         stemmer = Xapian::Stem(language);
 203         result = max(result, test_driver::run(tests));
 204     }
 205     return result;
 206 } catch (const char * e) {
 207     cout << e << '\n';
 208     return 1;
 209 }