xapian-core/languages/steminternal.h

   1 /** @file steminternal.h
   2  *  @brief Base class for implementations of stemming algorithms
   3  */
   4 /* Copyright (C) 2007,2009,2010,2016 Olly Betts
   5  * Copyright (C) 2010 Evgeny Sizikov
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #ifndef XAPIAN_INCLUDED_STEMINTERNAL_H
  23 #define XAPIAN_INCLUDED_STEMINTERNAL_H
  24
  25 #include <xapian/stem.h>
  26
  27 #include <cstdlib>
  28 #include <string>
  29
  30 typedef unsigned char symbol;
  31
  32 #define HEAD (2*sizeof(int))
  33
  34 // Cast via (void*) to avoid warnings about alignment (the pointers *are*
  35 // appropriately aligned).
  36
  37 inline int
  38 SIZE(const symbol* p)
  39 {
  40     const void * void_p = reinterpret_cast<const void *>(p);
  41     return reinterpret_cast<const int *>(void_p)[-1];
  42 }
  43
  44 inline void
  45 SET_SIZE(symbol* p, int n)
  46 {
  47     void * void_p = reinterpret_cast<void *>(p);
  48     reinterpret_cast<int *>(void_p)[-1] = n;
  49 }
  50
  51 inline int
  52 CAPACITY(const symbol* p)
  53 {
  54     const void * void_p = reinterpret_cast<const void *>(p);
  55     return reinterpret_cast<const int *>(void_p)[-2];
  56 }
  57
  58 inline void
  59 SET_CAPACITY(symbol* p, int n)
  60 {
  61     void * void_p = reinterpret_cast<void *>(p);
  62     reinterpret_cast<int *>(void_p)[-2] = n;
  63 }
  64
  65 typedef int (*among_function)(Xapian::StemImplementation *);
  66
  67 struct among {
  68     int s_size;         /* length of search string (in symbols) */
  69     unsigned s;         /* offset in pool to search string */
  70     int substring_i;    /* index to longest matching substring */
  71     int result;         /* result of the lookup */
  72 };
  73
  74 extern symbol * create_s();
  75
  76 inline void lose_s(symbol * p) {
  77     if (p) std::free(reinterpret_cast<char *>(p) - HEAD);
  78 }
  79
  80 extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
  81
  82 namespace Xapian {
  83
  84 class SnowballStemImplementation : public StemImplementation {
  85     int slice_check();
  86
  87   protected:
  88     symbol * p;
  89     int c, l, lb, bra, ket;
  90
  91     int get_utf8(int * slot);
  92     int get_b_utf8(int * slot);
  93
  94     int in_grouping_U(const unsigned char * s, int min, int max, int repeat);
  95     int in_grouping_b_U(const unsigned char * s, int min, int max, int repeat);
  96     int out_grouping_U(const unsigned char * s, int min, int max, int repeat);
  97     int out_grouping_b_U(const unsigned char * s, int min, int max, int repeat);
  98
  99     int eq_s(int s_size, const symbol * s);
 100     int eq_s_b(int s_size, const symbol * s);
 101     int eq_v(const symbol * v) { return eq_s(SIZE(v), v); }
 102     int eq_v_b(const symbol * v) { return eq_s_b(SIZE(v), v); }
 103
 104     int find_among(const symbol *pool, const struct among * v, int v_size,
 105                    const unsigned char * fnum, const among_function * f);
 106     int find_among_b(const symbol *pool, const struct among * v, int v_size,
 107                      const unsigned char * fnum, const among_function * f);
 108
 109     int replace_s(int c_bra, int c_ket, int s_size, const symbol * s);
 110     int slice_from_s(int s_size, const symbol * s);
 111     int slice_from_v(const symbol * v) { return slice_from_s(SIZE(v), v); }
 112
 113     int slice_del() { return slice_from_s(0, 0); }
 114
 115     void insert_s(int c_bra, int c_ket, int s_size, const symbol * s);
 116     void insert_v(int c_bra, int c_ket, const symbol * v) {
 117         insert_s(c_bra, c_ket, SIZE(v), v);
 118     }
 119
 120     symbol * slice_to(symbol * v);
 121     symbol * assign_to(symbol * v);
 122
 123     int len_utf8(const symbol * v);
 124
 125 #if 0
 126     void debug(int number, int line_count);
 127 #endif
 128
 129   public:
 130     /// Perform initialisation common to all Snowball stemmers.
 131     SnowballStemImplementation()
 132         : p(create_s()), c(0), l(0), lb(0), bra(0), ket(0) { }
 133
 134     /// Perform cleanup common to all Snowball stemmers.
 135     virtual ~SnowballStemImplementation();
 136
 137     /// Stem the specified word.
 138     virtual std::string operator()(const std::string & word);
 139
 140     /// Virtual method implemented by the subclass to actually do the work.
 141     virtual int stem() = 0;
 142 };
 143
 144 }
 145
 146 #endif // XAPIAN_INCLUDED_STEMINTERNAL_H