apertium-unicode/apertium/morpho_stream.cc

   1 /*
   2  * Copyright (C) 2005 Universitat d'Alacant / Universidad de Alicante
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of the
   7  * License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  17  * 02111-1307, USA.
  18  */
  19 /**
  20  *  Word class and MorphoStream class definitions
  21  *
  22  *  @author     Felipe Sánchez-Martínez
  23  */
  24
  25 #include <apertium/morpho_stream.h>
  26 #include <apertium/constant_manager.h>
  27 #include <vector>
  28 #include <apertium/string_utils.h>
  29
  30 using namespace Apertium;
  31 MorphoStream::MorphoStream(FILE *ftxt, bool d, TaggerData *t)
  32 {
  33   foundEOF = false;
  34   debug=d;
  35   td = t;
  36   me = td->getPatternList().newMatchExe();
  37   alphabet = td->getPatternList().getAlphabet();
  38   input = ftxt;
  39   ca_any_char = alphabet(PatternList::ANY_CHAR);
  40   ca_any_tag = alphabet(PatternList::ANY_TAG);
  41
  42   ConstantManager &constants = td->getConstants();
  43   ca_kignorar = constants.getConstant(L"kIGNORAR");
  44   ca_kbarra = constants.getConstant(L"kBARRA");
  45   ca_kdollar = constants.getConstant(L"kDOLLAR");
  46   ca_kbegin = constants.getConstant(L"kBEGIN");
  47   ca_kmot = constants.getConstant(L"kMOT");
  48   ca_kmas = constants.getConstant(L"kMAS");
  49   ca_kunknown = constants.getConstant(L"kUNKNOWN");
  50
  51   map<wstring, int, Ltstr> &tag_index = td->getTagIndex();
  52   ca_tag_keof = tag_index[L"TAG_kEOF"];
  53   ca_tag_kundef = tag_index[L"TAG_kUNDEF"];
  54 }
  55
  56 MorphoStream::~MorphoStream()
  57 {
  58   delete me;
  59 }
  60
  61 TaggerWord*
  62 MorphoStream::get_next_word()
  63 {
  64   if(vwords.size() != 0)
  65   {
  66     TaggerWord* word=vwords.front();
  67     vwords.erase(vwords.begin());
  68 //    cout << *word << endl;
  69     return word;
  70   }
  71
  72   if(feof(input))
  73   {
  74     return NULL;
  75   }
  76
  77   int ivwords = 0;
  78   vwords.push_back(new TaggerWord());
  79
  80   while(true)
  81   {
  82     int symbol = fgetwc_unlocked(input);
  83     if(feof(input))
  84     {
  85       vwords[ivwords]->add_tag(ca_tag_keof, L"", td->getPreferRules());
  86       return get_next_word();
  87     }
  88     if(symbol == L'^')
  89     {
  90       readRestOfWord(ivwords);
  91       return get_next_word();
  92     }
  93     else
  94     {
  95       wstring str = L"";
  96       if(symbol == L'\\')
  97       {
  98         symbol = fgetwc_unlocked(input);
  99         str += L'\\';
 100       }
 101       str += static_cast<wchar_t>(symbol);
 102
 103       while(symbol != L'^')
 104       {
 105         symbol = fgetwc_unlocked(input);
 106         if(feof(input))
 107         {
 108           vwords[ivwords]->add_ignored_string(str);
 109           vwords[ivwords]->add_tag(ca_tag_keof, L"", td->getPreferRules());
 110           return get_next_word();
 111         }
 112         else if(symbol == L'\\')
 113         {
 114           str += L'\\';
 115           symbol = fgetwc_unlocked(input);
 116           if(feof(input))
 117           {
 118             vwords[ivwords]->add_ignored_string(str);
 119             vwords[ivwords]->add_tag(ca_tag_keof, L"", td->getPreferRules());
 120             return get_next_word();
 121           }
 122           str += static_cast<wchar_t>(symbol);
 123         }
 124         else if(symbol == L'^')
 125         {
 126           if(str.size() > 0)
 127           {
 128             vwords[ivwords]->add_ignored_string(str);
 129           }
 130           readRestOfWord(ivwords);
 131           return get_next_word();
 132         }
 133         else
 134         {
 135           str += static_cast<wchar_t>(symbol);
 136         }
 137       }
 138     }
 139   }
 140 }
 141
 142 void
 143 MorphoStream::lrlmClassify(wstring const &str, int &ivwords)
 144 {
 145   int floor = 0;
 146   int last_type = -1;
 147   int last_pos = 0;
 148
 149   ms.init(me->getInitial());
 150   for(int i = 0, limit = str.size(); i != limit; i++)
 151   {
 152     if(str[i] != L'<')
 153     {
 154       if(str[i] == L'+')
 155       {
 156         int val = ms.classifyFinals(me->getFinals());
 157         if(val != -1)
 158         {
 159           last_pos = i-1;
 160           last_type = val;
 161         }
 162       }
 163       ms.step(towlower(str[i]), ca_any_char);
 164     }
 165     else
 166     {
 167       wstring tag = L"";
 168       for(int j = i+1; j != limit; j++)
 169       {
 170         if(str[j] == L'\\')
 171         {
 172           j++;
 173         }
 174         else if(str[j] == L'>')
 175         {
 176           tag = str.substr(i, j-i+1);
 177           i = j;
 178           break;
 179         }
 180       }
 181
 182       int symbol = alphabet(tag);
 183       if(symbol)
 184       {
 185         ms.step(symbol, ca_any_tag);
 186       }
 187       else
 188       {
 189         ms.step(ca_any_tag);
 190       }
 191     }
 192
 193     if(ms.size() == 0)
 194     {
 195       if(last_pos != floor)
 196       {
 197         vwords[ivwords]->add_tag(last_type,
 198                                  str.substr(floor, last_pos - floor + 1),
 199                                  td->getPreferRules());
 200         if(str[last_pos+1] == L'+' && last_pos+1 < limit )
 201         {
 202           floor = last_pos + 1;
 203           last_pos = floor;
 204           vwords[ivwords]->set_plus_cut(true);
 205           if (((int)vwords.size())<=((int)(ivwords+1)))
 206             vwords.push_back(new TaggerWord(true));
 207           ivwords++;
 208           ms.init(me->getInitial());
 209         }
 210         i = floor++;
 211       }
 212       else
 213       {
 214         if (debug)
 215         {
 216           wcerr<<L"Warning: There is not coarse tag for the fine tag '"<< str.substr(floor) <<L"'\n";
 217           wcerr<<L"         This is because of an incomplete tagset definition or a dictionary error\n";
 218         }
 219         vwords[ivwords]->add_tag(ca_tag_kundef, str.substr(floor) , td->getPreferRules());
 220         return;
 221       }
 222     }
 223     else if(i == limit - 1)
 224     {
 225       if(ms.classifyFinals(me->getFinals()) == -1)
 226       {
 227         if(last_pos != floor)
 228         {
 229           vwords[ivwords]->add_tag(last_type,
 230                                    str.substr(floor, last_pos - floor + 1),
 231                                    td->getPreferRules());
 232           if(str[last_pos+1] == L'+' && last_pos+1 < limit )
 233           {
 234             floor = last_pos + 1;
 235             last_pos = floor;
 236             vwords[ivwords]->set_plus_cut(true);
 237             if (((int)vwords.size())<=((int)(ivwords+1)))
 238               vwords.push_back(new TaggerWord(true));
 239             ivwords++;
 240             ms.init(me->getInitial());
 241           }
 242           i = floor++;
 243         }
 244         else
 245         {
 246           if (debug)
 247           {
 248             wcerr<<L"Warning: There is not coarse tag for the fine tag '"<< str.substr(floor) <<L"'\n";
 249             wcerr<<L"         This is because of an incomplete tagset definition or a dictionary error\n";
 250           }
 251           vwords[ivwords]->add_tag(ca_tag_kundef, str.substr(floor) , td->getPreferRules());
 252           return;
 253         }
 254       }
 255     }
 256   }
 257
 258   int val = ms.classifyFinals(me->getFinals());
 259   if(val == -1)
 260   {
 261     val = ca_tag_kundef;
 262     if (debug)
 263     {
 264       wcerr<<L"Warning: There is not coarse tag for the fine tag '"<< str.substr(floor) <<L"'\n";
 265       wcerr<<L"         This is because of an incomplete tagset definition or a dictionary error\n";
 266     }
 267
 268   }
 269   vwords[ivwords]->add_tag(val, str.substr(floor), td->getPreferRules());
 270 }
 271
 272 void
 273 MorphoStream::readRestOfWord(int &ivwords)
 274 {
 275   // first we have the superficial form
 276   wstring  str = L"";
 277
 278   while(true)
 279   {
 280     int symbol = fgetwc_unlocked(input);
 281     if(feof(input))
 282     {
 283       if(str.size() > 0)
 284       {
 285         vwords[ivwords]->add_ignored_string(str);
 286         wcerr<<L"Warning (internal): kIGNORE was returned while reading a word\n";
 287         wcerr<<L"Word being read: "<<vwords[ivwords]->get_superficial_form()<<L"\n";
 288         wcerr<<L"Debug: "<< str <<L"\n";
 289       }
 290       vwords[ivwords]->add_tag(ca_tag_keof, L"", td->getPreferRules());
 291       return;
 292     }
 293     else if(symbol == L'\\')
 294     {
 295       symbol = fgetwc_unlocked(input);
 296       str += L'\\';
 297       str += static_cast<wchar_t>(symbol);
 298     }
 299     else if(symbol == L'/')
 300     {
 301       vwords[ivwords]->set_superficial_form(str);
 302       str = L"";
 303       break;
 304     }
 305     else if(symbol == L'$')
 306     {
 307       vwords[ivwords]->set_superficial_form(str);
 308       vwords[ivwords]->add_ignored_string(L"$");
 309       break;
 310     }
 311     else
 312     {
 313       str += static_cast<wchar_t>(symbol);
 314     }
 315   }
 316
 317   // then we read the acceptions
 318
 319   while(true)
 320   {
 321     int symbol = fgetwc_unlocked(input);
 322     if(feof(input))
 323     {
 324       if(str.size() > 0)
 325       {
 326         vwords[ivwords]->add_ignored_string(str);
 327         wcerr<<L"Warning (internal): kIGNORE was returned while reading a word\n";
 328         wcerr<<L"Word being read: "<<vwords[ivwords]->get_superficial_form()<<L"\n";
 329         wcerr<<L"Debug: "<< str <<L"\n";
 330       }
 331       vwords[ivwords]->add_tag(ca_tag_keof, L"", td->getPreferRules());
 332       return;
 333     }
 334     else if(symbol == L'\\')
 335     {
 336       symbol = fgetwc_unlocked(input);
 337       str += L'\\';
 338       str += static_cast<wchar_t>(symbol);
 339       symbol = L'\\';  // to prevent exiting with '\$'
 340     }
 341     else if(symbol == L'/')
 342     {
 343       lrlmClassify(str, ivwords);
 344       str = L"";
 345       ivwords = 0;
 346       continue;
 347     }
 348     else if(symbol == L'$')
 349     {
 350       if(str[0] != L'*')// do nothing with unknown words
 351       {
 352         lrlmClassify(str, ivwords);
 353       }
 354       return;
 355     }
 356     else
 357     {
 358       str += static_cast<wchar_t>(symbol);
 359     }
 360   }
 361 }