lttoolbox-unicode/lttoolbox/pattern_list.cc

   1 /*
   2  * Copyright (C) 2005 Universitat d'Alacant / Universidad de Alicante
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of the
   7  * License, or (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  17  * 02111-1307, USA.
  18  */
  19 #include <lttoolbox/pattern_list.h>
  20 #include <lttoolbox/compression.h>
  21
  22 #include <cstdlib>
  23 #include <iostream>
  24
  25 wstring const PatternList::ANY_CHAR  = L"<ANY_CHAR>";
  26 wstring const PatternList::ANY_TAG   = L"<ANY_TAG>";
  27 wstring const PatternList::QUEUE = L"<QUEUE>";
  28
  29 void
  30 PatternList::copy(PatternList const &o)
  31 {
  32   sequence = o.sequence;
  33   sequence_data = o.sequence_data;
  34   patterns = o.patterns;
  35   alphabet = o.alphabet;
  36   transducer = o.transducer;
  37   final_type = o.final_type;
  38   sequence_id = o.sequence_id;
  39 }
  40
  41 void
  42 PatternList::destroy()
  43 {
  44 }
  45
  46 PatternList::PatternList()
  47 {
  48   sequence = false;
  49   alphabet.includeSymbol(ANY_TAG);
  50   alphabet.includeSymbol(ANY_CHAR);
  51   alphabet.includeSymbol(QUEUE);
  52 }
  53
  54 PatternList::~PatternList()
  55 {
  56   destroy();
  57 }
  58
  59 PatternList::PatternList(PatternList const &o)
  60 {
  61   copy(o);
  62 }
  63
  64 PatternList &
  65 PatternList::operator =(PatternList const &o)
  66 {
  67   if(this != &o)
  68   {
  69     destroy();
  70     copy(o);
  71   }
  72   return *this;
  73 }
  74
  75 void
  76 PatternList::beginSequence()
  77 {
  78   if(sequence)
  79   {
  80     cerr << "Error: opening an unended sequence" << endl;
  81     exit(EXIT_FAILURE);
  82   }
  83   sequence = true;
  84   sequence_data.clear();
  85 }
  86
  87 void
  88 PatternList::endSequence()
  89 {
  90   if(!sequence)
  91   {
  92     cerr << "Error: ending an unopened sequence" << endl;
  93     exit(EXIT_FAILURE);
  94   }
  95   sequence = false;
  96
  97   for(list<vector<int> >::iterator it = sequence_data.begin(),
  98         limit = sequence_data.end();
  99       it != limit; it++)
 100   {
 101     it->push_back(alphabet(QUEUE));
 102     patterns.insert(pair<int, vector<int> >(sequence_id, *it));
 103   }
 104 }
 105
 106 void
 107 PatternList::insertOutOfSequence(wstring const &lemma, wstring const &tags,
 108                                  vector<int> &result)
 109 {
 110   if(lemma == L"")
 111   {
 112     result.push_back(alphabet(ANY_CHAR));
 113   }
 114   else
 115   {
 116     for(unsigned int i = 0, limit = lemma.size(); i < limit; i++)
 117     {
 118       if(lemma[i] == L'*')
 119       {
 120         result.push_back(alphabet(ANY_CHAR));
 121       }
 122       else
 123       {
 124         result.push_back(int((unsigned char) lemma[i]));
 125       }
 126     }
 127   }
 128   if(tags == L"")
 129   {
 130     result.push_back(alphabet(ANY_TAG));
 131   }
 132   else
 133   {
 134     for(unsigned int i = 0, limit = tagCount(tags); i < limit; i++)
 135     {
 136       wstring tag = L"<" + tagAt(tags, i) + L">";
 137
 138       if(tag == L"<*>")
 139       {
 140         result.push_back(alphabet(ANY_TAG));
 141       }
 142       else
 143       {
 144         alphabet.includeSymbol(tag);
 145         result.push_back(alphabet(tag));
 146       }
 147     }
 148   }
 149 }
 150
 151 void
 152 PatternList::insertIntoSequence(int const id, wstring const &lemma,
 153                                 wstring const &tags)
 154 {
 155   sequence_id = id;
 156
 157   if(sequence_data.size() == 0)
 158   {
 159     vector<int> new_vector;
 160     insertOutOfSequence(lemma, tags, new_vector);
 161     sequence_data.push_back(new_vector);
 162   }
 163   else
 164   {
 165     list<vector<int> >::iterator it    = sequence_data.begin();
 166     list<vector<int> >::iterator limit = sequence_data.end();
 167     for(; it != limit; it++)
 168     {
 169       it->push_back(L'+');
 170       insertOutOfSequence(lemma, tags, *it);
 171     }
 172   }
 173 }
 174
 175 void
 176 PatternList::insert(int const id, wstring const &lemma, wstring const &tags)
 177 {
 178   if(!sequence)
 179   {
 180     vector<int> local;
 181     insertOutOfSequence(lemma, tags, local);
 182     local.push_back(alphabet(QUEUE));
 183     patterns.insert(pair<int, vector<int> >(id, local));
 184   }
 185   else
 186   {
 187     insertIntoSequence(id, lemma, tags);
 188   }
 189 }
 190
 191 void
 192 PatternList::insert(int const id, int const otherid)
 193 {
 194   if(!sequence)
 195   {
 196     cerr << "Error: using labels outside of a sequence" << endl;
 197     exit(EXIT_FAILURE);
 198   }
 199
 200   sequence_id = id;
 201
 202   if(sequence_data.size() == 0)
 203   {
 204     PatternRange p = patterns.equal_range(otherid);
 205     for(; p.first != p.second; p.first++)
 206     {
 207       sequence_data.push_back(p.first->second);
 208     }
 209   }
 210   else
 211   {
 212     list<vector<int> > new_sequence_data;
 213
 214     for(list<vector<int> >::iterator it = sequence_data.begin(),
 215           limit = sequence_data.end(); it != limit; it++)
 216     {
 217       for(PatternRange p = patterns.equal_range(otherid);
 218           p.first != p.second; p.first++)
 219       {
 220         vector<int> temp = *it;
 221         temp.push_back(L'+');
 222         temp.insert(temp.end(), (p.first->second).begin(),
 223                     (p.first->second).end());
 224         new_sequence_data.push_back(temp);
 225       }
 226     }
 227
 228     sequence_data = new_sequence_data;
 229   }
 230 }
 231
 232 int
 233 PatternList::tagCount(wstring const &tags)
 234 {
 235   int count = 0;
 236
 237   for(unsigned int i = 0, limit = tags.size(); i < limit; i++)
 238   {
 239     if(i == 0)
 240     {
 241       count++;
 242     }
 243     else if(tags[i] == L'.')
 244     {
 245       count++;
 246     }
 247   }
 248
 249   return count;
 250 }
 251
 252 wstring
 253 PatternList::tagAt(wstring const &tags, int const index)
 254 {
 255   int start = 0;
 256   int end = 0;
 257   int count = 0;
 258
 259   for(unsigned int i = 0, limit = tags.size(); i < limit; i++)
 260   {
 261     if(tags[i] == L'.')
 262     {
 263       count++;
 264       if(end == 0)
 265       {
 266         start = 0;
 267       }
 268       else
 269       {
 270         start = end + 1;
 271       }
 272       end = i;
 273     }
 274     if(count == index + 1)
 275     {
 276       return tags.substr(start, end - start);
 277     }
 278   }
 279
 280   if(index > count)
 281   {
 282     return L"";
 283   }
 284   if(end != 0)
 285   {
 286     return tags.substr(end + 1);
 287   }
 288   else
 289   {
 290     return tags.substr(end);
 291   }
 292 }
 293
 294 PatternStore const &
 295 PatternList::getPatterns()
 296 {
 297   return patterns;
 298 }
 299
 300 void
 301 PatternList::buildTransducer()
 302 {
 303   for(PatternStore::const_iterator it = patterns.begin(), limit = patterns.end();
 304       it != limit; it++)
 305   {
 306     int state = transducer.getInitial();
 307     int prevstate = -1;
 308     for(unsigned int i = 0, limit2 = it->second.size(); i != limit2; i++)
 309     {
 310       int const val = it->second[i];
 311       if(alphabet(ANY_CHAR) == val || alphabet(ANY_TAG) == val)
 312       {
 313         state = transducer.insertSingleTransduction(val, state);
 314         if(prevstate != -1)
 315         {
 316           transducer.linkStates(prevstate, state, val);
 317           prevstate = -1;
 318         }
 319         transducer.linkStates(state, state, val);
 320       }
 321       else if(alphabet(QUEUE) == val)
 322       {
 323         if(prevstate != -1)
 324         {
 325           // ignore second (and next) possible consecutive queues
 326           continue;
 327         }
 328
 329         // optional queue
 330         prevstate = state;
 331         state = transducer.insertSingleTransduction(static_cast<int>(L'_'), state);
 332         transducer.linkStates(prevstate, state, static_cast<int>(L' '));
 333         transducer.linkStates(prevstate, state, static_cast<int>(L'#'));
 334         transducer.linkStates(state, state, alphabet(ANY_CHAR));
 335       }
 336       else
 337       {
 338         state = transducer.insertSingleTransduction(val, state);
 339         if(prevstate != -1)
 340         {
 341           transducer.linkStates(prevstate, state, val);
 342           prevstate = -1;
 343         }
 344       }
 345     }
 346     if(prevstate != -1)
 347     {
 348       if(!transducer.isFinal(prevstate))
 349       {
 350         transducer.setFinal(prevstate);
 351         final_type[prevstate] = it->first;
 352       }
 353       prevstate = -1;
 354     }
 355     if(!transducer.isFinal(state))
 356     {
 357       transducer.setFinal(state);
 358       final_type[state] = it->first;
 359     }
 360   }
 361 }
 362
 363 void
 364 PatternList::write(FILE *output)
 365 {
 366   alphabet.write(output);
 367   wstring const tagger_name = L"tagger";
 368
 369   Compression::multibyte_write(1, output);
 370   Compression::wstring_write(tagger_name, output);
 371   transducer.write(output, alphabet.size());
 372
 373   Compression::multibyte_write(final_type.size(), output);
 374
 375   for(map<int, int>::const_iterator it = final_type.begin(), limit = final_type.end();
 376       it != limit; it++)
 377   {
 378     Compression::multibyte_write(it->first, output);
 379     Compression::multibyte_write(it->second, output);
 380   }
 381 }
 382
 383 void
 384 PatternList::read(FILE *input)
 385 {
 386   sequence = false;
 387   final_type.clear();
 388
 389   alphabet.read(input);
 390   if(Compression::multibyte_read(input) == 1)
 391   {
 392     wstring mystr = Compression::wstring_read(input);
 393     transducer.read(input, alphabet.size());
 394
 395     int finalsize = Compression::multibyte_read(input);
 396     for(; finalsize != 0; finalsize--)
 397     {
 398       int key = Compression::multibyte_read(input);
 399       final_type[key] = Compression::multibyte_read(input);
 400     }
 401   }
 402 }
 403
 404 MatchExe *
 405 PatternList::newMatchExe()
 406 {
 407   return new MatchExe(transducer, final_type);
 408 }
 409
 410 Alphabet &
 411 PatternList::getAlphabet()
 412 {
 413   return alphabet;
 414 }