mTVolca.cpp

   1 // Implementation of mTVolca methods
   2 // Copyright © 2009 The University of Chicago
   3 #include "mTVolca.h"
   4
   5 #include <Q3FileDialog>
   6 #include <QMessageBox>
   7 #include <Q3TextStream>
   8 #include <QFile>
   9 #include <QString>
  10 #include "Typedefs.h"
  11
  12 //////////////////////////////////////////////////////////////////////
  13 // Construction/Destruction
  14 //////////////////////////////////////////////////////////////////////
  15
  16 mTVolca::mTVolca(cMT* myMT, QString projectDirectory)
  17 {
  18         m_myMT = myMT;
  19         m_projectDirectory = projectDirectory;
  20
  21         m_language1TotalWords =0;
  22         m_language2TotalWords =0;
  23
  24         m_language1FileName = "";
  25         m_language2FileName = "";
  26
  27         m_countOfSentences = 0;
  28 }
  29
  30
  31 mTVolca::~mTVolca()
  32 {
  33         // To do clean
  34
  35 }
  36
  37 void    mTVolca::initVolList()
  38 {
  39
  40         m_language1Words.clear();
  41         m_language2Words.clear();
  42         m_language1WordIndex.clear();
  43         m_language2WordIndex.clear();
  44
  45
  46         QString                                 language1FileName;
  47         QString                                 language2FileName;
  48         QFile*                                  language1File;
  49         QFile*                                  language2File;
  50         QString                                 oneWord;
  51         int                                             wordIndex;
  52         StringToInt::iterator   StringToIntIt;
  53
  54
  55         language1FileName = Q3FileDialog::getOpenFileName( m_projectDirectory,
  56                                                                                                    "Text File (*.txt);;All files (*.*);;Corpus File (*.corpus)",
  57                                                                                                    NULL,
  58                                                                                                    "open file dialog",
  59                                                                                                    "Choose Language1 corpus" );
  60
  61         language2FileName = Q3FileDialog::getOpenFileName( m_projectDirectory,
  62                                                                                                    "Text File (*.txt);;All files (*.*);;Corpus File (*.corpus)",
  63                                                                                                    NULL,
  64                                                                                                    "open file dialog",
  65                                                                                                    "Choose Language2 corpus" );
  66
  67
  68         m_language1FileName = language1FileName;
  69         m_language2FileName = language2FileName;
  70
  71         language1File = new QFile( language1FileName );
  72     if ( language1File->open( QIODevice::ReadOnly ) )
  73         {
  74                 Q3TextStream stream( language1File );
  75                 stream.setEncoding( Q3TextStream::Locale );
  76
  77                 while( !stream.atEnd() )
  78                 {
  79                         stream >> oneWord;
  80
  81                         // a little filtering
  82                         if (oneWord.length() >0 && oneWord != QString(".") && oneWord != QString(","))
  83                         {
  84                                 m_language1Words.insert(oneWord, 0);
  85                         }
  86                 }
  87
  88
  89                 m_language1TotalWords = m_language1Words.size();
  90
  91                 wordIndex =0;
  92                 for ( StringToIntIt = m_language1Words.begin(); StringToIntIt != m_language1Words.end(); StringToIntIt++ )
  93                 {
  94                         oneWord = StringToIntIt.key();
  95                         m_language1Words[oneWord] = wordIndex;
  96                         m_language1WordIndex.insert(wordIndex, oneWord);
  97                         wordIndex++;
  98                 }
  99
 100                 StringToIntIt--;
 101                 wordIndex = StringToIntIt.data();
 102
 103                 //QMessageBox::information ( NULL, "Linguistica : MT Model1", QString("The total is %1 and last index is %2.").arg(m_language1TotalWords).arg(wordIndex), "OK" );
 104
 105                 language1File ->close();
 106         }
 107         else
 108         {
 109                 QMessageBox::information ( NULL, "Linguistica : MT Model1", "Can't Open Language 1 Corpus !", "OK" );
 110                 return;
 111         }
 112
 113         language2File = new QFile( language2FileName );
 114     if ( language2File->open( QIODevice::ReadOnly ) )
 115         {
 116                 Q3TextStream stream( language2File );
 117                 stream.setEncoding( Q3TextStream::Locale );
 118
 119                 while( !stream.atEnd() )
 120                 {
 121                         stream >> oneWord;
 122
 123                         // a little filtering
 124                         if (oneWord.length() >0 && oneWord != QString(".") && oneWord != QString(","))
 125                         {
 126                                 m_language2Words.insert(oneWord, 0);
 127                         }
 128                 }
 129
 130                 m_language2TotalWords = m_language2Words.size();
 131
 132                 wordIndex =0;
 133                 for ( StringToIntIt = m_language2Words.begin(); StringToIntIt != m_language2Words.end(); StringToIntIt++ )
 134                 {
 135                         oneWord = StringToIntIt.key();
 136                         m_language2Words[oneWord] = wordIndex;
 137                         m_language2WordIndex.insert(wordIndex, oneWord);
 138                         wordIndex++;
 139                 }
 140
 141                 StringToIntIt--;
 142                 wordIndex = StringToIntIt.data();
 143
 144                 //QMessageBox::information ( NULL, "Linguistica : MT Model1", QString("The total is %1 and last index is %2.").arg(m_language2TotalWords).arg(wordIndex), "OK" );
 145
 146                 language2File ->close();
 147
 148         }
 149         else
 150         {
 151                 QMessageBox::information ( NULL, "Linguistica : MT Model1", "Can't Open Language 2 Corpus !", "OK" );
 152                 return;
 153         }
 154
 155
 156 }
 157
 158
 159 void    mTVolca::readSentences()
 160 {
 161
 162         int                                                     countOfLanguage1Sentence,countOfLanguage2Sentence;
 163         QFile*                                          language1File;
 164         QFile*                                          language2File;
 165         QString                                         oneLine;
 166         QString                                         oneWord;
 167         int                                                     indexInsideSentence;
 168         int                                                     idOfTheWord;
 169         IntToInt*                                       oneSentence;
 170
 171
 172
 173         countOfLanguage1Sentence =0;
 174         language1File = new QFile( m_language1FileName );
 175     if ( language1File->open( QIODevice::ReadOnly ) )
 176         {
 177                 Q3TextStream stream( language1File );
 178                 stream.setEncoding( Q3TextStream::Locale );
 179
 180                 while( !stream.atEnd() )
 181                 {
 182                         oneLine = stream.readLine();
 183
 184
 185                         Q3TextStream lineStream( &oneLine, QIODevice::ReadOnly );
 186
 187                         indexInsideSentence = 0;
 188                         oneSentence = new IntToInt();
 189                         while ( ! lineStream.atEnd())
 190                         {
 191                                 lineStream >> oneWord;
 192
 193                                 if (oneWord.length() >0 && oneWord != QString(".") && oneWord != QString(","))
 194                                 {
 195                                         idOfTheWord = m_language1Words[oneWord];
 196                                         oneSentence ->insert(indexInsideSentence, idOfTheWord);
 197                                         indexInsideSentence++;
 198                                 }
 199                         }
 200
 201                         if ( indexInsideSentence >= 1)
 202                         {
 203                                 m_language1Sentences.insert(countOfLanguage1Sentence,oneSentence);
 204                                 countOfLanguage1Sentence++;
 205                         }
 206                         else
 207                         {
 208                                 delete oneSentence;
 209                         }
 210                 }
 211
 212         }
 213         else
 214         {
 215                 QMessageBox::information ( NULL, "Linguistica : MT Model1", "Can't Open Language 1 Corpus !", "OK" );
 216                 return;
 217         }
 218
 219
 220         countOfLanguage2Sentence =0;
 221         language2File = new QFile( m_language2FileName );
 222     if ( language2File->open( QIODevice::ReadOnly ) )
 223         {
 224                 Q3TextStream stream( language2File );
 225                 stream.setEncoding( Q3TextStream::Locale );
 226
 227                 while( !stream.atEnd() )
 228                 {
 229                         oneLine = stream.readLine();
 230
 231
 232                         Q3TextStream lineStream( &oneLine, QIODevice::ReadOnly );
 233
 234                         indexInsideSentence = 0;
 235                         oneSentence = new IntToInt();
 236                         while ( ! lineStream.atEnd())
 237                         {
 238                                 lineStream >> oneWord;
 239
 240                                 if (oneWord.length() >0 && oneWord != QString(".") && oneWord != QString(","))
 241                                 {
 242                                         idOfTheWord = m_language2Words[oneWord];
 243                                         oneSentence ->insert(indexInsideSentence, idOfTheWord);
 244                                         indexInsideSentence++;
 245                                 }
 246                         }
 247
 248                         if ( indexInsideSentence >= 1)
 249                         {
 250                                 m_language2Sentences.insert(countOfLanguage2Sentence,oneSentence);
 251                                 countOfLanguage2Sentence++;
 252                         }
 253                         else
 254                         {
 255                                 QMessageBox::information ( NULL, "Linguistica : MT Model1 Error", QString("At Sentence %1").arg(countOfLanguage2Sentence), "OK" );
 256                                 delete oneSentence;
 257                         }
 258                 }
 259
 260         }
 261         else
 262         {
 263                 QMessageBox::information ( NULL, "Linguistica : MT Model1", "Can't Open Language 2 Corpus !", "OK" );
 264                 return;
 265         }
 266
 267
 268         if ( countOfLanguage1Sentence != countOfLanguage2Sentence )
 269         {
 270                 QMessageBox::information ( NULL, "Linguistica : MT Model1", "Language1 Sentence# != Language2 Sentence#!", "OK" );
 271                 // To do  Clean sentences
 272                 return;
 273         }
 274
 275         m_countOfSentences = countOfLanguage1Sentence;
 276
 277
 278         //QMessageBox::information ( NULL, "Linguistica : MT Model1", QString("Read In %1 sentences !").arg(m_countOfSentences), "OK" );
 279
 280
 281         // Debug
 282         /*
 283         QString                                                         oneSentenceStr;
 284         QString                                                         oneVolcaWord;
 285         QString                                                         twoVolcaWord;
 286         int                                                                     oneWordId;
 287         int                                                                     count;
 288         int                                                                     i;
 289
 290
 291         oneSentence = m_language1Sentences[1];
 292         count = oneSentence ->size();
 293
 294
 295         oneSentenceStr = "";
 296         for ( i=0; i < count; i++)
 297         {
 298                 oneWordId = (*oneSentence)[i];
 299                 oneSentenceStr += QString("%1-").arg(oneWordId);
 300         }
 301
 302
 303         QMessageBox::information ( NULL, "Linguistica : MT Model1", "Lan1 Sentence 1 is: " +  oneSentenceStr, "OK" );
 304
 305         oneSentence = m_language2Sentences[1];
 306         count = oneSentence ->size();
 307
 308
 309         oneSentenceStr = "";
 310         for ( i=0; i < count; i++)
 311         {
 312                 oneWordId = (*oneSentence)[i];
 313                 oneSentenceStr += QString("%1-").arg(oneWordId);
 314         }
 315
 316         QMessageBox::information ( NULL, "Linguistica : MT Model1", "Lan2 Sentence 1 is: " +  oneSentenceStr, "OK" );
 317         */
 318
 319 }
 320
 321 void    mTVolca::setFastSearchPairsForT()
 322 {
 323         QString                                                         fastSearchPairs;
 324         int                                                                     i, l, m;
 325         int                                                                     idOfOneLanguage1Word;
 326         int                                                                     idOfOneLanguage2Word;
 327         IntToInt*                                                       oneLan1Sentence;
 328         IntToInt*                                                       oneLan2Sentence;
 329         IntToDouble*                                            lan2WordsForOneLan1Word;
 330         IntToDouble*                                            softCountLan2WordsForOneLan1Word;
 331
 332
 333
 334         for ( i=0; i < m_countOfSentences; i++)
 335         {
 336                 oneLan1Sentence = m_language1Sentences[i];
 337                 oneLan2Sentence = m_language2Sentences[i];
 338
 339                 for ( l=0; l< static_cast <int> (oneLan1Sentence ->size() ); l++)
 340                 {
 341                         idOfOneLanguage1Word = (*oneLan1Sentence)[l];
 342
 343                         if ( m_fastWordsPairs.contains(idOfOneLanguage1Word))
 344                         {
 345                                 lan2WordsForOneLan1Word = m_fastWordsPairs[idOfOneLanguage1Word];
 346                         }
 347                         else
 348                         {
 349                                 lan2WordsForOneLan1Word = new IntToDouble();
 350                                 m_fastWordsPairs.insert(idOfOneLanguage1Word, lan2WordsForOneLan1Word);
 351                         }
 352
 353                         if ( m_fastWordsSoftCounts.contains(idOfOneLanguage1Word))
 354                         {
 355                                 softCountLan2WordsForOneLan1Word = m_fastWordsSoftCounts[idOfOneLanguage1Word];
 356                         }
 357                         else
 358                         {
 359                                 softCountLan2WordsForOneLan1Word = new IntToDouble();
 360                                 m_fastWordsSoftCounts.insert(idOfOneLanguage1Word, softCountLan2WordsForOneLan1Word);
 361                         }
 362
 363                         for ( m=0; m < static_cast <int> ( oneLan2Sentence ->size() ); m++)
 364                         {
 365                                 idOfOneLanguage2Word = (*oneLan2Sentence)[m];
 366
 367                                 lan2WordsForOneLan1Word ->insert(idOfOneLanguage2Word, 0.0);
 368                                 softCountLan2WordsForOneLan1Word ->insert(idOfOneLanguage2Word, 0.0);
 369                         }
 370                 }
 371
 372         }
 373
 374         QMessageBox::information ( NULL, "Linguistica : MT Model1", "Finished Indexing the Word Pairs", "OK" );
 375
 376 }
 377
 378
 379 void    mTVolca::clearSentenceViterbiAlignment()
 380 {
 381         IntToIntToInt::iterator                 IntToIntToIntIt;
 382         IntToInt*                                               oneAlignment;
 383
 384         for ( IntToIntToIntIt = m_sentenceAlignments.begin(); IntToIntToIntIt != m_sentenceAlignments.end(); IntToIntToIntIt++)
 385         {
 386                 oneAlignment = IntToIntToIntIt.data();
 387                 delete oneAlignment;
 388         }
 389
 390         m_sentenceAlignments.clear();
 391 }
 392