cMT.cpp

   1 // Implementation of the cMT class
   2 // Copyright © 2009 The University of Chicago
   3 #include "cMT.h"
   4
   5 #include <QMessageBox>
   6 #include <Q3TextStream>
   7 #include <QFile>
   8 #include <Q3SortedList>
   9 #include <QMap>
  10 #include "cMTModel2Norm.h"
  11 #include "cMTModel1.h"
  12 #include "mTVolca.h"
  13 #include "Typedefs.h"
  14
  15
  16 //////////////////////////////////////////////////////////////////////
  17 // Construction/Destruction
  18 //////////////////////////////////////////////////////////////////////
  19
  20 cMT::cMT(LinguisticaMainWindow* parent, QString projectDirectory)
  21         : m_parent(parent),
  22         m_projectDirectory(projectDirectory),
  23         m_Volca(),
  24         m_Model1(),
  25         m_Model2Norm(),
  26         m_MTLog("LinguisticaMTLog.txt") { }
  27
  28 cMT::~cMT()
  29 {
  30         delete m_Model2Norm;
  31         delete m_Model1;
  32         delete m_Volca;
  33 }
  34
  35 void cMT::readTrainingCorpus()
  36 {
  37         delete m_Volca;
  38         m_Volca = new mTVolca(this, m_projectDirectory);
  39
  40         m_Volca ->initVolList();
  41
  42         m_Volca ->readSentences();
  43
  44         m_Volca ->setFastSearchPairsForT();
  45
  46         QMessageBox::information(NULL, "Status", QString("Words # in language 1 is %1!").arg(m_Volca ->m_language1TotalWords), "OK");
  47         QMessageBox::information(NULL, "Status", QString("Words # in language 2 is %1!").arg(m_Volca ->m_language2TotalWords), "OK");
  48
  49 }
  50
  51 void cMT::trainModel1(int model1Iterations)
  52 {
  53         delete m_Model1;
  54         m_Model1 = new cMTModel1(this, model1Iterations);
  55
  56         m_Model1 ->initT();
  57
  58         m_Model1 ->EMLoops(m_Model1 ->m_Iterations);
  59
  60         QMessageBox::information(NULL, "Status", "Logging T After Model1...!", "OK");
  61         logTAfterModel1();
  62         QMessageBox::information(NULL, "Status", "Done logging T After Model1...!", "OK");
  63
  64 }
  65
  66
  67 void cMT::logTAfterModel1()
  68 {
  69         int                                                                     language1WordId;
  70         int                                                                     language2WordId;
  71         QString                                                         language1Word;
  72         QString                                                         language2Word;
  73         double                                                          TValue;
  74         IntToIntToDouble::iterator                      IntToIntToDoubleIt;
  75         IntToDouble*                                            oneList;
  76         IntToDouble::iterator                           IntToDoubleIt;
  77         mTSortedList                                            sortPlatForm;
  78         mTForSortingItem*                                       oneSortItem;
  79         int                                                                     outputTopLimit = 20;
  80         int                                                                     outputIndex;
  81         QFile                                                           file( m_MTLog );
  82
  83         if ( !file.open( QIODevice::WriteOnly | QIODevice::Append ) )
  84         {
  85                 QMessageBox::information(NULL, "Error", "Can't Open the MT Log file!", "OK");
  86                 return;
  87         }
  88
  89         Q3TextStream outf( &file );
  90
  91         outf << "******T Tables After Model1******" << endl <<endl;
  92
  93
  94         sortPlatForm.setAutoDelete(TRUE);
  95
  96         for ( IntToIntToDoubleIt = (m_Model1 ->m_T).begin(); IntToIntToDoubleIt != (m_Model1 ->m_T).end(); IntToIntToDoubleIt++)
  97         {
  98                 language1WordId = IntToIntToDoubleIt.key();
  99                 language1Word = (m_Volca ->m_language1WordIndex)[language1WordId] ;
 100                 outf << language1Word << "      :"<<endl ;
 101
 102                 oneList = IntToIntToDoubleIt.data();
 103
 104                 for (IntToDoubleIt = oneList ->begin(); IntToDoubleIt != oneList ->end(); IntToDoubleIt++)
 105                 {
 106                         language2WordId = IntToDoubleIt.key();
 107                         language2Word = (m_Volca ->m_language2WordIndex)[language2WordId] ;
 108
 109                         TValue = IntToDoubleIt.data();
 110
 111                         oneSortItem = new mTForSortingItem(language2Word, TValue, 1);
 112
 113                         sortPlatForm.append(oneSortItem);
 114                 }
 115
 116                 sortPlatForm.sort();
 117
 118                 outputIndex = 0;
 119                 for ( oneSortItem=sortPlatForm.first(); oneSortItem != 0; oneSortItem=sortPlatForm.next())
 120                 {
 121                         outputIndex++;
 122
 123                         language2Word = oneSortItem ->m_name;
 124                         TValue = oneSortItem ->m_doubleValue;
 125
 126                         outf <<"        "<< outputIndex <<" : " <<language2Word << "    " << TValue <<endl ;
 127
 128                         if ( outputIndex >= outputTopLimit) break;
 129
 130                 }
 131
 132                 sortPlatForm.clear();
 133         }
 134
 135
 136         file.close();
 137
 138 }
 139
 140
 141 void cMT::trainModel2Norm(int model2Iterations, bool getTFromModel1)
 142 {
 143         delete m_Model2Norm;
 144         m_Model2Norm = new cMTModel2Norm(this, model2Iterations, getTFromModel1);
 145
 146         m_Model2Norm ->initTandA();
 147
 148         m_Model2Norm ->EMLoops(m_Model2Norm ->m_Iterations);
 149
 150         QMessageBox::information(NULL, "Status", "Doing Viterbi for Model2Norm...", "OK");
 151
 152         m_Model2Norm ->viterbiAll();
 153
 154         QMessageBox::information(NULL, "Status", "Logging TandA After Model2Norm...!", "OK");
 155
 156         logTandAAfterModel2Norm();
 157
 158         QMessageBox::information(NULL, "Status", "Done logging TandA After Model2Norm...!", "OK");
 159
 160 }
 161
 162
 163 void cMT::logTandAAfterModel2Norm()
 164 {
 165         int                                                                     language1WordId;
 166         int                                                                     language2WordId;
 167         QString                                                         language1Word;
 168         QString                                                         language2Word;
 169         int                                                                     language1ChunkId;
 170         int                                                                     language2ChunkId;
 171         QString                                                         language1ChunkStr;
 172         QString                                                         language2ChunkStr;
 173         double                                                          TValue;
 174         double                                                          AValue;
 175         IntToIntToDouble::iterator                      IntToIntToDoubleIt;
 176         IntToDouble*                                            oneList;
 177         IntToDouble::iterator                           IntToDoubleIt;
 178         mTSortedList                                            sortPlatForm;
 179         mTForSortingItem*                                       oneSortItem;
 180         int                                                                     outputTopLimit = 20;
 181         int                                                                     outputIndex;
 182         QFile                                                           file( m_MTLog );
 183
 184         if ( !file.open( QIODevice::WriteOnly | QIODevice::Append ) )
 185         {
 186                 QMessageBox::information(NULL, "Error", "Can't Open the MT Log file!", "OK");
 187                 return;
 188         }
 189
 190         Q3TextStream outf( &file );
 191
 192         // Log T
 193         outf << "******T Tables After Model2Norm******" << endl <<endl;
 194
 195
 196         sortPlatForm.setAutoDelete(TRUE);
 197
 198         for ( IntToIntToDoubleIt = (m_Model2Norm ->m_T).begin(); IntToIntToDoubleIt != (m_Model2Norm ->m_T).end(); IntToIntToDoubleIt++)
 199         {
 200                 language1WordId = IntToIntToDoubleIt.key();
 201                 language1Word = (m_Volca ->m_language1WordIndex)[language1WordId] ;
 202                 outf << language1Word << "      :"<<endl ;
 203
 204                 oneList = IntToIntToDoubleIt.data();
 205
 206                 for (IntToDoubleIt = oneList ->begin(); IntToDoubleIt != oneList ->end(); IntToDoubleIt++)
 207                 {
 208                         language2WordId = IntToDoubleIt.key();
 209                         language2Word = (m_Volca ->m_language2WordIndex)[language2WordId] ;
 210
 211                         TValue = IntToDoubleIt.data();
 212
 213                         oneSortItem = new mTForSortingItem(language2Word, TValue, 1);
 214
 215                         sortPlatForm.append(oneSortItem);
 216                 }
 217
 218                 sortPlatForm.sort();
 219
 220                 outputIndex = 0;
 221                 for ( oneSortItem=sortPlatForm.first(); oneSortItem != 0; oneSortItem=sortPlatForm.next())
 222                 {
 223                         outputIndex++;
 224
 225                         language2Word = oneSortItem ->m_name;
 226                         TValue = oneSortItem ->m_doubleValue;
 227
 228                         outf <<"        "<< outputIndex <<" : " <<language2Word << "    " << TValue <<endl ;
 229
 230                         if ( outputIndex >= outputTopLimit) break;
 231
 232                 }
 233
 234                 sortPlatForm.clear();
 235         }
 236
 237         // Log A
 238         outf << "******A Tables After Model2Norm******" << endl <<endl;
 239
 240
 241         sortPlatForm.setAutoDelete(TRUE);
 242
 243         for ( IntToIntToDoubleIt = (m_Model2Norm ->m_A).begin(); IntToIntToDoubleIt != (m_Model2Norm ->m_A).end(); IntToIntToDoubleIt++)
 244         {
 245                 language2ChunkId = IntToIntToDoubleIt.key();
 246                 language2ChunkStr = QString("%1").arg(language2ChunkId);
 247                 outf << language2ChunkStr << "  :"<<endl ;
 248
 249                 oneList = IntToIntToDoubleIt.data();
 250
 251                 for (IntToDoubleIt = oneList ->begin(); IntToDoubleIt != oneList ->end(); IntToDoubleIt++)
 252                 {
 253                         language1ChunkId = IntToDoubleIt.key();
 254                         language1ChunkStr = QString("%1").arg(language1ChunkId);
 255
 256                         AValue = IntToDoubleIt.data();
 257
 258                         oneSortItem = new mTForSortingItem(language1ChunkStr, AValue, 1);
 259
 260                         sortPlatForm.append(oneSortItem);
 261                 }
 262
 263                 sortPlatForm.sort();
 264
 265                 outputIndex = 0;
 266                 outputTopLimit = 100;
 267                 for ( oneSortItem=sortPlatForm.first(); oneSortItem != 0; oneSortItem=sortPlatForm.next())
 268                 {
 269                         outputIndex++;
 270
 271                         language1ChunkStr = oneSortItem ->m_name;
 272                         AValue = oneSortItem ->m_doubleValue;
 273
 274                         outf <<"        " <<language1ChunkStr << "      " << AValue <<endl ;
 275
 276                         if ( outputIndex >= outputTopLimit) break;
 277
 278                 }
 279
 280                 sortPlatForm.clear();
 281         }
 282
 283
 284         // Log Viterbi Alignments after model 2
 285         outf << "******Viterbi Alignments After Model2Norm******" << endl <<endl;
 286
 287         mTVolca*                                                myVolca;
 288         int                                                             i;
 289         int                                                             l,m;
 290         IntToInt*                                               oneLan1Sentence;
 291         IntToInt*                                               oneLan2Sentence;
 292         IntToInt*                                               oneAlignment;
 293         int                                                             language1SentenceLen;
 294         int                                                             language2SentenceLen;
 295         QString                                                 oneWordStr;
 296         QString                                                 twoWordStr;
 297
 298
 299
 300         myVolca = m_Volca;
 301
 302         for ( i=0; i < myVolca ->m_countOfSentences; i++)
 303         {
 304                 outf <<"Sentence " << i << " : " << endl;
 305
 306                 oneLan1Sentence = myVolca ->m_language1Sentences[i];
 307                 oneLan2Sentence = myVolca ->m_language2Sentences[i];
 308
 309                 language1SentenceLen = oneLan1Sentence ->size();
 310                 language2SentenceLen = oneLan2Sentence ->size();
 311
 312                 // Output sentence in lan1
 313                 outf << "Sentence 1 -> ";
 314                 for ( l =0; l < language1SentenceLen; l++)
 315                 {
 316                         language1WordId = (*oneLan1Sentence)[l];
 317                         oneWordStr = (m_Volca ->m_language1WordIndex)[language1WordId] ;
 318                         outf << l << " : " <<  oneWordStr << "  ";
 319                 }
 320
 321                 outf << endl;
 322
 323                 // Output sentence in lan2
 324                 outf << "Sentence 2 -> ";
 325                 for ( m =0; m < language2SentenceLen; m++)
 326                 {
 327                         language2WordId = (*oneLan2Sentence)[m];
 328                         oneWordStr = (m_Volca ->m_language2WordIndex)[language2WordId] ;
 329                         outf << m << " : " <<  oneWordStr << "  ";
 330                 }
 331
 332                 outf << endl;
 333
 334                 // Output the alignment
 335                 outf << "Alignments are language2Word -> language1Word" << endl;
 336                 oneAlignment = (myVolca ->m_sentenceAlignments)[i];
 337
 338                 for ( m=0; m< language2SentenceLen; m++)
 339                 {
 340                         language2WordId = (*oneLan2Sentence)[m];
 341                         twoWordStr = (m_Volca ->m_language2WordIndex)[language2WordId] ;
 342
 343                         language1WordId = (*oneLan1Sentence)[(*oneAlignment)[m]];
 344                         oneWordStr = (m_Volca ->m_language1WordIndex)[language1WordId] ;
 345
 346                         outf << "       " << twoWordStr << " --> " << oneWordStr << endl;
 347
 348                 }
 349
 350         }
 351
 352         file.close();
 353
 354 }
 355