cMTModel2Norm.cpp

   1 // Implementation of the cMTModel2Norm class
   2 // Copyright © 2009 The University of Chicago
   3 #include "cMTModel2Norm.h"
   4
   5 #include <QMessageBox>
   6 #include "cMTModel1.h"
   7 #include "mTVolca.h"
   8 #include "cMT.h"
   9
  10 //////////////////////////////////////////////////////////////////////
  11 // Construction/Destruction
  12 //////////////////////////////////////////////////////////////////////
  13
  14 cMTModel2Norm::cMTModel2Norm(cMT* myMT, int Iterations, bool fromModel1)
  15         : m_myMT(myMT),
  16         m_Iterations(Iterations),
  17         m_sentenceNormChunk(10),
  18         m_getTfromModel1(fromModel1),
  19         m_T(), m_A(),
  20         m_softCountOfT(), m_softCountOfA(),
  21         m_lamdaT(), m_lamdaA() { }
  22
  23 cMTModel2Norm::~cMTModel2Norm()
  24 {
  25         // XXX. not clear who owns what, so there are definitely some
  26         // leaks here.
  27 }
  28
  29 void            cMTModel2Norm::initTandA()
  30 {
  31         mTVolca*                                                                myVolca;
  32         int                                                                             totalAssociatedLanguage2Words;
  33         IntToIntToDouble::iterator                              IntToIntToDoubleIt;
  34         IntToDouble::iterator                                   IntToDoubleIt;
  35         IntToDouble*                                                    oneListForOneLanguage1Word;
  36         double                                                                  uniformProb;
  37         int                                                                             key;
  38         IntToDouble*                                                    oneListForOneLanuage2Chunk;
  39         IntToDouble*                                                    oneSoftCountListForOneLanuage2Chunk;
  40         int                                                                             i,j;
  41
  42
  43
  44         myVolca = m_myMT ->m_Volca;
  45
  46
  47         // Init m_T
  48         if ( !m_getTfromModel1)
  49         {
  50                 m_T = myVolca ->m_fastWordsPairs;
  51
  52                 for ( IntToIntToDoubleIt = m_T.begin(); IntToIntToDoubleIt != m_T.end();IntToIntToDoubleIt++)
  53                 {
  54                         oneListForOneLanguage1Word = IntToIntToDoubleIt.data();
  55
  56                         totalAssociatedLanguage2Words = oneListForOneLanguage1Word ->size();
  57
  58                         uniformProb = 1.0 / totalAssociatedLanguage2Words;
  59
  60                         for ( IntToDoubleIt = oneListForOneLanguage1Word ->begin(); IntToDoubleIt != oneListForOneLanguage1Word ->end();  IntToDoubleIt++)
  61                         {
  62                                 key = IntToDoubleIt.key();
  63                                 (*oneListForOneLanguage1Word)[key] = uniformProb;
  64                         }
  65                 }
  66         }
  67         else
  68         {
  69                 m_T = m_myMT ->m_Model1 ->m_T;
  70         }
  71
  72
  73         // init m_A and m_softcountofA
  74         for ( j=0; j< m_sentenceNormChunk; j++)
  75         {
  76                 oneListForOneLanuage2Chunk = new IntToDouble();
  77                 oneSoftCountListForOneLanuage2Chunk = new IntToDouble();
  78                 m_A.insert(j, oneListForOneLanuage2Chunk);
  79                 m_softCountOfA.insert(j, oneSoftCountListForOneLanuage2Chunk);
  80
  81                 uniformProb = 1.0/ m_sentenceNormChunk;
  82
  83                 for ( i=0; i< m_sentenceNormChunk; i++)
  84                 {
  85                         oneListForOneLanuage2Chunk ->insert(i, uniformProb);
  86                         oneSoftCountListForOneLanuage2Chunk ->insert(i, 0.0);
  87                 }
  88         }
  89
  90
  91         // init m_softcountofT
  92         m_softCountOfT = myVolca ->m_fastWordsSoftCounts;
  93
  94         for ( IntToIntToDoubleIt = m_softCountOfT.begin(); IntToIntToDoubleIt != m_softCountOfT.end();IntToIntToDoubleIt++)
  95         {
  96                 oneListForOneLanguage1Word = IntToIntToDoubleIt.data();
  97
  98                 for ( IntToDoubleIt = oneListForOneLanguage1Word ->begin(); IntToDoubleIt != oneListForOneLanguage1Word ->end();  IntToDoubleIt++)
  99                 {
 100                         IntToDoubleIt.data() = 0.0;
 101                 }
 102         }
 103
 104
 105
 106         QMessageBox::information ( NULL, "Linguistica : MT Model2Norm", "Finished InitTandA", "OK" );
 107 }
 108
 109
 110
 111 void                    cMTModel2Norm::EMLoops(int numberOfIterations)
 112 {
 113         int                                                     loopI;
 114
 115         for (loopI =0; loopI < numberOfIterations; loopI++)
 116         {
 117                 // E step
 118                 EStep();
 119
 120                 //QMessageBox::information ( NULL, "Linguistica : MT Model1", "Finished E-Step", "OK" );
 121
 122                 // M step
 123                 MStep();
 124
 125                 //QMessageBox::information ( NULL, "Linguistica : MT Model1", "Finished M-Step", "OK" );
 126
 127                 // Clear softcouts for T
 128                 clearSoftCounts();
 129         }
 130
 131         // Release softcountT memory
 132         releaseSoftCounts();
 133 }
 134
 135 void                    cMTModel2Norm::clearSoftCounts()
 136 {
 137         IntToIntToDouble::iterator                              IntToIntToDoubleIt;
 138         IntToDouble*                                                    oneList;
 139         IntToDouble*                                                    oneListForOneLanuage2Chunk;
 140         IntToDouble::iterator                                   IntToDoubleIt;
 141         int                                                                             key;
 142         int                                                                             i, j;
 143
 144         for ( IntToIntToDoubleIt = m_softCountOfT.begin(); IntToIntToDoubleIt != m_softCountOfT.end(); IntToIntToDoubleIt++)
 145         {
 146                 oneList = IntToIntToDoubleIt.data();
 147
 148                 for (IntToDoubleIt = oneList ->begin(); IntToDoubleIt != oneList ->end(); IntToDoubleIt++)
 149                 {
 150                         key = IntToDoubleIt.key();
 151                         (*oneList)[key] = 0.0;
 152                 }
 153         }
 154
 155         for ( j=0; j< m_sentenceNormChunk; j++)
 156         {
 157                 oneListForOneLanuage2Chunk = m_softCountOfA[j];
 158                 for ( i=0; i< m_sentenceNormChunk; i++)
 159                 {
 160                         (*oneListForOneLanuage2Chunk)[i] = 0.0;
 161                 }
 162         }
 163
 164
 165 }
 166
 167 void                    cMTModel2Norm::EStep()
 168 {
 169         mTVolca*                                                myVolca;
 170         int                                                             i;
 171
 172         myVolca = m_myMT ->m_Volca;
 173
 174         m_lamdaT.clear();
 175         m_lamdaA.clear();
 176
 177         for ( i=0; i < myVolca ->m_countOfSentences; i++)
 178         {
 179                 addSoftCountOfTandA(i);
 180         }
 181 }
 182
 183
 184 void                    cMTModel2Norm::MStep()
 185 {
 186         IntToIntToDouble::iterator                              IntToIntToDoubleIt;
 187         IntToDouble*                                                    oneList;
 188         IntToDouble::iterator                                   IntToDoubleIt;
 189         int                                                                             language1Id;
 190         int                                                                             language2Id;
 191         int                                                                             language1chunkId;
 192         int                                                                             language2chunkId;
 193         double                                                                  oneTotalSoftCount;
 194         double                                                                  oneLamda;
 195
 196
 197         // Mstep for T
 198         for ( IntToIntToDoubleIt = m_softCountOfT.begin(); IntToIntToDoubleIt != m_softCountOfT.end(); IntToIntToDoubleIt++)
 199         {
 200                 language1Id = IntToIntToDoubleIt.key();
 201                 oneLamda = m_lamdaT[language1Id] ;
 202                 oneList = IntToIntToDoubleIt.data();
 203
 204
 205                 for (IntToDoubleIt = oneList ->begin(); IntToDoubleIt != oneList ->end(); IntToDoubleIt++)
 206                 {
 207                         language2Id = IntToDoubleIt.key();
 208                         oneTotalSoftCount = IntToDoubleIt.data();
 209                         (*(m_T[language1Id]))[language2Id] = oneTotalSoftCount / oneLamda ;
 210                 }
 211         }
 212
 213         // Mstep for A
 214         for ( IntToIntToDoubleIt = m_softCountOfA.begin(); IntToIntToDoubleIt != m_softCountOfA.end(); IntToIntToDoubleIt++)
 215         {
 216                 language2chunkId = IntToIntToDoubleIt.key();
 217                 oneLamda = m_lamdaA[language2chunkId] ;
 218                 oneList = IntToIntToDoubleIt.data();
 219
 220                 for (IntToDoubleIt = oneList ->begin(); IntToDoubleIt != oneList ->end(); IntToDoubleIt++)
 221                 {
 222                         language1chunkId = IntToDoubleIt.key();
 223                         oneTotalSoftCount = IntToDoubleIt.data();
 224                         (*(m_A[language2chunkId]))[language1chunkId] = oneTotalSoftCount / oneLamda ;
 225                 }
 226         }
 227
 228 }
 229
 230 void                    cMTModel2Norm::addSoftCountOfTandA(int sentenceId)
 231 {
 232         mTVolca*                                                myVolca;
 233         double                                                  oneT;
 234         double                                                  oneA;
 235         double                                                  oneCount;
 236         int                                                             language1SentenceLen;
 237         int                                                             language2SentenceLen;
 238         int                                                             language1ChunkId;
 239         int                                                             language2ChunkId;
 240         double                                                  deNumerator;
 241         int                                                             l,m;
 242         int                                                             language1WordId;
 243         int                                                             language2WordId;
 244         IntToInt*                                               oneLan1Sentence;
 245         IntToInt*                                               oneLan2Sentence;
 246
 247
 248         // add softcount of T
 249         myVolca = m_myMT ->m_Volca;
 250
 251         oneLan1Sentence = myVolca ->m_language1Sentences[sentenceId];
 252         oneLan2Sentence = myVolca ->m_language2Sentences[sentenceId];
 253
 254         language1SentenceLen = oneLan1Sentence ->size();
 255         language2SentenceLen = oneLan2Sentence ->size();
 256
 257         for (m = 0; m < oneLan2Sentence->size(); ++m) {
 258                 language2WordId = (*oneLan2Sentence)[m];
 259                 language2ChunkId = static_cast<int>(
 260                         double(m) / language2SentenceLen * m_sentenceNormChunk);
 261
 262                 Q_ASSERT(language2ChunkId >=0 && language2ChunkId <m_sentenceNormChunk);
 263
 264
 265                 deNumerator =0;
 266
 267                 for (l = 0; l < oneLan1Sentence->size(); ++l) {
 268                         language1WordId = (*oneLan1Sentence)[l];
 269                         language1ChunkId = static_cast<int>(
 270                                 double(l) / language1SentenceLen *
 271                                 m_sentenceNormChunk);
 272
 273                         Q_ASSERT(language1ChunkId >=0 && language1ChunkId <m_sentenceNormChunk);
 274
 275                         oneT = (*(m_T[language1WordId]))[language2WordId];
 276                         oneA = (*(m_A[language2ChunkId]))[language1ChunkId];
 277                         deNumerator += oneT*oneA;
 278                 }
 279
 280                 for (l = 0; l < oneLan1Sentence->size(); ++l) {
 281                         language1WordId = (*oneLan1Sentence)[l];
 282                         language1ChunkId = static_cast<int>(
 283                                 double(l) / language1SentenceLen *
 284                                 m_sentenceNormChunk);
 285
 286                         oneA = (*(m_A[language2ChunkId]))[language1ChunkId];
 287                         oneT = (*(m_T[language1WordId]))[language2WordId] ;
 288                         oneCount = oneT*oneA/ deNumerator;
 289
 290                         (*(m_softCountOfT[language1WordId]))[language2WordId] += oneCount;
 291
 292                         if ( m_lamdaT.contains(language1WordId))
 293                         {
 294                                 m_lamdaT[language1WordId] += oneCount;
 295                         }
 296                         else
 297                         {
 298                                 m_lamdaT.insert(language1WordId, oneCount);
 299                         }
 300
 301                         // add softcount of A
 302                         (*(m_softCountOfA[language2ChunkId]))[language1ChunkId] += oneCount;
 303
 304                         if ( m_lamdaA.contains(language2ChunkId))
 305                         {
 306                                 m_lamdaA[language2ChunkId] += oneCount;
 307                         }
 308                         else
 309                         {
 310                                 m_lamdaA.insert(language2ChunkId, oneCount);
 311                         }
 312                 }
 313         }
 314 }
 315
 316 void                    cMTModel2Norm::releaseSoftCounts()
 317 {
 318         IntToIntToDouble::iterator                              IntToIntToDoubleIt;
 319         IntToDouble*                                                    oneList;
 320
 321         for ( IntToIntToDoubleIt = m_softCountOfT.begin(); IntToIntToDoubleIt != m_softCountOfT.end(); IntToIntToDoubleIt++)
 322         {
 323                 oneList = IntToIntToDoubleIt.data();
 324
 325                 delete oneList;
 326         }
 327
 328
 329         for ( IntToIntToDoubleIt = m_softCountOfA.begin(); IntToIntToDoubleIt != m_softCountOfA.end(); IntToIntToDoubleIt++)
 330         {
 331                 oneList = IntToIntToDoubleIt.data();
 332
 333                 delete oneList;
 334         }
 335
 336 }
 337
 338 void                    cMTModel2Norm::viterbiAll()
 339 {
 340         mTVolca*                                                myVolca;
 341         int                                                             i;
 342
 343         myVolca = m_myMT ->m_Volca;
 344
 345         myVolca ->clearSentenceViterbiAlignment();
 346
 347         for ( i=0; i < myVolca ->m_countOfSentences; i++)
 348         {
 349                 viterbi(i);
 350         }
 351
 352 }
 353
 354 void                    cMTModel2Norm::viterbi(int sentenceId)
 355 {
 356         mTVolca*                                                myVolca;
 357         double                                                  oneT;
 358         double                                                  oneA;
 359         double                                                  oneCount;
 360         double                                                  bestCount;
 361         int                                                             bestAlignmentId;
 362         int                                                             language1SentenceLen;
 363         int                                                             language2SentenceLen;
 364         int                                                             language1ChunkId;
 365         int                                                             language2ChunkId;
 366         int                                                             l,m;
 367         int                                                             language1WordId;
 368         int                                                             language2WordId;
 369         IntToInt*                                               oneLan1Sentence;
 370         IntToInt*                                               oneLan2Sentence;
 371         IntToInt*                                               oneAlignment;
 372
 373
 374         // add softcount of T
 375         myVolca = m_myMT ->m_Volca;
 376
 377         oneAlignment = new IntToInt();
 378         myVolca ->m_sentenceAlignments.insert(sentenceId, oneAlignment);
 379
 380         oneLan1Sentence = myVolca ->m_language1Sentences[sentenceId];
 381         oneLan2Sentence = myVolca ->m_language2Sentences[sentenceId];
 382
 383         language1SentenceLen = oneLan1Sentence ->size();
 384         language2SentenceLen = oneLan2Sentence ->size();
 385
 386         for (m = 0; m < language2SentenceLen; ++m) {
 387                 language2WordId = (*oneLan2Sentence)[m];
 388                 language2ChunkId = static_cast<int>(
 389                         double(m) / language2SentenceLen *
 390                         m_sentenceNormChunk);
 391
 392                 Q_ASSERT(language2ChunkId >=0 && language2ChunkId <m_sentenceNormChunk);
 393
 394                 bestAlignmentId = -1;
 395                 bestCount = 0.0;
 396
 397                 for (l = 0; l < language1SentenceLen; ++l) {
 398                         language1WordId = (*oneLan1Sentence)[l];
 399                         language1ChunkId = static_cast<int>(
 400                                 double(l) / language1SentenceLen *
 401                                 m_sentenceNormChunk);
 402
 403                         oneA = (*(m_A[language2ChunkId]))[language1ChunkId];
 404                         oneT = (*(m_T[language1WordId]))[language2WordId] ;
 405                         oneCount = oneT*oneA;
 406
 407                         if ( oneCount > bestCount)
 408                         {
 409                                 bestAlignmentId = l;
 410                                 bestCount = oneCount;
 411                         }
 412                 }
 413
 414                 oneAlignment->insert(m, bestAlignmentId);
 415         }
 416 }