1 // ConversionManager.cpp
3 // Copyright (c) 2007 The Dasher Team
5 // This file is part of Dasher.
7 // Dasher is free software; you can redistribute it and/or modify
8 // it under the terms of the GNU General Public License as published by
9 // the Free Software Foundation; either version 2 of the License, or
10 // (at your option) any later version.
12 // Dasher is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
17 // You should have received a copy of the GNU General Public License
18 // along with Dasher; if not, write to the Free Software
19 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25 #include "ConversionManager.h"
27 #include "EventHandler.h"
28 #include "NodeCreationManager.h"
35 using namespace Dasher
;
37 CConversionManager::CConversionManager(CNodeCreationManager
*pNCManager
, CConversionHelper
*pHelper
, CAlphabet
*pAlphabet
, int CMid
)
40 m_pNCManager
= pNCManager
;
42 m_pAlphabet
= pAlphabet
;
45 //DOESN'T SEEM INTRINSIC
46 //and check why pHelper may be empty
48 m_pLanguageModel
= pHelper
->GetLanguageModel();
50 m_pLanguageModel
= NULL
;
53 m_iLearnContext
= m_pLanguageModel
->CreateEmptyContext();
62 CConversionManager::~CConversionManager(){
64 // for (int i(0);i<m_iHZCount; i++)
65 // std::cout << "Unref: " << this << std::endl;
68 RecursiveDelTree(m_pRoot
[0]);
72 CDasherNode
*CConversionManager::GetRoot(CDasherNode
*pParent
, int iLower
, int iUpper
, void *pUserData
) {
73 CDasherNode
*pNewNode
;
75 int iOffset
= *(static_cast<int *>(pUserData
));
77 // TODO: Parameters here are placeholders - need to figure out what's right
79 CDasherNode::SDisplayInfo
*pDisplayInfo
= new CDasherNode::SDisplayInfo
;
80 pDisplayInfo
->iColour
= 9; // TODO: Hard coded value
81 pDisplayInfo
->bShove
= true;
82 pDisplayInfo
->bVisible
= true;
83 pDisplayInfo
->strDisplayText
= ">"; // TODO: Hard coded value, needs i18n
85 pNewNode
= new CDasherNode(pParent
, iLower
, iUpper
, pDisplayInfo
);
87 // FIXME - handle context properly
88 // TODO: Reimplemnt -----
89 // pNewNode->SetContext(m_pLanguageModel->CreateEmptyContext());
92 pNewNode
->m_pNodeManager
= this;
93 pNewNode
->m_pNodeManager
->Ref();
96 SConversionData
*pNodeUserData
= new SConversionData
;
97 pNewNode
->m_pUserData
= pNodeUserData
;
98 pNodeUserData
->bType
= false;
99 pNodeUserData
->iOffset
= iOffset
;
102 pNodeUserData
->pLanguageModel
= m_pHelper
->GetLanguageModel();
104 pNodeUserData
->pLanguageModel
= NULL
;
106 CLanguageModel::Context iContext
;
108 // // std::cout<<m_pLanguageModel<<"lala"<<std::endl;
109 if(m_pLanguageModel
) {
110 iContext
= m_pLanguageModel
->CreateEmptyContext();
111 pNodeUserData
->iContext
= iContext
;
114 pNodeUserData
->pSCENode
= 0;
119 // TODO: This function needs to be significantly tidied up
120 // TODO: get rid of pSizes
122 void CConversionManager::AssignChildSizes(SCENode
*pNode
, CLanguageModel::Context context
, int iNChildren
) {
123 DASHER_ASSERT(m_pHelper
);
125 // Calculate sizes for the children. Note that normalisation is
126 // done additiviely rather than multiplicatively, so it's not
127 // quite what was originally planned (but I don't think this is
128 // much of a problem). More serious is the fact that the ordering
129 // is being lost when the tree is created, as nodes begininning
130 // with the same character are merged. This needs to be though
131 // out, but the probabilities should probably be done at the time
132 // of construction of the candidate tree rather than the Dasher
133 // tree (aside - is there any real point having two separate trees
134 // - surely we should just create Dasher nodes right away?).
136 // The algorithm should also allow for the possibility of the
137 // conversion engine returning probabilities itself, which should
138 // be used in preference to the values infered from the ordering
140 // Finally, maybe the choices should be presented in lexographic
141 // order, rather than in order returned (really not sure about
142 // this - it needs to be thought through).
145 // std::cout << "b" << std::endl;
147 //TESTING FOR CALCULATESCORE STAGE 1
149 //test = CalculateScore(pNode, 1);
150 //std::cout<<"current character"<<pCurrentSCENode->pszConversion<<std::endl;
151 //std::cout<<"the score for the second candidate is"<<test<<std::endl;
155 //ASSIGNING SCORES AND CALCULATING NODE SIZE
156 //Ph: feel free to edit this part to make it more structured
157 // int iSize[pCurrentSCEChild->IsHeadAndCandNum];
158 // int score[pCurrentSCEChild->IsHeadAndCandNum];
161 // int CandNum = pCurrentSCEChild -> IsHeadAndCandNum;
163 // CHANGE int iRemaining(m_pNCManager->GetLongParameter(LP_NORMALIZATION));
165 // Thoughts on the general idea here - this is very close to being
166 // a fully fledged language model, so I think we should go with
167 // that idea, but maybe we need something mode flexible. I'd
170 // 1. Probabilities provided directly with translation? Maybe hard
171 // to represent in the lattice itself.
173 // 2. Full n-gram language model provided - in general assign
174 // probabilities to paths through the lattice
176 // 3. Ordered results, but no probabilities - using a power law
179 // Tempted to assume (1) and (2) can be implemented together, with
180 // a second call to the library at node creation time, and (3) can
181 // be implemented as a fallback if that doesn't work.
183 // Things to be thought out:
184 // - How to deal with contexts - backtrace at time of call or stored in node?
185 // - Sharing of language model infrastructure?
189 // Lookup scores for each of the children
191 // TODO: Reimplement -----
193 // for(int i(0); i < pCurrentSCEChild->IsHeadAndCandNum; ++i){
194 // score[i] = CalculateScore(pNode, i);
195 // total += score[i];
197 // if (score[i]>score[i-1])
203 // Use the scores to calculate the size of the nodes
207 SCENode
*pChild(pNode
);
210 pChild
= pChild
->GetNext();
215 m_pHelper
->AssignSizes(pNode
, context
, m_pNCManager
->GetLongParameter(LP_NORMALIZATION
), m_pNCManager
->GetLongParameter(LP_UNIFORM
), iNChildren
);
218 // for(int i(0); i < iNChildren; ++i) {
220 //TESTING FOR RESIZING FREQUENT HZ CHARACTERS
222 // if(score[i]<max-5){
223 // std::cout<<"first scores are"<<score[i]<<std::endl;
227 // TODO: Reimplement new model -----
230 // iSize[i] = m_pNCManager->GetLongParameter(LP_NORMALIZATION);
232 // iSize[i] = m_pNCManager->GetLongParameter(LP_NORMALIZATION)*((CandNum-i-1)+2*CandNum*score[i])/(CandNum*(CandNum-1)/2+2*CandNum*total);
236 //PREVIOUS MODEL: m_pNCManager->GetLongParameter(LP_NORMALIZATION)/((i + 1) * (i + 2));
244 uint freq[iNChildren];
245 for(int i(0); i<iNChildren; i++)
252 freq[pIt->CandIndex]=pIt->HZFreq;
253 totalFreq+=freq[pIt->CandIndex];
254 if(pIt->HZFreq>maxFreq)
259 pSizes[i] = m_pNCManager->GetLongParameter(LP_NORMALIZATION)*(100+5*freq[i])/(100*iNChildren+5*totalFreq);
261 //((i + 1) * (i + 2));
268 iRemaining -= pSizes[i];
271 // Distribute the remaining space evenly
273 int iLeft(iNChildren);
275 for(int i(0); i < iNChildren; ++i) {
276 int iDiff(iRemaining / iLeft);
287 void CConversionManager::PopulateChildren( CDasherNode
*pNode
) {
288 DASHER_ASSERT(m_pNCManager
);
290 SConversionData
* pCurrentDataNode (static_cast<SConversionData
*>(pNode
->m_pUserData
));
291 CDasherNode
*pNewNode
;
293 // If no helper class is present then just drop straight back to an
294 // alphabet root. This should only happen in error cases, and the
295 // user should have been warned here.
299 int iHbnd(m_pNCManager
->GetLongParameter(LP_NORMALIZATION
));
301 CAlphabetManager::SRootData oRootData
;
302 oRootData
.szContext
= NULL
;
303 oRootData
.iOffset
= pCurrentDataNode
->iOffset
+ 1;
305 pNewNode
= m_pNCManager
->GetRoot(0, pNode
, iLbnd
, iHbnd
, &oRootData
);
306 pNewNode
->SetFlag(NF_SEEN
, false);
308 pNode
->Children().push_back(pNewNode
);
313 // Do the conversion and build the tree (lattice) if it hasn't been
321 SCENode
*pCurrentSCEChild
;
323 if(pCurrentDataNode
->pSCENode
)
324 pCurrentSCEChild
= pCurrentDataNode
->pSCENode
->GetChild();
326 if(m_pRoot
&& !pCurrentDataNode
->bType
)
327 pCurrentSCEChild
= m_pRoot
[0];
329 pCurrentSCEChild
= 0;
332 if(pCurrentSCEChild
) {
334 // TODO: Reimplement (in subclass) -----
337 // if(!m_bPhrasesProcessed[pCurrentSCEChild->AcCharCount-1])
338 // if(pCurrentSCEChild->AcCharCount<m_iHZCount)
339 // ProcessPhrase(pCurrentSCEChild->AcCharCount-1);
345 // iSize = new int[pCurrentSCEChild->IsHeadAndCandNum];
350 AssignChildSizes(pCurrentSCEChild
, pCurrentDataNode
->iContext
, pCurrentSCEChild
->IsHeadAndCandNum
);
355 // int parentClr = pNode->Colour();
359 // Finally loop through and create the children
363 int iHbnd(iCum
+ pCurrentSCEChild
->NodeSize
);
367 // TODO: Parameters here are placeholders - need to figure out
371 CDasherNode::SDisplayInfo
*pDisplayInfo
= new CDasherNode::SDisplayInfo
;
372 pDisplayInfo
->iColour
= m_pHelper
->AssignColour(parentClr
, pCurrentSCEChild
, iIdx
);
373 pDisplayInfo
->bShove
= true;
374 pDisplayInfo
->bVisible
= true;
376 // std::cout << "#" << pCurrentSCEChild->pszConversion << "#" << std::endl;
378 pDisplayInfo
->strDisplayText
= pCurrentSCEChild
->pszConversion
;
380 pNewNode
= new CDasherNode(pNode
, iLbnd
, iHbnd
, pDisplayInfo
);
382 // TODO: Reimplement ----
384 // FIXME - handle context properly
385 // pNewNode->SetContext(m_pLanguageModel->CreateEmptyContext());
388 pNewNode
->m_pNodeManager
= this;
389 pNewNode
->m_pNodeManager
->Ref();
391 SConversionData
*pNodeUserData
= new SConversionData
;
392 pNodeUserData
->bType
= false;
393 pNodeUserData
->pSCENode
= pCurrentSCEChild
;
394 pNodeUserData
->pLanguageModel
= pCurrentDataNode
->pLanguageModel
;
395 pNodeUserData
->iOffset
= pCurrentDataNode
->iOffset
+ 1;
397 CLanguageModel::Context iContext
;
398 iContext
= pCurrentDataNode
->pLanguageModel
->CloneContext(pCurrentDataNode
->iContext
);
400 if(pCurrentSCEChild
->Symbol
!=-1)
401 pNodeUserData
->pLanguageModel
->EnterSymbol(iContext
, pCurrentSCEChild
->Symbol
); // TODO: Don't use symbols?
404 pNodeUserData
->iContext
= iContext
;
406 pNewNode
->m_pUserData
= pNodeUserData
;
407 // SAlphabetData *pNodeUserData = new SAlphabetData;
409 //pNewNode->m_pUserData = pNodeUserData;
411 //pNodeUserData->iPhase = iNewPhase;
412 //pNodeUserData->iSymbol = iIdx;
414 pNode
->Children().push_back(pNewNode
);
416 pCurrentSCEChild
= pCurrentSCEChild
->GetNext();
418 }while(pCurrentSCEChild
);
425 if((static_cast<SConversionData
*>(pNode
->m_pUserData
))->bType
) {
426 // TODO: Placeholder algorithm here
427 // TODO: Add an 'end of conversion' node?
429 int iHbnd(m_pNCManager
->GetLongParameter(LP_NORMALIZATION
));
431 CAlphabetManager::SRootData oRootData
;
432 oRootData
.szContext
= NULL
;
433 oRootData
.iOffset
= pCurrentDataNode
->iOffset
+ 1;
435 pNewNode
= m_pNCManager
->GetRoot(0, pNode
, iLbnd
, iHbnd
, &oRootData
);
436 pNewNode
->SetFlag(NF_SEEN
, false);
438 pNode
->Children().push_back(pNewNode
);
439 // pNode->SetHasAllChildren(false);
443 int iHbnd(m_pNCManager
->GetLongParameter(LP_NORMALIZATION
));
445 CDasherNode::SDisplayInfo
*pDisplayInfo
= new CDasherNode::SDisplayInfo
;
446 pDisplayInfo
->iColour
= m_pHelper
->AssignColour(0, pCurrentSCEChild
, 0);
447 pDisplayInfo
->bShove
= true;
448 pDisplayInfo
->bVisible
= true;
449 pDisplayInfo
->strDisplayText
= "";
451 pNewNode
= new CDasherNode(pNode
, iLbnd
, iHbnd
, pDisplayInfo
);
453 // TODO: Reimplement ----
455 // FIXME - handle context properly
456 // pNewNode->SetContext(m_pLanguageModel->CreateEmptyContext());
459 pNewNode
->m_pNodeManager
= this;
460 pNewNode
->m_pNodeManager
->Ref();
462 SConversionData
*pNodeUserData
= new SConversionData
;
463 pNodeUserData
->bType
= true;
464 pNodeUserData
->pSCENode
= NULL
;
465 pNodeUserData
->pLanguageModel
= pCurrentDataNode
->pLanguageModel
;
466 pNodeUserData
->iOffset
= pCurrentDataNode
->iOffset
;
468 pNewNode
->m_pUserData
= pNodeUserData
;
470 pNewNode
->SetFlag(NF_SEEN
, false);
472 pNode
->Children().push_back(pNewNode
);
477 void CConversionManager::ClearNode( CDasherNode
*pNode
) {
478 // pNode->m_pNodeManager->Unref();
481 void CConversionManager::RecursiveDumpTree(SCENode
*pCurrent
, unsigned int iDepth
) {
483 for(unsigned int i(0); i
< iDepth
; ++i
)
486 std::cout
<< " " << pCurrent
->pszConversion
<< " " << pCurrent
->IsHeadAndCandNum
<< " " << pCurrent
->CandIndex
<< " " << pCurrent
->IsComplete
<< " " << pCurrent
->AcCharCount
<< std::endl
;
488 RecursiveDumpTree(pCurrent
->GetChild(), iDepth
+ 1);
489 pCurrent
= pCurrent
->GetNext();
493 void CConversionManager::BuildTree(CDasherNode
*pRoot
) {
494 DASHER_ASSERT(m_pHelper
);
496 CDasherNode
*pCurrentNode(pRoot
->Parent());
498 std::string strCurrentString
;
499 m_pHelper
->ClearData(m_iCMID
);
501 while(pCurrentNode
) {
502 if(pCurrentNode
->m_pNodeManager
->GetID() == 2)
505 // TODO: Need to make this the edit text rather than the display text
506 CAlphabetManager::SAlphabetData
*pAlphabetData
=
507 static_cast<CAlphabetManager::SAlphabetData
*>(pCurrentNode
->m_pUserData
);
509 strCurrentString
= m_pAlphabet
->GetText(pAlphabetData
->iSymbol
) + strCurrentString
;
510 pCurrentNode
= pCurrentNode
->Parent();
513 // TODO: The remainder of this function is messy - to be sorted out
516 bool ConversionSuccess
;
518 ConversionSuccess
= m_pHelper
->Convert(strCurrentString
, &pStartTemp
, &iHZCount
, m_iCMID
);
520 if((!ConversionSuccess
)||(iHZCount
==0)) {
524 m_pRoot
= new SCENode
*[1];
525 m_pRoot
[0] = pStartTemp
;
529 void CConversionManager::Output( CDasherNode
*pNode
, Dasher::VECTOR_SYMBOL_PROB
* pAdded
, int iNormalization
) {
530 // TODO: Reimplement this
531 // m_pNCManager->m_bContextSensitive = true;
533 SCENode
*pCurrentSCENode((static_cast<SConversionData
*>(pNode
->m_pUserData
))->pSCENode
);
535 if(pCurrentSCENode
) {
536 Dasher::CEditEvent
oEvent(1, pCurrentSCENode
->pszConversion
);
537 m_pNCManager
->InsertEvent(&oEvent
);
539 if((pNode
->GetChildren())[0]->m_pNodeManager
!= this) {
540 Dasher::CEditEvent
oEvent(11, "");
541 m_pNCManager
->InsertEvent(&oEvent
);
545 if((static_cast<SConversionData
*>(pNode
->m_pUserData
))->bType
) {
546 Dasher::CEditEvent
oOPEvent(1, "|");
547 m_pNCManager
->InsertEvent(&oOPEvent
);
550 Dasher::CEditEvent
oOPEvent(1, ">");
551 m_pNCManager
->InsertEvent(&oOPEvent
);
554 Dasher::CEditEvent
oEvent(10, "");
555 m_pNCManager
->InsertEvent(&oEvent
);
559 void CConversionManager::Undo( CDasherNode
*pNode
) {
560 SCENode
*pCurrentSCENode((static_cast<SConversionData
*>(pNode
->m_pUserData
))->pSCENode
);
562 if(pCurrentSCENode
) {
563 if(pCurrentSCENode
->pszConversion
&& (strlen(pCurrentSCENode
->pszConversion
) > 0)) {
564 Dasher::CEditEvent
oEvent(2, pCurrentSCENode
->pszConversion
);
565 m_pNCManager
->InsertEvent(&oEvent
);
569 if((static_cast<SConversionData
*>(pNode
->m_pUserData
))->bType
) {
570 Dasher::CEditEvent
oOPEvent(2, "|");
571 m_pNCManager
->InsertEvent(&oOPEvent
);
574 Dasher::CEditEvent
oOPEvent(2, ">");
575 m_pNCManager
->InsertEvent(&oOPEvent
);
580 bool CConversionManager::RecursiveDelTree(SCENode
* pNode
){
581 // TODO: Do we actually care about the return value?
583 // TODO: Function now obsolete
591 // // Note that this is a lattice, not a tree, so we need to be careful
592 // // about deleting thing twice
595 // RecursiveDeleteTree(pNode->pChild);
598 // RecursiveDeleteTree(pNode->pNext);
604 // else if(pNode->pChild)
605 // return RecursiveDelTree(pNode->pChild);
608 // while(!pNode->pChild){
609 // pTemp = pNode->pNext;
615 // return RecursiveDelTree(pNode->pChild);
620 void CConversionManager::SetFlag(CDasherNode
*pNode
, int iFlag
, bool bValue
) {
624 // TODO: Reimplement (need a learning context, check whether
625 // symbol actually corresponds to character)
627 CLanguageModel
* pLan
= static_cast<SConversionData
*>(pNode
->m_pUserData
)->pLanguageModel
;
629 SCENode
* pSCENode
= static_cast<SConversionData
*>(pNode
->m_pUserData
)->pSCENode
;
634 symbol s
=pSCENode
->Symbol
;
637 if((s
!=-1) && m_pLanguageModel
)
638 pLan
->LearnSymbol(m_iLearnContext
, s
);