1 ######################## BEGIN LICENSE BLOCK ########################
2 # The Original Code is mozilla.org code.
4 # The Initial Developer of the Original Code is
5 # Netscape Communications Corporation.
6 # Portions created by the Initial Developer are Copyright (C) 1998
7 # the Initial Developer. All Rights Reserved.
10 # Mark Pilgrim - port to Python
12 # This library is free software; you can redistribute it and/or
13 # modify it under the terms of the GNU Lesser General Public
14 # License as published by the Free Software Foundation; either
15 # version 2.1 of the License, or (at your option) any later version.
17 # This library is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 # Lesser General Public License for more details.
22 # You should have received a copy of the GNU Lesser General Public
23 # License along with this library; if not, write to the Free Software
24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
26 ######################### END LICENSE BLOCK #########################
29 from .mbcharsetprober
import MultiByteCharSetProber
30 from .codingstatemachine
import CodingStateMachine
31 from .chardistribution
import SJISDistributionAnalysis
32 from .jpcntx
import SJISContextAnalysis
33 from .mbcssm
import SJISSMModel
34 from . import constants
37 class SJISProber(MultiByteCharSetProber
):
39 MultiByteCharSetProber
.__init
__(self
)
40 self
._mCodingSM
= CodingStateMachine(SJISSMModel
)
41 self
._mDistributionAnalyzer
= SJISDistributionAnalysis()
42 self
._mContextAnalyzer
= SJISContextAnalysis()
46 MultiByteCharSetProber
.reset(self
)
47 self
._mContextAnalyzer
.reset()
49 def get_charset_name(self
):
54 for i
in range(0, aLen
):
55 codingState
= self
._mCodingSM
.next_state(aBuf
[i
])
56 if codingState
== constants
.eError
:
58 sys
.stderr
.write(self
.get_charset_name()
59 + ' prober hit error at byte ' + str(i
)
61 self
._mState
= constants
.eNotMe
63 elif codingState
== constants
.eItsMe
:
64 self
._mState
= constants
.eFoundIt
66 elif codingState
== constants
.eStart
:
67 charLen
= self
._mCodingSM
.get_current_charlen()
69 self
._mLastChar
[1] = aBuf
[0]
70 self
._mContextAnalyzer
.feed(self
._mLastChar
[2 - charLen
:],
72 self
._mDistributionAnalyzer
.feed(self
._mLastChar
, charLen
)
74 self
._mContextAnalyzer
.feed(aBuf
[i
+ 1 - charLen
:i
+ 3
76 self
._mDistributionAnalyzer
.feed(aBuf
[i
- 1:i
+ 1],
79 self
._mLastChar
[0] = aBuf
[aLen
- 1]
81 if self
.get_state() == constants
.eDetecting
:
82 if (self
._mContextAnalyzer
.got_enough_data() and
83 (self
.get_confidence() > constants
.SHORTCUT_THRESHOLD
)):
84 self
._mState
= constants
.eFoundIt
86 return self
.get_state()
88 def get_confidence(self
):
89 contxtCf
= self
._mContextAnalyzer
.get_confidence()
90 distribCf
= self
._mDistributionAnalyzer
.get_confidence()
91 return max(contxtCf
, distribCf
)