2ff12f98faaa3a5e4b49f0b1f6036306fed6e643
1 # Todo: since we evaluate _after_ we reestimate, we loose the icharts
2 # made while reestimating. If we had these available, evaluate and
3 # corpus_likelihood would be a lot faster, but since they need to run
4 # _after_ reestimate, we'll have to store an ichart per sentence. So
5 # try storing those icharts in some loc_h_dmv global, and see if it's
6 # faster using space rather than time.
8 from common_dmv
import MPPROOT
, test
, node_str
9 from wsjdep
import WSJDepCorpusReader
10 #HARMONIC_C: 509.637290698, FNONSTOP_MIN: 30.1124584139, FSTOP_MIN: 13.0830178845
11 def initialize_loc_h(tagonlys
):
12 import loc_h_harmonic
# since we need to change constants (is there a better way?)
13 reload(loc_h_harmonic
)
15 # loc_h_harmonic.HARMONIC_C = 380.111684914
16 # loc_h_harmonic.FSTOP_MIN = 13.5744632704
17 # loc_h_harmonic.FNONSTOP_MIN = 34.8939452454
18 loc_h_harmonic
.HARMONIC_C
= 20.0 * random
.random() # 509.63
19 loc_h_harmonic
.FSTOP_MIN
= 10.0 # random.random() # 13.08
20 loc_h_harmonic
.STOP_C
= 3.0 * random
.random()
21 loc_h_harmonic
.NSTOP_C
= 1.0 # 5 * random.random() # 0.1
23 loc_h_harmonic
.RIGHT_FIRST
= 1.0
24 loc_h_harmonic
.OLD_STOP_CALC
= False
26 HARMONIC_C: %s, STOP_C: %s, NSTOP_C: %s, FSTOP_MIN: %s
27 RIGHT_FIRST: %s, OLD_STOP_CALC: %s'''%(loc_h_harmonic
.HARMONIC_C
,
28 loc_h_harmonic
.STOP_C
,
29 loc_h_harmonic
.NSTOP_C
,
30 loc_h_harmonic
.FSTOP_MIN
,
31 loc_h_harmonic
.RIGHT_FIRST
,
32 loc_h_harmonic
.OLD_STOP_CALC
)
33 g
= loc_h_harmonic
.initialize(tagonlys
)
36 def initialize_cnf(tagonlys
):
37 import cnf_harmonic
# since we need to change constants (is there a better way?)
39 cnf_harmonic
.HARMONIC_C
= 0.0
40 cnf_harmonic
.FNONSTOP_MIN
= 25
41 cnf_harmonic
.FSTOP_MIN
= 5
42 return cnf_harmonic
.initialize(tagonlys
)
45 def test_likelihood(reestimate
, initialize
, inner_sent
,
46 corpus_size
=20, corpus_offset
=1000, iterations
=4, EVAL
=False):
47 def run_IO(g
, iterations
, tagonlys
, tags_and_parses
):
48 sumlog
,msg
= corpus_likelihood(g
, tagonlys
)
51 E
= evaluate(g
, tags_and_parses
)
53 for i
in range(iterations
):
54 g
= reestimate(g
, tagonlys
)
55 print "reestimation number %d done\n"%i
57 E
= evaluate(g
, tags_and_parses
)
60 sumlog
,msg
= corpus_likelihood(g
, tagonlys
)
61 if sumlog
< prev_sumlog
:
62 raise Exception, msg
+"but previous was %s"%prev_sumlog
64 # since I want to be able to do stuff with it afterwards:
65 from pickle
import dump
# let us say g = pickle.load(open('..','rb'))
66 filehandler
= open('current_grammar.obj','w')
71 print "underproposed:"
72 pprint
.pprint(E
.underproposed
)
74 pprint
.pprint(E
.overproposed
)
78 def corpus_likelihood(g
, tagsonly
):
82 p_sent
= inner_sent(g
, sent
, {})
84 print "%s had zero probability!"%sent
87 avg
= sumlog
/ len(tagsonly
)
88 return (sumlog
, "Sum of log P_{sentence}: %.4f (should move towards 0), avg: %s"%(sumlog
,avg
))
90 reader
= WSJDepCorpusReader(None)
91 tagonlys
= reader
.tagonly_sents()[corpus_offset
:corpus_offset
+corpus_size
]
92 tags_and_parses
= reader
.tagged_and_parsed_sents()[corpus_offset
:corpus_offset
+corpus_size
]
94 # from loc_h_dmv import testcorpus
95 # tagonlys = testcorpus
97 print "\ninitializing %d sentences..." % corpus_size
,
98 g
= initialize(tagonlys
)
101 g
= run_IO(g
, iterations
, tagonlys
, tags_and_parses
) # make iterations argument, todo
107 "Just a class to hold evaluation-relevant information, sum it up, and print it."
109 self
.underproposed
, self
.overproposed
= {}, {}
110 self
.R
, self
.R_r
, self
.P
, self
.P_r
= {}, {}, {}, {}
111 for nd
in ['num', 'den']:
112 self
.R
[nd
], self
.R_r
[nd
], self
.P
[nd
], self
.P_r
[nd
] = 0, 0, 0, 0
114 self
.unrooted
= 0 # parses where we couldn't add_root
115 self
._precision
, self
._recall
, self
._precision
_r
, self
._recall
_r
= 0.0, 0.0, 0.0, 0.0
116 self
._F
1, self
._F
1_r
= 0.0, 0.0
118 def calc_F1_P_R(self
):
119 "F1 = (2 * P * R)/(P + R), harmonic avg. of P and R"
120 self
._recall
= float(self
.R
['num']) / float(self
.R
['den'])
121 self
._precision
= float(self
.P
['num']) / float(self
.P
['den'])
122 self
._recall
_r
= float(self
.R
['num']+self
.R_r
['num']) / \
123 float(self
.R
['den']+self
.R_r
['den'])
124 self
._precision
_r
= float(self
.P
['num']+self
.P_r
['num']) / \
125 float(self
.P
['den']+self
.P_r
['den'])
127 if (self
._precision
+ self
._recall
) > 0.0:
128 self
._F
1 = (2 * self
._recall
* self
._precision
) / (self
._precision
+ self
._recall
)
129 if (self
._precision
_r
+ self
._recall
_r
) > 0.0:
130 self
._F
1_r
= (2 * self
._recall
_r
* self
._precision
_r
) / (self
._precision
_r
+ self
._recall
_r
)
134 R_rnum
= self
.R
['num']+self
.R_r
['num']
135 R_rden
= self
.R
['den']+self
.R_r
['den']
136 P_rnum
= self
.P
['num']+self
.P_r
['num']
137 P_rden
= self
.P
['den']+self
.P_r
['den']
138 str_vals
= (self
.R
['num'],self
.R
['den'],self
._recall
, R_rnum
,R_rden
,self
._recall
_r
,
139 self
.P
['num'],self
.P
['den'],self
._precision
, P_rnum
,P_rden
,self
._precision
_r
,
140 self
._F
1, self
._F
1_r
, self
.unrooted
)
141 return '''R: %5d/%5d = %s | R_r: %5d/%5d = %s
142 P: %5d/%5d = %s | P_r: %5d/%5d = %s
143 F1: %s | F1_r: %s (unrooted gold parses: %d)'''%str
_vals
147 def evaluate(g
, tagged_and_parsed_sents
):
148 ''' tagged_and_parsed_sents is a list of pairs:
149 (tagonly_sent, parsed_sent)
151 R_num += 1 if pair from parsed is in mpp
152 R_den += 1 per pair from parsed
154 P_num += 1 if pair from mpp is in parsed
155 P_den += 1 per pair from mpp '''
156 from loc_h_dmv
import mpp
157 from wsjdep
import add_root
160 for sent
, gold_parse
in tagged_and_parsed_sents
:
161 mpp_sent
= mpp(g
, sent
)
162 try: gold_parse
= add_root(gold_parse
)
163 except ValueError: E
.unrooted
+= 1
165 for pair
in gold_parse
:
167 if pair
[0] == MPPROOT
: dict = E
.R_r
169 if pair
in mpp_sent
: dict['num'] += 1
171 try: E
.underproposed
[pair
] += 1
172 except KeyError: E
.underproposed
[pair
] = 1
174 for pair
in mpp_sent
:
176 if pair
[0] == MPPROOT
: dict = E
.P_r
178 if pair
in gold_parse
: dict['num'] += 1
180 try: E
.overproposed
[pair
] += 1
181 except KeyError: E
.overproposed
[pair
] = 1
187 def compare_loc_h_cnf():
188 reader
= WSJDepCorpusReader(None)
191 tagonlys
= reader
.tagonly_sents()[corpus_offset
:corpus_offset
+corpus_size
]
193 import loc_h_harmonic
, cnf_harmonic
194 g_l
= loc_h_harmonic
.initialize(tagonlys
)
195 g_c
= cnf_harmonic
.initialize(tagonlys
)
198 (g_l
.p_ROOT
.iteritems(), g_c
.p_ROOT
),
199 (g_c
.p_ROOT
.iteritems(), g_l
.p_ROOT
),
200 (g_l
.p_STOP
.iteritems(), g_c
.p_STOP
),
201 (g_c
.p_STOP
.iteritems(), g_l
.p_STOP
),
202 (g_l
.p_ATTACH
.iteritems(), g_c
.p_ATTACH
),
203 (g_c
.p_ATTACH
.iteritems(), g_l
.p_ATTACH
)]
204 for a_items
, b
in initials
:
206 if k
not in b
.keys(): raise Warning, "a[%s]=%s, but %s not in b"(k
,v
,k
)
207 if (k
,v
) not in b
.iteritems(): raise Warning, "a[%s]=%s, but b[%s]=%s"(k
,v
,k
,b
[k
])
210 import loc_h_dmv
, cnf_dmv
211 from common_dmv
import GOR
212 for sent
in tagonlys
:
213 ochart_l
, ochart_c
, ichart_l
, ichart_c
= {},{},{},{}
214 i_l
= loc_h_dmv
.inner_sent(g_l
, sent
, ichart_l
)
215 i_c
= cnf_dmv
.inner_sent(g_c
, sent
, ichart_c
)
216 test( "%s"%i_l, "%s"%i_c
, "i_l","i_c")
218 for loc_w
,w
in enumerate(sent
):
219 w_node
= (GOR
, g_l
.tagnum(w
))
220 o_l
= loc_h_dmv
.outer(loc_w
,loc_w
+1,w_node
,loc_w
, g_l
, sent
, ichart_l
,ochart_l
)
221 o_c
= cnf_dmv
.outer(loc_w
,loc_w
+1,w_node
, g_c
, sent
, ichart_c
,ochart_c
)
222 print "%s, %s, %s"%(sent
,node_str(w_node
),loc_w
)
223 test("%s"%o_l, "%s"%o_c
, "o_l(0,1,(GOR,%s),%d,...)"%(w
,loc_w
),"o_c")
225 # end compare_loc_h_cnf()
228 def init_nothing(g
,H
,S
,N
,M
):
230 HARMONIC_C: %s, STOP_C: %s, NSTOP_C: %s, FSTOP_MIN: %s'''%(H
,S
,N
,M
)
231 return lambda corpus
:g
233 def rnd_grammars_test():
239 g
= test_likelihood(loc_h_dmv
.reestimate
,
241 loc_h_dmv
.inner_sent
,
246 rnd_grammars0
+= [(g
, g
.HARMONIC_C
, g
.STOP_C
, g
.NSTOP_C
, g
.FSTOP_MIN
)]
248 rnd_grammars1
= [(test_likelihood(loc_h_dmv
.reestimate
,
249 init_nothing(g
,H
,S
,N
,M
),
250 loc_h_dmv
.inner_sent
,
256 for g
,H
,S
,N
,M
in rnd_grammars0
]
257 rnd_grammars2
= [(test_likelihood(loc_h_dmv
.reestimate
,
258 init_nothing(g
,H
,S
,N
,M
),
259 loc_h_dmv
.inner_sent
,
265 for g
,H
,S
,N
,M
in rnd_grammars1
]
267 if __name__
== "__main__":
275 print "\ntrying reestimate v.1 ##############################"
276 g
= test_likelihood(loc_h_dmv
.reestimate
,
278 loc_h_dmv
.inner_sent
,
285 # print "\ntrying reestimate v.2 ##############################"
286 # g = test_likelihood(loc_h_dmv.reestimate2,
288 # loc_h_dmv.inner_sent,
292 # print "main.py: done"
296 # compare_loc_h_cnf()
299 # print "\ntrying cnf-reestimate ##############################"
300 # g = test_likelihood(cnf_dmv.reestimate,
302 # cnf_dmv.inner_sent,