src/main.py

   1 # Todo: since we evaluate _after_ we reestimate, we loose the icharts
   2 # made while reestimating. If we had these available, evaluate and
   3 # corpus_likelihood would be a lot faster, but since they need to run
   4 # _after_ reestimate, we'll have to store an ichart per sentence. So
   5 # try storing those icharts in some loc_h_dmv global, and see if it's
   6 # faster using space rather than time.
   7
   8 from common_dmv import MPPROOT, test, node_str
   9 from wsjdep import WSJDepCorpusReader
  10
  11 #HARMONIC_C: 509.637290698, FNONSTOP_MIN: 30.1124584139, FSTOP_MIN: 13.0830178845
  12 def initialize_loc_h(tagonlys):
  13     import loc_h_harmonic # since we need to change constants (is there a better way?)
  14     reload(loc_h_harmonic)
  15     import random
  16 #     loc_h_harmonic.HARMONIC_C = 380.111684914
  17 #     loc_h_harmonic.FSTOP_MIN = 13.5744632704
  18 #     loc_h_harmonic.FNONSTOP_MIN = 34.8939452454
  19     loc_h_harmonic.HARMONIC_C =  0.0 #120.0 * random.random() # 509.63
  20     loc_h_harmonic.STOP_C =      1.0 #3.0 * random.random()
  21     loc_h_harmonic.NSTOP_C =     0.1 #5.0 * random.random() # 0.1
  22     loc_h_harmonic.FSTOP_MIN =  10.0 #20.0 * random.random() # 13.08
  23 #   $C_A=0; C_S=1;C_N=0.1;C_M=10$
  24     loc_h_harmonic.RIGHT_FIRST = 1.0
  25     loc_h_harmonic.OLD_STOP_CALC = False
  26     print '''
  27 HARMONIC_C: %s, STOP_C: %s, NSTOP_C: %s, FSTOP_MIN: %s
  28 RIGHT_FIRST: %s, OLD_STOP_CALC: %s'''%(loc_h_harmonic.HARMONIC_C,
  29                                        loc_h_harmonic.STOP_C,
  30                                        loc_h_harmonic.NSTOP_C,
  31                                        loc_h_harmonic.FSTOP_MIN,
  32                                        loc_h_harmonic.RIGHT_FIRST,
  33                                        loc_h_harmonic.OLD_STOP_CALC)
  34     g = loc_h_harmonic.initialize(tagonlys)
  35     return g
  36
  37 def initialize_uniform_loc_h(tagonlys):
  38     import loc_h_harmonic
  39     return loc_h_harmonic.uniform_init(tagonlys)
  40
  41 def initialize_cnf(tagonlys):
  42     import cnf_harmonic # since we need to change constants (is there a better way?)
  43     reload(cnf_harmonic)
  44     cnf_harmonic.HARMONIC_C = 0.0
  45     cnf_harmonic.FNONSTOP_MIN = 25
  46     cnf_harmonic.FSTOP_MIN = 5
  47     return cnf_harmonic.initialize(tagonlys)
  48
  49
  50 def test_likelihood(reestimate, initialize, inner_sent,
  51                     corpus_size=20, corpus_offset=1000, iterations=4, EVAL=False):
  52     def run_IO(g, iterations, tagonlys, tags_and_parses):
  53         sumlog,msg = corpus_likelihood(g, tagonlys)
  54         print msg
  55         if EVAL:
  56             g.E = evaluate(g, tags_and_parses)
  57             print g.E
  58         for i in range(iterations):
  59             g = reestimate(g, tagonlys)
  60             print "reestimation number %d done\n"%i
  61             if EVAL:
  62                 g.E = evaluate(g, tags_and_parses)
  63                 print g.E
  64             prev_sumlog = sumlog
  65             sumlog,msg = corpus_likelihood(g, tagonlys)
  66             if sumlog < prev_sumlog:
  67                 raise Exception, msg+"but previous was %s"%prev_sumlog
  68             print msg
  69             # since I want to be able to do stuff with it afterwards:
  70             from pickle import dump # let us say g = pickle.load(open('..','rb'))
  71             filehandler = open('current_grammar.obj','w')
  72             dump(g, filehandler)
  73             filehandler.close()
  74
  75         return g
  76
  77     def corpus_likelihood(g, tagsonly):
  78         from math import log
  79         sumlog = 0.0
  80         for sent in tagsonly:
  81             p_sent = inner_sent(g, sent, {})
  82             if p_sent == 0.0:
  83                 print "%s had zero probability!"%sent
  84             else:
  85                 sumlog += log(p_sent)
  86         avg = sumlog / len(tagsonly)
  87         return (sumlog, "Sum of log P_{sentence}: %.4f (should move towards 0), avg: %s"%(sumlog,avg))
  88
  89     reader = WSJDepCorpusReader(None)
  90     tagonlys = reader.tagonly_sents()[corpus_offset:corpus_offset+corpus_size]
  91     tags_and_parses = reader.tagged_and_parsed_sents()[corpus_offset:corpus_offset+corpus_size]
  92
  93     print "\ninitializing %d sentences..." % corpus_size,
  94     g = initialize(tagonlys)
  95     print "initialized"
  96
  97     g = run_IO(g, iterations, tagonlys, tags_and_parses) # make iterations argument, todo
  98
  99     if EVAL:
 100         import pprint
 101         print "underproposed:"
 102         pprint.pprint(g.E.underproposed)
 103         print "overproposed:"
 104         pprint.pprint(g.E.overproposed)
 105
 106     return g
 107
 108
 109 class Evaluation():
 110     "Just a class to hold evaluation-relevant information, sum it up, and print it."
 111     def __init__(self):
 112         self.underproposed, self.overproposed = {}, {}
 113         self.R, self.R_r, self.P, self.P_r = {}, {}, {}, {}
 114         for nd in ['num', 'den']:
 115             self.R[nd], self.R_r[nd], self.P[nd], self.P_r[nd] = 0, 0, 0, 0
 116
 117         self.unrooted = 0 # parses where we couldn't add_root
 118         self.double_heads = 0 # parses w/ two heads to one argument
 119         self._precision, self._recall, self._precision_r, self._recall_r = 0.0, 0.0, 0.0, 0.0
 120         self._F1, self._F1_r = 0.0, 0.0
 121
 122     def calc_F1_P_R(self):
 123         "F1 = (2 * P * R)/(P + R), harmonic avg. of P and R"
 124         self._recall = float(self.R['num']) / float(self.R['den'])
 125         self._precision = float(self.P['num']) / float(self.P['den'])
 126         self._recall_r = float(self.R['num']+self.R_r['num']) / \
 127             float(self.R['den']+self.R_r['den'])
 128         self._precision_r = float(self.P['num']+self.P_r['num']) / \
 129             float(self.P['den']+self.P_r['den'])
 130
 131         if (self._precision + self._recall) > 0.0:
 132             self._F1 = (2 * self._recall * self._precision) / (self._precision + self._recall)
 133         if (self._precision_r + self._recall_r) > 0.0:
 134             self._F1_r = (2 * self._recall_r * self._precision_r) / (self._precision_r + self._recall_r)
 135
 136     def __str__(self):
 137         self.calc_F1_P_R()
 138         R_rnum = self.R['num']+self.R_r['num']
 139         R_rden = self.R['den']+self.R_r['den']
 140         P_rnum = self.P['num']+self.P_r['num']
 141         P_rden = self.P['den']+self.P_r['den']
 142         str_vals = (self.P['num'],self.P['den'],self._precision, P_rnum,P_rden,self._precision_r,
 143                     self.R['num'],self.R['den'],self._recall,    R_rnum,R_rden,self._recall_r,
 144                     self._F1, self._F1_r, self.unrooted, self.double_heads)
 145         regular_str = '''P:  %5d/%5d = %s | P_r:  %5d/%5d = %s
 146 R:  %5d/%5d = %s | R_r:  %5d/%5d = %s
 147 F1:               %s | F1_r:               %s (unrooted gold parses: %d, double-headed: %d)'''%str_vals
 148
 149         tex_str_vals = tuple([p * 100 for p in (self._precision,self._precision_r,self._recall,self._recall_r,self._F1,self._F1_r)])
 150         tex_str = "$C_A=; C_S=;C_N=;C_M=$   & %.1f (%.1f)   & %.1f (%.1f)    & %.1f (%.1f) \\"%tex_str_vals
 151
 152         return tex_str # todo make variable
 153
 154
 155
 156 def evaluate(g, tagged_and_parsed_sents):
 157     ''' tagged_and_parsed_sents is a list of pairs:
 158     (tagonly_sent, parsed_sent)
 159
 160     R_num += 1 if pair from parsed is in mpp
 161     R_den += 1 per pair from parsed
 162
 163     P_num += 1 if pair from mpp is in parsed
 164     P_den += 1 per pair from mpp '''
 165     from loc_h_dmv import mpp
 166     from wsjdep import add_root
 167     E = Evaluation()
 168
 169     for sent, gold_parse in tagged_and_parsed_sents:
 170         if len(sent)-1 != len(gold_parse):
 171             E.double_heads += 1
 172             continue
 173         mpp_sent = mpp(g, sent)
 174         try: gold_parse = add_root(gold_parse)
 175         except RuntimeError: E.unrooted += 1
 176
 177         for pair in gold_parse:
 178             dict = E.R
 179             if pair[0] == MPPROOT: dict = E.R_r
 180             dict['den'] += 1
 181             if pair in mpp_sent: dict['num'] += 1
 182             else:
 183                 try: E.underproposed[pair] += 1
 184                 except KeyError: E.underproposed[pair] = 1
 185
 186         for pair in mpp_sent:
 187             dict = E.P
 188             if pair[0] == MPPROOT: dict = E.P_r
 189             dict['den'] += 1
 190             if pair in gold_parse: dict['num'] += 1
 191             else:
 192                 try: E.overproposed[pair] += 1
 193                 except KeyError: E.overproposed[pair] = 1
 194
 195     return E
 196
 197
 198
 199 def compare_loc_h_cnf():
 200     reader = WSJDepCorpusReader(None)
 201     corpus_size = 200
 202     corpus_offset = 1000
 203     tagonlys = reader.tagonly_sents()[corpus_offset:corpus_offset+corpus_size]
 204
 205     import loc_h_harmonic, cnf_harmonic
 206     g_l = loc_h_harmonic.initialize(tagonlys)
 207     g_c = cnf_harmonic.initialize(tagonlys)
 208
 209     initials = [
 210          (g_l.p_ROOT.iteritems(), g_c.p_ROOT),
 211          (g_c.p_ROOT.iteritems(), g_l.p_ROOT),
 212          (g_l.p_STOP.iteritems(), g_c.p_STOP),
 213          (g_c.p_STOP.iteritems(), g_l.p_STOP),
 214          (g_l.p_ATTACH.iteritems(), g_c.p_ATTACH),
 215          (g_c.p_ATTACH.iteritems(), g_l.p_ATTACH)]
 216     for a_items, b in initials:
 217         for k,v in a_items:
 218             if k not in b.keys(): raise Warning, "a[%s]=%s, but %s not in b"(k,v,k)
 219             if (k,v) not in b.iteritems(): raise Warning, "a[%s]=%s, but b[%s]=%s"(k,v,k,b[k])
 220
 221
 222     import loc_h_dmv, cnf_dmv
 223     from common_dmv import GOR
 224     for sent in tagonlys:
 225         ochart_l, ochart_c, ichart_l, ichart_c = {},{},{},{}
 226         i_l = loc_h_dmv.inner_sent(g_l, sent, ichart_l)
 227         i_c = cnf_dmv.inner_sent(g_c, sent, ichart_c)
 228         test( "%s"%i_l, "%s"%i_c, "i_l","i_c")
 229
 230         for loc_w,w in enumerate(sent):
 231             w_node = (GOR, g_l.tagnum(w))
 232             o_l = loc_h_dmv.outer(loc_w,loc_w+1,w_node,loc_w, g_l, sent, ichart_l,ochart_l)
 233             o_c =   cnf_dmv.outer(loc_w,loc_w+1,w_node,       g_c, sent, ichart_c,ochart_c)
 234             print "%s, %s, %s"%(sent,node_str(w_node),loc_w)
 235             test("%s"%o_l, "%s"%o_c, "o_l(0,1,(GOR,%s),%d,...)"%(w,loc_w),"o_c")
 236
 237 # end compare_loc_h_cnf()
 238
 239
 240 def init_nothing(g,H,S,N,M):
 241     print '''
 242 HARMONIC_C: %s, STOP_C: %s, NSTOP_C: %s, FSTOP_MIN: %s'''%(H,S,N,M)
 243     return lambda corpus:g
 244
 245 def rnd_grammars_test():
 246     import loc_h_dmv
 247     reload(loc_h_dmv)
 248
 249     rnd_grammars0 = []
 250     for i in xrange(20):
 251         g = test_likelihood(loc_h_dmv.reestimate,
 252                             initialize_loc_h,
 253                             loc_h_dmv.inner_sent,
 254                             corpus_size=6268,
 255                             iterations=0,
 256                             corpus_offset=0,
 257                             EVAL=True)
 258         rnd_grammars0 += [(g, g.HARMONIC_C, g.STOP_C, g.NSTOP_C, g.FSTOP_MIN)]
 259
 260     rnd_grammars1 = [(test_likelihood(loc_h_dmv.reestimate,
 261                                       init_nothing(g,H,S,N,M),
 262                                       loc_h_dmv.inner_sent,
 263                                       corpus_size=6268,
 264                                       iterations=1,
 265                                       corpus_offset=0,
 266                                       EVAL=True),
 267                       H,S,N,M)
 268                     for g,H,S,N,M in rnd_grammars0]
 269     rnd_grammars2 = [(test_likelihood(loc_h_dmv.reestimate,
 270                                       init_nothing(g,H,S,N,M),
 271                                       loc_h_dmv.inner_sent,
 272                                       corpus_size=6268,
 273                                       iterations=1,
 274                                       corpus_offset=0,
 275                                       EVAL=True),
 276                       H,S,N,M)
 277                     for g,H,S,N,M in rnd_grammars1]
 278
 279
 280 if __name__ == "__main__":
 281     print "main.py:"
 282
 283     if False:
 284         rnd_grammars_test()
 285     else:
 286         import loc_h_dmv
 287         reload(loc_h_dmv)
 288         print "\ntrying reestimate v.1 ##############################"
 289         g = test_likelihood(loc_h_dmv.reestimate,
 290                             initialize_loc_h,
 291                             loc_h_dmv.inner_sent,
 292                             corpus_size=6268,
 293                             iterations=30,
 294                             corpus_offset=0,
 295                             EVAL=True)
 296         print g
 297
 298 #         print "\ntrying reestimate v.2 ##############################"
 299 #         g = test_likelihood(loc_h_dmv.reestimate2,
 300 #                             initialize_loc_h,
 301 #                             loc_h_dmv.inner_sent,
 302 #                             corpus_size=5,
 303 #                             iterations=4,
 304 #                             corpus_offset=0)
 305 #         print "main.py: done"
 306 #         print g
 307
 308
 309 #     compare_loc_h_cnf()
 310 #     import cnf_dmv
 311 #     reload(cnf_dmv)
 312 #     print "\ntrying cnf-reestimate ##############################"
 313 #     g = test_likelihood(cnf_dmv.reestimate,
 314 #                         initialize_cnf,
 315 #                         cnf_dmv.inner_sent,
 316 #                         corpus_size=5,
 317 #                         iterations=4)