src/main.py

   1 # Todo: since we evaluate _after_ we reestimate, we loose the icharts
   2 # made while reestimating. If we had these available, evaluate and
   3 # corpus_likelihood would be a lot faster, but since they need to run
   4 # _after_ reestimate, we'll have to store an ichart per sentence. So
   5 # try storing those icharts in some loc_h_dmv global, and see if it's
   6 # faster using space rather than time.
   7
   8 from common_dmv import MPPROOT, test, node_str
   9 from wsjdep import WSJDepCorpusReader
  10 #HARMONIC_C: 509.637290698, FNONSTOP_MIN: 30.1124584139, FSTOP_MIN: 13.0830178845
  11 def initialize_loc_h(tagonlys):
  12     import loc_h_harmonic # since we need to change constants (is there a better way?)
  13     reload(loc_h_harmonic)
  14     import random
  15 #     loc_h_harmonic.HARMONIC_C = 380.111684914
  16 #     loc_h_harmonic.FSTOP_MIN = 13.5744632704
  17 #     loc_h_harmonic.FNONSTOP_MIN = 34.8939452454
  18     loc_h_harmonic.HARMONIC_C = 20.0 * random.random() # 509.63
  19     loc_h_harmonic.FSTOP_MIN = 10.0 # random.random() # 13.08
  20     loc_h_harmonic.STOP_C = 3.0 * random.random()
  21     loc_h_harmonic.NSTOP_C = 1.0 # 5 * random.random() # 0.1
  22
  23     loc_h_harmonic.RIGHT_FIRST = 1.0
  24     loc_h_harmonic.OLD_STOP_CALC = False
  25     print '''
  26 HARMONIC_C: %s, STOP_C: %s, NSTOP_C: %s, FSTOP_MIN: %s
  27 RIGHT_FIRST: %s, OLD_STOP_CALC: %s'''%(loc_h_harmonic.HARMONIC_C,
  28                                        loc_h_harmonic.STOP_C,
  29                                        loc_h_harmonic.NSTOP_C,
  30                                        loc_h_harmonic.FSTOP_MIN,
  31                                        loc_h_harmonic.RIGHT_FIRST,
  32                                        loc_h_harmonic.OLD_STOP_CALC)
  33     g = loc_h_harmonic.initialize(tagonlys)
  34     return g
  35
  36 def initialize_cnf(tagonlys):
  37     import cnf_harmonic # since we need to change constants (is there a better way?)
  38     reload(cnf_harmonic)
  39     cnf_harmonic.HARMONIC_C = 0.0
  40     cnf_harmonic.FNONSTOP_MIN = 25
  41     cnf_harmonic.FSTOP_MIN = 5
  42     return cnf_harmonic.initialize(tagonlys)
  43
  44
  45 def test_likelihood(reestimate, initialize, inner_sent,
  46                     corpus_size=20, corpus_offset=1000, iterations=4, EVAL=False):
  47     def run_IO(g, iterations, tagonlys, tags_and_parses):
  48         sumlog,msg = corpus_likelihood(g, tagonlys)
  49         print msg
  50         if EVAL:
  51             E = evaluate(g, tags_and_parses)
  52             print E
  53         for i in range(iterations):
  54             g = reestimate(g, tagonlys)
  55             print "reestimation number %d done\n"%i
  56             if EVAL:
  57                 E = evaluate(g, tags_and_parses)
  58                 print E
  59             prev_sumlog = sumlog
  60             sumlog,msg = corpus_likelihood(g, tagonlys)
  61             if sumlog < prev_sumlog:
  62                 raise Exception, msg+"but previous was %s"%prev_sumlog
  63             print msg
  64             # since I want to be able to do stuff with it afterwards:
  65             from pickle import dump # let us say g = pickle.load(open('..','rb'))
  66             filehandler = open('current_grammar.obj','w')
  67             dump(g, filehandler)
  68
  69         if EVAL:
  70             import pprint
  71             print "underproposed:"
  72             pprint.pprint(E.underproposed)
  73             print "overproposed:"
  74             pprint.pprint(E.overproposed)
  75
  76         return g
  77
  78     def corpus_likelihood(g, tagsonly):
  79         from math import log
  80         sumlog = 0.0
  81         for sent in tagsonly:
  82             p_sent = inner_sent(g, sent, {})
  83             if p_sent == 0.0:
  84                 print "%s had zero probability!"%sent
  85             else:
  86                 sumlog += log(p_sent)
  87         avg = sumlog / len(tagsonly)
  88         return (sumlog, "Sum of log P_{sentence}: %.4f (should move towards 0), avg: %s"%(sumlog,avg))
  89
  90     reader = WSJDepCorpusReader(None)
  91     tagonlys = reader.tagonly_sents()[corpus_offset:corpus_offset+corpus_size]
  92     tags_and_parses = reader.tagged_and_parsed_sents()[corpus_offset:corpus_offset+corpus_size]
  93
  94 #     from loc_h_dmv import testcorpus
  95 #     tagonlys = testcorpus
  96
  97     print "\ninitializing %d sentences..." % corpus_size,
  98     g = initialize(tagonlys)
  99     print "initialized"
 100
 101     g = run_IO(g, iterations, tagonlys, tags_and_parses) # make iterations argument, todo
 102
 103     return g
 104
 105
 106 class Evaluation():
 107     "Just a class to hold evaluation-relevant information, sum it up, and print it."
 108     def __init__(self):
 109         self.underproposed, self.overproposed = {}, {}
 110         self.R, self.R_r, self.P, self.P_r = {}, {}, {}, {}
 111         for nd in ['num', 'den']:
 112             self.R[nd], self.R_r[nd], self.P[nd], self.P_r[nd] = 0, 0, 0, 0
 113
 114         self.unrooted = 0 # parses where we couldn't add_root
 115         self._precision, self._recall, self._precision_r, self._recall_r = 0.0, 0.0, 0.0, 0.0
 116         self._F1, self._F1_r = 0.0, 0.0
 117
 118     def calc_F1_P_R(self):
 119         "F1 = (2 * P * R)/(P + R), harmonic avg. of P and R"
 120         self._recall = float(self.R['num']) / float(self.R['den'])
 121         self._precision = float(self.P['num']) / float(self.P['den'])
 122         self._recall_r = float(self.R['num']+self.R_r['num']) / \
 123             float(self.R['den']+self.R_r['den'])
 124         self._precision_r = float(self.P['num']+self.P_r['num']) / \
 125             float(self.P['den']+self.P_r['den'])
 126
 127         if (self._precision + self._recall) > 0.0:
 128             self._F1 = (2 * self._recall * self._precision) / (self._precision + self._recall)
 129         if (self._precision_r + self._recall_r) > 0.0:
 130             self._F1_r = (2 * self._recall_r * self._precision_r) / (self._precision_r + self._recall_r)
 131
 132     def __str__(self):
 133         self.calc_F1_P_R()
 134         R_rnum = self.R['num']+self.R_r['num']
 135         R_rden = self.R['den']+self.R_r['den']
 136         P_rnum = self.P['num']+self.P_r['num']
 137         P_rden = self.P['den']+self.P_r['den']
 138         str_vals = (self.R['num'],self.R['den'],self._recall,    R_rnum,R_rden,self._recall_r,
 139                     self.P['num'],self.P['den'],self._precision, P_rnum,P_rden,self._precision_r,
 140                     self._F1, self._F1_r, self.unrooted)
 141         return '''R:  %5d/%5d = %s | R_r:  %5d/%5d = %s
 142 P:  %5d/%5d = %s | P_r:  %5d/%5d = %s
 143 F1:               %s | F1_r:               %s (unrooted gold parses: %d)'''%str_vals
 144
 145
 146
 147 def evaluate(g, tagged_and_parsed_sents):
 148     ''' tagged_and_parsed_sents is a list of pairs:
 149     (tagonly_sent, parsed_sent)
 150
 151     R_num += 1 if pair from parsed is in mpp
 152     R_den += 1 per pair from parsed
 153
 154     P_num += 1 if pair from mpp is in parsed
 155     P_den += 1 per pair from mpp '''
 156     from loc_h_dmv import mpp
 157     from wsjdep import add_root
 158     E = Evaluation()
 159
 160     for sent, gold_parse in tagged_and_parsed_sents:
 161         mpp_sent = mpp(g, sent)
 162         try: gold_parse = add_root(gold_parse)
 163         except ValueError: E.unrooted += 1
 164
 165         for pair in gold_parse:
 166             dict = E.R
 167             if pair[0] == MPPROOT: dict = E.R_r
 168             dict['den'] += 1
 169             if pair in mpp_sent: dict['num'] += 1
 170             else:
 171                 try: E.underproposed[pair] += 1
 172                 except KeyError: E.underproposed[pair] = 1
 173
 174         for pair in mpp_sent:
 175             dict = E.P
 176             if pair[0] == MPPROOT: dict = E.P_r
 177             dict['den'] += 1
 178             if pair in gold_parse: dict['num'] += 1
 179             else:
 180                 try: E.overproposed[pair] += 1
 181                 except KeyError: E.overproposed[pair] = 1
 182
 183     return E
 184
 185
 186
 187 def compare_loc_h_cnf():
 188     reader = WSJDepCorpusReader(None)
 189     corpus_size = 200
 190     corpus_offset = 1000
 191     tagonlys = reader.tagonly_sents()[corpus_offset:corpus_offset+corpus_size]
 192
 193     import loc_h_harmonic, cnf_harmonic
 194     g_l = loc_h_harmonic.initialize(tagonlys)
 195     g_c = cnf_harmonic.initialize(tagonlys)
 196
 197     initials = [
 198          (g_l.p_ROOT.iteritems(), g_c.p_ROOT),
 199          (g_c.p_ROOT.iteritems(), g_l.p_ROOT),
 200          (g_l.p_STOP.iteritems(), g_c.p_STOP),
 201          (g_c.p_STOP.iteritems(), g_l.p_STOP),
 202          (g_l.p_ATTACH.iteritems(), g_c.p_ATTACH),
 203          (g_c.p_ATTACH.iteritems(), g_l.p_ATTACH)]
 204     for a_items, b in initials:
 205         for k,v in a_items:
 206             if k not in b.keys(): raise Warning, "a[%s]=%s, but %s not in b"(k,v,k)
 207             if (k,v) not in b.iteritems(): raise Warning, "a[%s]=%s, but b[%s]=%s"(k,v,k,b[k])
 208
 209
 210     import loc_h_dmv, cnf_dmv
 211     from common_dmv import GOR
 212     for sent in tagonlys:
 213         ochart_l, ochart_c, ichart_l, ichart_c = {},{},{},{}
 214         i_l = loc_h_dmv.inner_sent(g_l, sent, ichart_l)
 215         i_c = cnf_dmv.inner_sent(g_c, sent, ichart_c)
 216         test( "%s"%i_l, "%s"%i_c, "i_l","i_c")
 217
 218         for loc_w,w in enumerate(sent):
 219             w_node = (GOR, g_l.tagnum(w))
 220             o_l = loc_h_dmv.outer(loc_w,loc_w+1,w_node,loc_w, g_l, sent, ichart_l,ochart_l)
 221             o_c =   cnf_dmv.outer(loc_w,loc_w+1,w_node,       g_c, sent, ichart_c,ochart_c)
 222             print "%s, %s, %s"%(sent,node_str(w_node),loc_w)
 223             test("%s"%o_l, "%s"%o_c, "o_l(0,1,(GOR,%s),%d,...)"%(w,loc_w),"o_c")
 224
 225 # end compare_loc_h_cnf()
 226
 227
 228 def init_nothing(g,H,S,N,M):
 229     print '''
 230 HARMONIC_C: %s, STOP_C: %s, NSTOP_C: %s, FSTOP_MIN: %s'''%(H,S,N,M)
 231     return lambda corpus:g
 232
 233 def rnd_grammars_test():
 234     import loc_h_dmv
 235     reload(loc_h_dmv)
 236
 237     rnd_grammars0 = []
 238     for i in xrange(20):
 239         g = test_likelihood(loc_h_dmv.reestimate,
 240                             initialize_loc_h,
 241                             loc_h_dmv.inner_sent,
 242                             corpus_size=6268,
 243                             iterations=0,
 244                             corpus_offset=0,
 245                             EVAL=True)
 246         rnd_grammars0 += [(g, g.HARMONIC_C, g.STOP_C, g.NSTOP_C, g.FSTOP_MIN)]
 247
 248     rnd_grammars1 = [(test_likelihood(loc_h_dmv.reestimate,
 249                                       init_nothing(g,H,S,N,M),
 250                                       loc_h_dmv.inner_sent,
 251                                       corpus_size=6268,
 252                                       iterations=1,
 253                                       corpus_offset=0,
 254                                       EVAL=True),
 255                       H,S,N,M)
 256                     for g,H,S,N,M in rnd_grammars0]
 257     rnd_grammars2 = [(test_likelihood(loc_h_dmv.reestimate,
 258                                       init_nothing(g,H,S,N,M),
 259                                       loc_h_dmv.inner_sent,
 260                                       corpus_size=6268,
 261                                       iterations=1,
 262                                       corpus_offset=0,
 263                                       EVAL=True),
 264                       H,S,N,M)
 265                     for g,H,S,N,M in rnd_grammars1]
 266
 267 if __name__ == "__main__":
 268     print "main.py:"
 269
 270     if False:
 271         rnd_grammars_test()
 272     else:
 273         import loc_h_dmv
 274         reload(loc_h_dmv)
 275         print "\ntrying reestimate v.1 ##############################"
 276         g = test_likelihood(loc_h_dmv.reestimate,
 277                             initialize_loc_h,
 278                             loc_h_dmv.inner_sent,
 279                             corpus_size=6268,
 280                             iterations=50,
 281                             corpus_offset=0,
 282                             EVAL=True)
 283         print g
 284
 285 #         print "\ntrying reestimate v.2 ##############################"
 286 #         g = test_likelihood(loc_h_dmv.reestimate2,
 287 #                             initialize_loc_h,
 288 #                             loc_h_dmv.inner_sent,
 289 #                             corpus_size=5,
 290 #                             iterations=4,
 291 #                             corpus_offset=0)
 292 #         print "main.py: done"
 293 #         print g
 294
 295
 296 #     compare_loc_h_cnf()
 297 #     import cnf_dmv
 298 #     reload(cnf_dmv)
 299 #     print "\ntrying cnf-reestimate ##############################"
 300 #     g = test_likelihood(cnf_dmv.reestimate,
 301 #                         initialize_cnf,
 302 #                         cnf_dmv.inner_sent,
 303 #                         corpus_size=5,
 304 #                         iterations=4)