src/junk.py

   1
   2
   3 import timeit
   4 def adj_f(middle, loc_h):
   5     return middle+.5 == loc_h or middle-.5 == loc_h
   6 def adj_f2(middle, loc_h):
   7     return middle-1 < loc_h < middle+1
   8 def adj_p(middle, loc_h):
   9     return middle == loc_h[0] or middle == loc_h[1]
  10
  11 print "float:"
  12 print timeit.Timer("junk.adj_f(1,0.5)","import junk").timeit(1000000)
  13 print "pair:"
  14 print timeit.Timer("junk.adj_p(1,[0,1])","import junk").timeit(1000000)
  15
  16 print "float2:"
  17 print timeit.Timer("junk.adj_f2(1,0.5)","import junk").timeit(1000000)
  18
  19 # todo: some more testing on the Brown corpus:
  20 #     # first five sentences of the Brown corpus:
  21 #     g_brown = harmonic.initialize([['AT', 'NP-TL', 'NN-TL', 'JJ-TL', 'NN-TL', 'VBD', 'NR', 'AT', 'NN', 'IN', 'NP$', 'JJ', 'NN', 'NN', 'VBD', '``', 'AT', 'NN', "''", 'CS', 'DTI', 'NNS', 'VBD', 'NN', '.'], ['AT', 'NN', 'RBR', 'VBD', 'IN', 'NN', 'NNS', 'CS', 'AT', 'NN-TL', 'JJ-TL', 'NN-TL', ',', 'WDT', 'HVD', 'JJ', 'NN', 'IN', 'AT', 'NN', ',', '``', 'VBZ', 'AT', 'NN', 'CC', 'NNS', 'IN', 'AT', 'NN-TL', 'IN-TL', 'NP-TL', "''", 'IN', 'AT', 'NN', 'IN', 'WDT', 'AT', 'NN', 'BEDZ', 'VBN', '.'], ['AT', 'NP', 'NN', 'NN', 'HVD', 'BEN', 'VBN', 'IN', 'NP-TL', 'JJ-TL', 'NN-TL', 'NN-TL', 'NP', 'NP', 'TO', 'VB', 'NNS', 'IN', 'JJ', '``', 'NNS', "''", 'IN', 'AT', 'JJ', 'NN', 'WDT', 'BEDZ', 'VBN', 'IN', 'NN-TL', 'NP', 'NP', 'NP', '.'], ['``', 'RB', 'AT', 'JJ', 'NN', 'IN', 'JJ', 'NNS', 'BEDZ', 'VBN', "''", ',', 'AT', 'NN', 'VBD', ',', '``', 'IN', 'AT', 'JJ', 'NN', 'IN', 'AT', 'NN', ',', 'AT', 'NN', 'IN', 'NNS', 'CC', 'AT', 'NN', 'IN', 'DT', 'NN', "''", '.'], ['AT', 'NN', 'VBD', 'PPS', 'DOD', 'VB', 'CS', 'AP', 'IN', 'NP$', 'NN', 'CC', 'NN', 'NNS', '``', 'BER', 'JJ', 'CC', 'JJ', 'CC', 'RB', 'JJ', "''", '.'], ['PPS', 'VBD', 'CS', 'NP', 'NNS', 'VB', '``', 'TO', 'HV', 'DTS', 'NNS', 'VBN', 'CC', 'VBN', 'IN', 'AT', 'NN', 'IN', 'VBG', 'CC', 'VBG', 'PPO', "''", '.'], ['AT', 'JJ', 'NN', 'VBD', 'IN', 'AT', 'NN', 'IN', 'AP', 'NNS', ',', 'IN', 'PPO', 'AT', 'NP', 'CC', 'NP-TL', 'NN-TL', 'VBG', 'NNS', 'WDT', 'PPS', 'VBD', '``', 'BER', 'QL', 'VBN', 'CC', 'VB', 'RB', 'VBN', 'NNS', 'WDT', 'VB', 'IN', 'AT', 'JJT', 'NN', 'IN', 'ABX', 'NNS', "''", '.'], ['NN-HL', 'VBN-HL'], ['WRB', ',', 'AT', 'NN', 'VBD', 'PPS', 'VBZ', '``', 'DTS', 'CD', 'NNS', 'MD', 'BE', 'VBN', 'TO', 'VB', 'JJR', 'NN', 'CC', 'VB', 'AT', 'NN', 'IN', 'NN', "''", '.'], ['AT', 'NN-TL', 'VBG-TL', 'NN-TL', ',', 'AT', 'NN', 'VBD', ',', '``', 'BEZ', 'VBG', 'IN', 'VBN', 'JJ', 'NNS', 'CS', 'AT', 'NN', 'IN', 'NN', 'NNS', 'NNS', "''", '.']])
  22 #     # 36:'AT' in g_brown.numtag, 40:'NP-TL'
  23
  24 #     DEBUG = []
  25 #     test_brown = inner(0,2, (LRBAR,36), g_brown, ['AT', 'NP-TL' ,'NN-TL','JJ-TL'], {})
  26 #     if 1 in DEBUG:
  27 #         for r in  g_brown.rules((2,36)) + g_brown.rules((1,36)) + g_brown.rules((0,36)):
  28 #             L = r.L()
  29 #             R = r.R()
  30 #             if head(L) in [36,40,-2] and head(R) in [36,40,-2]:
  31 #                 print r
  32 #     print "Brown-test gives: %.8f" % test_brown
  33
  34
  35
  36     # this will give the tag sequences of all the 6218 Brown corpus
  37     # sentences of length < 7:
  38     # [[tag for (w, tag) in sent]
  39     #  for sent in nltk.corpus.brown.tagged_sents() if len(sent) < 7]
  40
  41
  42
  43 # ##############################
  44 # #          from dmv          #
  45 # ##############################
  46 # def prune2(s,t,LHS,loc_h, ichart,tree):
  47 #     '''TODO '''
  48 #     newichart = {}
  49 #     def prune2_helper(s,t,LHS,loc_h):
  50 #         newichart[(s,t,LHS,loc_h)] = ichart[(s,t,LHS,loc_h)]
  51 #         for d in tree[s,t,LHS,loc_h]:
  52 #             prune2_helper(d[0],d[1],d[2],d[3])
  53
  54 #     prune2_helper(s,t,LHS,loc_h)
  55 #     return newichart
  56
  57 # def prune(s,t,LHS, g, sent_nums, ichart):
  58 #     '''Removes unused subtrees with positive probability from the
  59 #     ichart.
  60
  61 #     Unused := any and all mothers (or grandmothers etc.) have
  62 #     probability 0.0'''
  63 #     def prune_helper(keep,s,t,LHS,loc_h):
  64 #         keep = keep and ichart[(s,t,LHS,loc_h)] > 0.0
  65 #         for rule in g.sent_rules(LHS, sent_nums):
  66 #             L = rule.L()
  67 #             R = rule.R()
  68 #             if R==STOP:
  69 #                 if (s,t,L,loc_h) in ichart:
  70 #                     prune_helper(keep, s,t, L,loc_h)
  71 #             elif L==STOP:
  72 #                 if (s,t,R,loc_h) in ichart:
  73 #                     prune_helper(keep, s,t, R,loc_h)
  74 #             else:
  75 #                 for r in xrange(s,t):
  76 #                     for loc_L in locs(head(L), sent_nums, s, r):
  77 #                         if (s,r,rule.L(),loc_L) in ichart:
  78 #                             prune_helper(keep, s ,r,rule.L(),loc_L)
  79 #                     for loc_R in locs(head(R), sent_nums, r+1, t):
  80 #                         if (r+1,t,rule.R(),loc_R) in ichart:
  81 #                             prune_helper(keep,r+1,t,rule.R(),loc_R)
  82
  83 #         if not (s,t,LHS,loc_h) in keepichart:
  84 #             keepichart[(s,t,LHS,loc_h)] = keep
  85 #         else: # eg. if previously some parent rule had 0.0, but then a
  86 #               # later rule said "No, I've got a use for this subtree"
  87 #             keepichart[(s,t,LHS,loc_h)] += keep
  88 #         return None
  89
  90 #     keepichart = {}
  91 #     for loc_h,h in enumerate(sent_nums):
  92 #         keep = ichart[(s,t,LHS,loc_h)] > 0.0
  93 #         keepichart[(s,t,LHS,loc_h)] = keep
  94 #         prune_helper(keep,s,t,LHS,loc_h)
  95
  96 #     for (s,t,LHS,loc_h),v in keepichart.iteritems():
  97 #         if not v:
  98 #             if  'PRUNE' in DEBUG:
  99 #                 print "popping s:%d t:%d LHS:%s loc_h:%d" % (s,t,LHS,loc_h)
 100 #             ichart.pop((s,t,LHS,loc_h))
 101 # # end prune(s,t,LHS,loc_h, g, sent_nums, ichart)
 102
 103 # def prune_sent(g, sent_nums, ichart):
 104 #     return prune(0, len(sent_nums)-1, ROOT, g, sent_nums, ichart)
 105
 106
 107
 108 # def P_STOP(STOP, h, dir, adj, g, corpus):
 109 #     P_STOP_num = 0
 110 #     P_STOP_den = 0
 111 #     h_tag = g.numtag(h)
 112 #     for sent in corpus:
 113 #         chart = {}
 114 #         locs_h = locs(h_tag, sent)
 115 #         io.debug( "locs_h:%s, sent:%s"%(locs_h,sent) , 'PSTOP')
 116 #         for loc_h in locs_h:
 117 #             inner(0, len(sent)-1, ROOT, loc_h, g, sent, chart)
 118 #             for s in range(loc_h): # s<loc(h), range gives strictly less
 119 #                 for t in range(loc_h, len(sent)):
 120 #                     io.debug( "s:%s t:%s loc:%d"%(s,t,loc_h)  , 'PSTOP')
 121 #                     if (s, t, (LRBAR,h), loc_h) in chart:
 122 #                         io.debug( "num+=%s"%chart[(s, t, (LRBAR,h), loc_h)]  , 'PSTOP')
 123 #                         P_STOP_num += chart[(s, t, (LRBAR,h), loc_h)]
 124 #                     if (s, t, (RBAR,h), loc_h) in chart:
 125 #                         io.debug( "den+=%s"%chart[(s, t, (RBAR,h), loc_h)]  , 'PSTOP')
 126 #                         P_STOP_den += chart[(s, t, (RBAR,h), loc_h)]
 127
 128 #     io.debug( "num/den: %s / %s"%(P_STOP_num, P_STOP_den) , 'PSTOP')
 129 #     if P_STOP_den > 0.0:
 130 #         io.debug( "num/den: %s / %s = %s"%(P_STOP_num, P_STOP_den,P_STOP_num / P_STOP_den) , 'PSTOP')
 131 #         return P_STOP_num / P_STOP_den # upside down in article
 132 #     else:
 133 #         return 0.0
 134
 135
 136 # def DMV(sent, g):
 137 #     '''Here it seems like they store rule information on a per-head (per
 138 #     direction) basis, in deps_D(h, dir) which gives us a list. '''
 139 #     def P_h(h):
 140 #         P_h = 1 # ?
 141 #         for dir in ['l', 'r']:
 142 #             for a in deps(h, dir):
 143 #                 # D(a)??
 144 #                 P_h *= \
 145 #                     P_STOP (0, h, dir, adj) * \
 146 #                     P_CHOOSE (a, h, dir) * \
 147 #                     P_h(D(a)) * \
 148 #                 P_STOP (STOP | h, dir, adj)
 149 #         return P_h
 150 #     return P_h(root(sent))
 151
 152
 153 # if __name__ == "__main__": # from dmv.py
 154 #     # these are not Real rules, just testing the classes. todo: make
 155 #     # a rule-set to test inner() on.
 156 #     b = {}
 157 #     s   = DMV_Rule((LRBAR,0), (NOBAR,1),(NOBAR,2), 1.0, 0.0) # s->np vp
 158 #     np  = DMV_Rule((NOBAR,1), (NOBAR,3),(NOBAR,4), 0.3, 0.0) # np->n p
 159 #     b[(NOBAR,1), 'n'] = 0.7 # np->'n'
 160 #     b[(NOBAR,3), 'n'] = 1.0 # n->'n'
 161 #     b[(NOBAR,4), 'p'] = 1.0 # p->'p'
 162 #     vp  = DMV_Rule((NOBAR,2), (NOBAR,5),(NOBAR,1), 0.1, 0.0) # vp->v np (two parses use this rule)
 163 #     vp2 = DMV_Rule((NOBAR,2), (NOBAR,2),(NOBAR,4), 0.9, 0.0) # vp->vp p
 164 #     b[(NOBAR,5), 'v'] = 1.0 # v->'v'
 165
 166 #     g = DMV_Grammar([s,np,vp,vp2], b, "todo","todo", "todo")
 167
 168 #     DEBUG = 0
 169 #     test1 = io.inner(0,0, (NOBAR,1), g, ['n'], {})
 170 #     if test1[0] != 0.7:
 171 #         print "should be 0.70 : %.2f" % test1[0]
 172 #         print ""
 173
 174 #     test2 = io.inner(0,2, (NOBAR,2), g, ['v','n','p'], {})
 175 #     if "%.2f" % test2[0] != "0.09": # 0.092999 etc, don't care about that
 176 #         print "should be 0.09 if the io.py-test is right : %.2f" % test2[0]
 177 #     # the following should manage to look stuff up in the chart:
 178 #     test2 = io.inner(0,2, (NOBAR,2), g, ['v','n','p'], test2[1])
 179 #     if "%.2f" % test2[0] != "0.09":
 180 #         print "should be 0.09 if the io.py-test is right : %.2f" % test2[0]