src/loc_h_harmonic.py

   1 # loc_h_harmonic.py, initialization for loc_h_dmv.py
   2
   3 from loc_h_dmv import * # better way to do this?
   4
   5 # todo: tweak these
   6 HARMONIC_C = 0.0
   7 FNONSTOP_MIN = 25
   8 FSTOP_MIN = 5
   9
  10 RIGHT_FIRST = 1.0 # apparently right-first is best for DMV-only
  11
  12 ##############################
  13 #      Initialization        #
  14 ##############################
  15 def taglist(corpus):
  16     '''sents is of this form:
  17 [['tag', ...], ['tag2', ...], ...]
  18
  19 Return a list of the tags. (Has to be ordered for enumerating to be
  20 consistent.)
  21
  22 Fortunately only has to run once.
  23 '''
  24     tagset = set()
  25     for sent in corpus:
  26         for tag in sent:
  27             tagset.add(tag)
  28     if 'ROOT' in tagset:
  29         raise ValueError("it seems we must have a new ROOT symbol")
  30     return list(tagset)
  31
  32
  33
  34
  35
  36 def init_zeros(tags):
  37     '''Return a frequency dictionary with DMV-relevant keys set to 0 or
  38     {}.
  39
  40     Todo: tweak (especially for f_STOP).'''
  41     f = {}
  42     for tag in tags:
  43         f['ROOT', tag] = 0
  44         f['sum', 'ROOT'] = 0
  45         for dir in [LEFT, RIGHT]:
  46             for adj in [ADJ, NON]:
  47                 f[tag, 'STOP', dir, adj] = FSTOP_MIN
  48                 f[tag, '-STOP', dir, adj] = FNONSTOP_MIN
  49         f[tag, RIGHT] = {}
  50         f[tag, LEFT] = {}
  51         f[tag, 'sum', RIGHT] = 0.0
  52         f[tag, 'sum', LEFT] = 0.0
  53     return f
  54
  55 def init_freq(corpus, tags):
  56     '''Returns f, a dictionary with these types of keys:
  57     - ('ROOT', tag) is basically just the frequency of tag
  58     - (tag, 'STOP', 'LN') is for P_STOP(STOP|tag, left, non_adj);
  59       etc. for 'RN', 'LA', 'LN', '-STOP'.
  60     - (tag, LEFT) is a dictionary of arg:f, where head could take arg
  61       to direction LEFT (etc. for RIGHT) and f is "harmonically" divided
  62       by distance, used for finding P_CHOOSE
  63
  64       Does this stuff:
  65       1. counts word frequencies for f_ROOT
  66       2. adds to certain f_STOP counters if a word is found first,
  67          last, first or second, or last or second to last in the sentence
  68          (Left Adjacent, Left Non-Adjacent, etc)
  69       3. adds to f_CHOOSE(arg|head) a "harmonic" number (divided by
  70          distance between arg and head)
  71     '''
  72     f = init_zeros(tags)
  73
  74     for sent in corpus: # sent is ['VBD', 'NN', ...]
  75         n = len(sent)
  76         # NOTE: head in DMV_Rule is a number, while this is the string
  77         for loc_h, head in enumerate(sent):
  78             # todo grok: how is this different from just using straight head
  79             # frequency counts, for the ROOT probabilities?
  80             f['ROOT', head] += 1
  81             f['sum', 'ROOT'] += 1
  82
  83             # True = 1, False = 0. todo: make prettier
  84             f[head,  'STOP',  LEFT,NON] += (loc_h == 1)     # second word
  85             f[head, '-STOP',  LEFT,NON] += (not loc_h == 1) # not second
  86             f[head,  'STOP',  LEFT,ADJ] += (loc_h == 0)     # first word
  87             f[head, '-STOP',  LEFT,ADJ] += (not loc_h == 0) # not first
  88             f[head,  'STOP', RIGHT,NON] += (loc_h == n - 2)     # second-to-last
  89             f[head, '-STOP', RIGHT,NON] += (not loc_h == n - 2) # not second-to-last
  90             f[head,  'STOP', RIGHT,ADJ] += (loc_h == n - 1)     # last word
  91             f[head, '-STOP', RIGHT,ADJ] += (not loc_h == n - 1) # not last
  92
  93             # this is where we make the "harmonic" distribution. quite.
  94             for loc_a, arg in enumerate(sent):
  95                 if loc_h != loc_a:
  96                     harmony = 1.0/abs(loc_h - loc_a) + HARMONIC_C
  97                     if loc_h > loc_a:
  98                         dir = LEFT
  99                     else:
 100                         dir = RIGHT
 101                     if arg not in f[head, dir]:
 102                         f[head, dir][arg] = 0.0
 103                     f[head, dir][arg] += harmony
 104                     f[head, 'sum', dir] += harmony
 105                 # todo, optimization: possible to do both directions
 106                 # at once here, and later on rule out the ones we've
 107                 # done? does it actually speed things up?
 108
 109     return f
 110
 111 def init_normalize(f, tags, numtag, tagnum):
 112     '''Use frequencies (and sums) in f to return create p_STOP, p_ATTACH
 113     and p_GO_AT (which is (1-p_STOP)*p_ATTACH).
 114
 115     Return a usable DMV_Grammar2.'''
 116     p_rules = []
 117     p_STOP, p_ROOT, p_ATTACH, p_ORDER = {},{},{},{}
 118     for h, head in numtag.iteritems():
 119         p_ROOT[h] = float(f['ROOT', head]) / f['sum', 'ROOT']
 120
 121         # p_STOP = STOP / (STOP + NOT_STOP)
 122         for dir in [LEFT,RIGHT]:
 123             for adj in [NON,ADJ]:
 124                 p_STOP[h, dir, adj] = \
 125                     float(f[head, 'STOP', dir, adj]) / \
 126                     (f[head, 'STOP', dir, adj] + f[head, '-STOP', dir, adj])
 127
 128         p_ORDER[GOR, h] = RIGHT_FIRST
 129         p_ORDER[GOL, h] = 1 - RIGHT_FIRST
 130
 131         for dir in [LEFT, RIGHT]:
 132             for arg, val in f[head, dir].iteritems():
 133                 p_ATTACH[tagnum[arg], h, dir] = float(val) / f[head,'sum',dir]
 134
 135     return DMV_Grammar(numtag, tagnum, p_ROOT, p_STOP, p_ATTACH, p_ORDER)
 136
 137 def initialize(corpus):
 138     ''' corpus is a list of lists of tags.'''
 139     tags = taglist(corpus)
 140     numtag, tagnum = {}, {}
 141     for num, tag in enumerate(tags):
 142         tagnum[tag] = num
 143         numtag[num] = tag
 144     # f: frequency counts used in initialization, mostly distances
 145     f = init_freq(corpus, tags)
 146     g = init_normalize(f, tags, numtag, tagnum)
 147     return g
 148
 149
 150
 151
 152 ##############################
 153 #     testing functions:     #
 154 ##############################
 155
 156 if __name__ == "__main__":
 157     print "--------initialization testing------------"
 158     g2 = initialize([['foo', 'two','foo','foo'],
 159                      ['zero', 'one','two','three']])
 160     print g2
 161
 162
 163 ##### todo: grok why there's so little difference in probN and probA values
 164 #     for (n,s) in [(95,5),(5,5)]:
 165 #         FNONSTOP_MIN = n
 166 #         FSTOP_MIN = s
 167
 168 #         testcorpus = [s.split() for s in ['det nn vbd c nn vbd nn','det nn vbd c nn vbd pp nn',
 169 #                                           'det nn vbd nn','det nn vbd c nn vbd pp nn',
 170 #                                           'det nn vbd nn','det nn vbd c nn vbd pp nn',
 171 #                                           'det nn vbd nn','det nn vbd c nn vbd pp nn',
 172 #                                           'det nn vbd nn','det nn vbd c nn vbd pp nn',
 173 #                                           'det nn vbd pp nn','det nn vbd det nn', ]]
 174 #         g = initialize(testcorpus)
 175
 176
 177
 178
 179 def tagset_brown():
 180     "472 tags, takes a while to extract with tagset(), hardcoded here."
 181     return set(['BEDZ-NC', 'NP$', 'AT-TL', 'CS', 'NP+HVZ', 'IN-TL-HL', 'NR-HL', 'CC-TL-HL', 'NNS$-HL', 'JJS-HL', 'JJ-HL', 'WRB-TL', 'JJT-TL', 'WRB', 'DOD*', 'BER*-NC', ')-HL', 'NPS$-HL', 'RB-HL', 'FW-PPSS', 'NP+HVZ-NC', 'NNS$', '--', 'CC-TL', 'FW-NN-TL', 'NP-TL-HL', 'PPSS+MD', 'NPS', 'RBR+CS', 'DTI', 'NPS-TL', 'BEM', 'FW-AT+NP-TL', 'EX+BEZ', 'BEG', 'BED', 'BEZ', 'DTX', 'DOD*-TL', 'FW-VB-NC', 'DTS', 'DTS+BEZ', 'QL-HL', 'NP$-TL', 'WRB+DOD*', 'JJR+CS', 'NN+MD', 'NN-TL-HL', 'HVD-HL', 'NP+BEZ-NC', 'VBN+TO', '*-TL', 'WDT-HL', 'MD', 'NN-HL', 'FW-BE', 'DT$', 'PN-TL', 'DT-HL', 'FW-NR-TL', 'VBG', 'VBD', 'VBN', 'DOD', 'FW-VBG-TL', 'DOZ', 'ABN-TL', 'VB+JJ-NC', 'VBZ', 'RB+CS', 'FW-PN', 'CS-NC', 'VBG-NC', 'BER-HL', 'MD*', '``', 'WPS-TL', 'OD-TL', 'PPSS-HL', 'PPS+MD', 'DO*', 'DO-HL', 'HVG-HL', 'WRB-HL', 'JJT', 'JJS', 'JJR', 'HV+TO', 'WQL', 'DOD-NC', 'CC-HL', 'FW-PPSS+HV', 'FW-NP-TL', 'MD+TO', 'VB+IN', 'JJT-NC', 'WDT+BEZ-TL', '---HL', 'PN$', 'VB+PPO', 'BE-TL', 'VBG-TL', 'NP$-HL', 'VBZ-TL', 'UH', 'FW-WPO', 'AP+AP-NC', 'FW-IN', 'NRS-TL', 'ABL', 'ABN', 'TO-TL', 'ABX', '*-HL', 'FW-WPS', 'VB-NC', 'HVD*', 'PPS+HVD', 'FW-IN+AT', 'FW-NP', 'QLP', 'FW-NR', 'FW-NN', 'PPS+HVZ', 'NNS-NC', 'DT+BEZ-NC', 'PPO', 'PPO-NC', 'EX-HL', 'AP$', 'OD-NC', 'RP', 'WPS+BEZ', 'NN+BEZ', '.-TL', ',', 'FW-DT+BEZ', 'RB', 'FW-PP$-NC', 'RN', 'JJ$-TL', 'MD-NC', 'VBD-NC', 'PPSS+BER-N', 'RB+BEZ-NC', 'WPS-HL', 'VBN-NC', 'BEZ-HL', 'PPL-NC', 'BER-TL', 'PP$$', 'NNS+MD', 'PPS-NC', 'FW-UH-NC', 'PPS+BEZ-NC', 'PPSS+BER-TL', 'NR-NC', 'FW-JJ', 'PPS+BEZ-HL', 'NPS$', 'RB-TL', 'VB-TL', 'BEM*', 'MD*-HL', 'FW-CC', 'NP+MD', 'EX+HVZ', 'FW-CD', 'EX+HVD', 'IN-HL', 'FW-CS', 'JJR-HL', 'FW-IN+NP-TL', 'JJ-TL-HL', 'FW-UH', 'EX', 'FW-NNS-NC', 'FW-JJ-NC', 'VBZ-HL', 'VB+RP', 'BEZ-NC', 'PPSS+HV-TL', 'HV*', 'IN', 'PP$-NC', 'NP-NC', 'BEN', 'PP$-TL', 'FW-*-TL', 'FW-OD-TL', 'WPS', 'WPO', 'MD+PPSS', 'WDT+BER', 'WDT+BEZ', 'CD-HL', 'WDT+BEZ-NC', 'WP$', 'DO+PPSS', 'HV-HL', 'DT-NC', 'PN-NC', 'FW-VBZ', 'HVD', 'HVG', 'NN+BEZ-TL', 'HVZ', 'FW-VBD', 'FW-VBG', 'NNS$-TL', 'JJ-TL', 'FW-VBN', 'MD-TL', 'WDT+DOD', 'HV-TL', 'NN-TL', 'PPSS', 'NR$', 'BER', 'FW-VB', 'DT', 'PN+BEZ', 'VBG-HL', 'FW-PPL+VBZ', 'FW-NPS-TL', 'RB$', 'FW-IN+NN', 'FW-CC-TL', 'RBT', 'RBR', 'PPS-TL', 'PPSS+HV', 'JJS-TL', 'NPS-HL', 'WPS+BEZ-TL', 'NNS-TL-HL', 'VBN-TL-NC', 'QL-TL', 'NN+NN-NC', 'JJR-TL', 'NN$-TL', 'FW-QL', 'IN-TL', 'BED-NC', 'NRS', '.-HL', 'QL', 'PP$-HL', 'WRB+BER', 'JJ', 'WRB+BEZ', 'NNS$-TL-HL', 'PPSS+BEZ', '(', 'PPSS+BER', 'DT+MD', 'DOZ-TL', 'PPSS+BEM', 'FW-PP$', 'RB+BEZ-HL', 'FW-RB+CC', 'FW-PPS', 'VBG+TO', 'DO*-HL', 'NR+MD', 'PPLS', 'IN+IN', 'BEZ*', 'FW-PPL', 'FW-PPO', 'NNS-HL', 'NIL', 'HVN', 'PPSS+BER-NC', 'AP-TL', 'FW-DT', '(-HL', 'DTI-TL', 'JJ+JJ-NC', 'FW-RB', 'FW-VBD-TL', 'BER-NC', 'NNS$-NC', 'JJ-NC', 'NPS$-TL', 'VB+VB-NC', 'PN', 'VB+TO', 'AT-TL-HL', 'BEM-NC', 'PPL-TL', 'ABN-HL', 'RB-NC', 'DO-NC', 'BE-HL', 'WRB+IN', 'FW-UH-TL', 'PPO-HL', 'FW-CD-TL', 'TO-HL', 'PPS+BEZ', 'CD$', 'DO', 'EX+MD', 'HVZ-TL', 'TO-NC', 'IN-NC', '.', 'WRB+DO', 'CD-NC', 'FW-PPO+IN', 'FW-NN$-TL', 'WDT+BEZ-HL', 'RP-HL', 'CC', 'NN+HVZ-TL', 'FW-NNS-TL', 'DT+BEZ', 'WPS+HVZ', 'BEDZ*', 'NP-TL', ':-TL', 'NN-NC', 'WPO-TL', 'QL-NC', 'FW-AT+NN-TL', 'WDT+HVZ', '.-NC', 'FW-DTS', 'NP-HL', ':-HL', 'RBR-NC', 'OD-HL', 'BEDZ-HL', 'VBD-TL', 'NPS-NC', ')', 'TO+VB', 'FW-IN+NN-TL', 'PPL', 'PPS', 'PPSS+VB', 'DT-TL', 'RP-NC', 'VB', 'FW-VB-TL', 'PP$', 'VBD-HL', 'DTI-HL', 'NN-TL-NC', 'PPL-HL', 'DOZ*', 'NR-TL', 'WRB+MD', 'PN+HVZ', 'FW-IN-TL', 'PN+HVD', 'BEN-TL', 'BE', 'WDT', 'WPS+HVD', 'DO-TL', 'FW-NN-NC', 'WRB+BEZ-TL', 'UH-TL', 'JJR-NC', 'NNS', 'PPSS-NC', 'WPS+BEZ-NC', ',-TL', 'NN$', 'VBN-TL-HL', 'WDT-NC', 'OD', 'FW-OD-NC', 'DOZ*-TL', 'PPSS+HVD', 'CS-TL', 'WRB+DOZ', 'CC-NC', 'HV', 'NN$-HL', 'FW-WDT', 'WRB+DOD', 'NN+HVZ', 'AT-NC', 'NNS-TL', 'FW-BEZ', 'CS-HL', 'WPO-NC', 'FW-BER', 'NNS-TL-NC', 'BEZ-TL', 'FW-IN+AT-T', 'ABN-NC', 'NR-TL-HL', 'BEDZ', 'NP+BEZ', 'FW-AT-TL', 'BER*', 'WPS+MD', 'MD-HL', 'BED*', 'HV-NC', 'WPS-NC', 'VBN-HL', 'FW-TO+VB', 'PPSS+MD-NC', 'HVZ*', 'PPS-HL', 'WRB-NC', 'VBN-TL', 'CD-TL-HL', ',-NC', 'RP-TL', 'AP-HL', 'FW-HV', 'WQL-TL', 'FW-AT', 'NN', 'NR$-TL', 'VBZ-NC', '*', 'PPSS-TL', 'JJT-HL', 'FW-NNS', 'NP', 'UH-HL', 'NR', ':', 'FW-NN$', 'RP+IN', ',-HL', 'JJ-TL-NC', 'AP-NC', '*-NC', 'VB-HL', 'HVZ-NC', 'DTS-HL', 'FW-JJT', 'FW-JJR', 'FW-JJ-TL', 'FW-*', 'RB+BEZ', "''", 'VB+AT', 'PN-HL', 'PPO-TL', 'CD-TL', 'UH-NC', 'FW-NN-TL-NC', 'EX-NC', 'PPSS+BEZ*', 'TO', 'WDT+DO+PPS', 'IN+PPO', 'AP', 'AT', 'DOZ-HL', 'FW-RB-TL', 'CD', 'NN+IN', 'FW-AT-HL', 'PN+MD', "'", 'FW-PP$-TL', 'FW-NPS', 'WDT+BER+PP', 'NN+HVD-TL', 'MD+HV', 'AT-HL', 'FW-IN+AT-TL'])