lib/wind/gen-normalize.py

   1 #!/usr/local/bin/python
   2 # -*- coding: iso-8859-1 -*-
   3
   4 # $Id$
   5
   6 # Copyright (c) 2004 Kungliga Tekniska Högskolan
   7 # (Royal Institute of Technology, Stockholm, Sweden).
   8 # All rights reserved.
   9 #
  10 # Redistribution and use in source and binary forms, with or without
  11 # modification, are permitted provided that the following conditions
  12 # are met:
  13 #
  14 # 1. Redistributions of source code must retain the above copyright
  15 #    notice, this list of conditions and the following disclaimer.
  16 #
  17 # 2. Redistributions in binary form must reproduce the above copyright
  18 #    notice, this list of conditions and the following disclaimer in the
  19 #    documentation and/or other materials provided with the distribution.
  20 #
  21 # 3. Neither the name of the Institute nor the names of its contributors
  22 #    may be used to endorse or promote products derived from this software
  23 #    without specific prior written permission.
  24 #
  25 # THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
  26 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28 # ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
  29 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  30 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  31 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  33 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  34 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35 # SUCH DAMAGE.
  36
  37 import re
  38 import string
  39 import sys
  40
  41 import generate
  42 import UnicodeData
  43 import util
  44
  45 if len(sys.argv) != 4:
  46     print "usage: %s UnicodeData.txt"
  47     " CompositionExclusions-3.2.0.txt out-dir" % sys.argv[0]
  48     sys.exit(1)
  49
  50 ud = UnicodeData.read(sys.argv[1])
  51
  52 def sortedKeys(d):
  53     """Return a sorted list of the keys of a dict"""
  54     keys = d.keys()
  55     keys.sort()
  56     return keys
  57
  58 trans = dict([(k, [re.sub('<[a-zA-Z]+>', '', v[4]), v[0]])
  59               for k,v in ud.items() if v[4]])
  60
  61 maxLength = 0
  62 for v in trans.values():
  63     maxLength = max(maxLength, len(v[0].split()))
  64
  65 normalize_h = generate.Header('%s/normalize_table.h' % sys.argv[3])
  66 normalize_c = generate.Implementation('%s/normalize_table.c' % sys.argv[3])
  67
  68 normalize_h.file.write(
  69 '''
  70 #include <krb5-types.h>
  71
  72 #define MAX_LENGTH_CANON %u
  73
  74 struct translation {
  75   uint32_t key;
  76   unsigned short val_len;
  77   unsigned short val_offset;
  78 };
  79
  80 extern const struct translation _wind_normalize_table[];
  81
  82 extern const uint32_t _wind_normalize_val_table[];
  83
  84 extern const size_t _wind_normalize_table_size;
  85
  86 struct canon_node {
  87   uint32_t val;
  88   unsigned char next_start;
  89   unsigned char next_end;
  90   unsigned short next_offset;
  91 };
  92
  93 extern const struct canon_node _wind_canon_table[];
  94
  95 extern const unsigned short _wind_canon_next_table[];
  96 ''' % maxLength)
  97
  98 normalize_c.file.write(
  99 '''
 100 #include "normalize_table.h"
 101
 102 const struct translation _wind_normalize_table[] = {
 103 ''')
 104
 105 normalizeValTable = []
 106
 107 for k in sortedKeys(trans) :
 108     v = trans[k]
 109     (key, value, description) = k, v[0], v[1]
 110     vec = [int(x, 0x10) for x in value.split()];
 111     offset = util.subList(normalizeValTable, vec)
 112     if not offset:
 113         offset = len(normalizeValTable)
 114         normalizeValTable.extend(vec) # [("0x%s" % i) for i in vec])
 115     normalize_c.file.write("  {0x%x, %u, %u}, /* %s */\n"
 116                            % (key, len(vec), offset, description))
 117
 118 normalize_c.file.write(
 119 '''};
 120
 121 ''')
 122
 123 normalize_c.file.write(
 124     "const size_t _wind_normalize_table_size = %u;\n\n" % len(trans))
 125
 126 normalize_c.file.write("const uint32_t _wind_normalize_val_table[] = {\n")
 127
 128 for v in normalizeValTable:
 129     normalize_c.file.write("  0x%x,\n" % v)
 130
 131 normalize_c.file.write("};\n\n");
 132
 133 exclusions = UnicodeData.read(sys.argv[2])
 134
 135 inv = dict([(''.join(["%05x" % int(x, 0x10) for x in v[4].split(' ')]),
 136              [k, v[0]])
 137             for k,v in ud.items()
 138             if v[4] and not re.search('<[a-zA-Z]+> *', v[4]) and not exclusions.has_key(k)])
 139
 140 table = 0
 141
 142 tables = {}
 143
 144 def createTable():
 145     """add a new table"""
 146     global table, tables
 147     ret = table
 148     table += 1
 149     tables[ret] = [0] + [None] * 16
 150     return ret
 151
 152 def add(table, k, v):
 153     """add an entry (k, v) to table (recursively)"""
 154     if len(k) == 0:
 155         table[0] = v[0]
 156     else:
 157         i = int(k[0], 0x10) + 1
 158         if table[i] == None:
 159             table[i] = createTable()
 160         add(tables[table[i]], k[1:], v)
 161
 162 top = createTable()
 163
 164 for k,v in inv.items():
 165     add(tables[top], k, v)
 166
 167 next_table  = []
 168 tableToNext = {}
 169 tableEnd    = {}
 170 tableStart  = {}
 171
 172 for k in sortedKeys(tables) :
 173     t = tables[k]
 174     tableToNext[k] = len(next_table)
 175     l = t[1:]
 176     start = 0
 177     while start < 16 and l[start] == None:
 178         start += 1
 179     end = 16
 180     while end > start and l[end - 1] == None:
 181         end -= 1
 182     tableStart[k] = start
 183     tableEnd[k]   = end
 184     n = []
 185     for i in range(start, end):
 186         x = l[i]
 187         if x:
 188             n.append(x)
 189         else:
 190             n.append(0)
 191     next_table.extend(n)
 192
 193 normalize_c.file.write("const struct canon_node _wind_canon_table[] = {\n")
 194
 195 for k in sortedKeys(tables) :
 196     t = tables[k]
 197     normalize_c.file.write("  {0x%x, %u, %u, %u},\n" %
 198                            (t[0], tableStart[k], tableEnd[k], tableToNext[k]))
 199
 200 normalize_c.file.write("};\n\n")
 201
 202 normalize_c.file.write("const unsigned short _wind_canon_next_table[] = {\n")
 203
 204 for k in next_table:
 205     normalize_c.file.write("  %u,\n" % k)
 206
 207 normalize_c.file.write("};\n\n")
 208
 209 normalize_h.close()
 210 normalize_c.close()