lib/wind/gen-normalize.py

   1 #!/usr/local/bin/python
   2 # -*- coding: iso-8859-1 -*-
   3
   4 # $Id$
   5
   6 # Copyright (c) 2004 Kungliga Tekniska Högskolan
   7 # (Royal Institute of Technology, Stockholm, Sweden).
   8 # All rights reserved.
   9 #
  10 # Redistribution and use in source and binary forms, with or without
  11 # modification, are permitted provided that the following conditions
  12 # are met:
  13 #
  14 # 1. Redistributions of source code must retain the above copyright
  15 #    notice, this list of conditions and the following disclaimer.
  16 #
  17 # 2. Redistributions in binary form must reproduce the above copyright
  18 #    notice, this list of conditions and the following disclaimer in the
  19 #    documentation and/or other materials provided with the distribution.
  20 #
  21 # 3. Neither the name of the Institute nor the names of its contributors
  22 #    may be used to endorse or promote products derived from this software
  23 #    without specific prior written permission.
  24 #
  25 # THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND
  26 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28 # ARE DISCLAIMED.  IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE
  29 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  30 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  31 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  33 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  34 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35 # SUCH DAMAGE.
  36
  37 import re
  38 import string
  39 import sys
  40
  41 import generate
  42 import UnicodeData
  43 import util
  44
  45 if len(sys.argv) != 4:
  46     print "usage: %s UnicodeData.txt"
  47     " CompositionExclusions-3.2.0.txt out-dir" % sys.argv[0]
  48     sys.exit(1)
  49
  50 ud = UnicodeData.read(sys.argv[1])
  51
  52 def sortedKeys(d):
  53     """Return a sorted list of the keys of a dict"""
  54     keys = d.keys()
  55     keys.sort()
  56     return keys
  57
  58 trans = dict([(k, [re.sub('<[a-zA-Z]+>', '', v[4]), v[0]])
  59               for k,v in ud.items() if v[4]])
  60
  61 maxLength = 0
  62 for v in trans.values():
  63     maxLength = max(maxLength, len(v[0].split()))
  64
  65 normalize_h = generate.Header('%s/normalize_table.h' % sys.argv[3])
  66 normalize_c = generate.Implementation('%s/normalize_table.c' % sys.argv[3])
  67
  68 normalize_h.file.write(
  69 '''
  70 #include <krb5-types.h>
  71
  72 #define MAX_LENGTH_CANON %u
  73
  74 struct translation {
  75   uint32_t key;
  76   unsigned short val_len;
  77   unsigned short val_offset;
  78 };
  79
  80 extern const struct translation _wind_normalize_table[];
  81
  82 extern const uint32_t _wind_normalize_val_table[];
  83
  84 extern const size_t _wind_normalize_table_size;
  85
  86 struct canon_node {
  87   uint32_t val;
  88   unsigned char next_start;
  89   unsigned char next_end;
  90   unsigned short next_offset;
  91 };
  92
  93 extern const struct canon_node _wind_canon_table[];
  94
  95 extern const unsigned short _wind_canon_next_table[];
  96 ''' % maxLength)
  97
  98 normalize_c.file.write(
  99 '''
 100 #include <stdlib.h>
 101 #include "normalize_table.h"
 102
 103 const struct translation _wind_normalize_table[] = {
 104 ''')
 105
 106 normalizeValTable = []
 107
 108 for k in sortedKeys(trans) :
 109     v = trans[k]
 110     (key, value, description) = k, v[0], v[1]
 111     vec = [int(x, 0x10) for x in value.split()];
 112     offset = util.subList(normalizeValTable, vec)
 113     if not offset:
 114         offset = len(normalizeValTable)
 115         normalizeValTable.extend(vec) # [("0x%s" % i) for i in vec])
 116     normalize_c.file.write("  {0x%x, %u, %u}, /* %s */\n"
 117                            % (key, len(vec), offset, description))
 118
 119 normalize_c.file.write(
 120 '''};
 121
 122 ''')
 123
 124 normalize_c.file.write(
 125     "const size_t _wind_normalize_table_size = %u;\n\n" % len(trans))
 126
 127 normalize_c.file.write("const uint32_t _wind_normalize_val_table[] = {\n")
 128
 129 for v in normalizeValTable:
 130     normalize_c.file.write("  0x%x,\n" % v)
 131
 132 normalize_c.file.write("};\n\n");
 133
 134 exclusions = UnicodeData.read(sys.argv[2])
 135
 136 inv = dict([(''.join(["%05x" % int(x, 0x10) for x in v[4].split(' ')]),
 137              [k, v[0]])
 138             for k,v in ud.items()
 139             if v[4] and not re.search('<[a-zA-Z]+> *', v[4]) and not exclusions.has_key(k)])
 140
 141 table = 0
 142
 143 tables = {}
 144
 145 def createTable():
 146     """add a new table"""
 147     global table, tables
 148     ret = table
 149     table += 1
 150     tables[ret] = [0] + [None] * 16
 151     return ret
 152
 153 def add(table, k, v):
 154     """add an entry (k, v) to table (recursively)"""
 155     if len(k) == 0:
 156         table[0] = v[0]
 157     else:
 158         i = int(k[0], 0x10) + 1
 159         if table[i] == None:
 160             table[i] = createTable()
 161         add(tables[table[i]], k[1:], v)
 162
 163 top = createTable()
 164
 165 for k,v in inv.items():
 166     add(tables[top], k, v)
 167
 168 next_table  = []
 169 tableToNext = {}
 170 tableEnd    = {}
 171 tableStart  = {}
 172
 173 for k in sortedKeys(tables) :
 174     t = tables[k]
 175     tableToNext[k] = len(next_table)
 176     l = t[1:]
 177     start = 0
 178     while start < 16 and l[start] == None:
 179         start += 1
 180     end = 16
 181     while end > start and l[end - 1] == None:
 182         end -= 1
 183     tableStart[k] = start
 184     tableEnd[k]   = end
 185     n = []
 186     for i in range(start, end):
 187         x = l[i]
 188         if x:
 189             n.append(x)
 190         else:
 191             n.append(0)
 192     next_table.extend(n)
 193
 194 normalize_c.file.write("const struct canon_node _wind_canon_table[] = {\n")
 195
 196 for k in sortedKeys(tables) :
 197     t = tables[k]
 198     normalize_c.file.write("  {0x%x, %u, %u, %u},\n" %
 199                            (t[0], tableStart[k], tableEnd[k], tableToNext[k]))
 200
 201 normalize_c.file.write("};\n\n")
 202
 203 normalize_c.file.write("const unsigned short _wind_canon_next_table[] = {\n")
 204
 205 for k in next_table:
 206     normalize_c.file.write("  %u,\n" % k)
 207
 208 normalize_c.file.write("};\n\n")
 209
 210 normalize_h.close()
 211 normalize_c.close()