ext/fts3/fts3_unicode2.c

   1 /*
   2 ** 2012 May 25
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 */
  13
  14 /*
  15 ** DO NOT EDIT THIS MACHINE GENERATED FILE.
  16 */
  17
  18 #ifndef SQLITE_DISABLE_FTS3_UNICODE
  19 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
  20
  21 #include <assert.h>
  22
  23 /*
  24 ** Return true if the argument corresponds to a unicode codepoint
  25 ** classified as either a letter or a number. Otherwise false.
  26 **
  27 ** The results are undefined if the value passed to this function
  28 ** is less than zero.
  29 */
  30 int sqlite3FtsUnicodeIsalnum(int c){
  31   /* Each unsigned integer in the following array corresponds to a contiguous
  32   ** range of unicode codepoints that are not either letters or numbers (i.e.
  33   ** codepoints for which this function should return 0).
  34   **
  35   ** The most significant 22 bits in each 32-bit value contain the first
  36   ** codepoint in the range. The least significant 10 bits are used to store
  37   ** the size of the range (always at least 1). In other words, the value
  38   ** ((C<<22) + N) represents a range of N codepoints starting with codepoint
  39   ** C. It is not possible to represent a range larger than 1023 codepoints
  40   ** using this format.
  41   */
  42   static const unsigned int aEntry[] = {
  43     0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
  44     0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
  45     0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
  46     0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
  47     0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
  48     0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
  49     0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
  50     0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
  51     0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
  52     0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
  53     0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
  54     0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
  55     0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
  56     0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
  57     0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
  58     0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
  59     0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
  60     0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
  61     0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
  62     0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
  63     0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
  64     0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
  65     0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
  66     0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
  67     0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
  68     0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
  69     0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
  70     0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
  71     0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
  72     0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
  73     0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
  74     0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
  75     0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
  76     0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
  77     0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
  78     0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
  79     0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
  80     0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
  81     0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
  82     0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
  83     0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
  84     0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
  85     0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
  86     0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
  87     0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
  88     0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
  89     0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
  90     0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
  91     0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
  92     0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
  93     0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
  94     0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
  95     0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
  96     0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
  97     0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
  98     0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
  99     0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
 100     0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
 101     0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
 102     0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
 103     0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
 104     0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
 105     0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
 106     0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
 107     0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
 108     0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
 109     0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
 110     0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
 111     0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
 112     0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
 113     0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
 114     0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
 115     0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
 116     0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
 117     0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
 118     0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
 119     0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
 120     0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
 121     0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
 122     0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
 123     0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
 124     0x380400F0,
 125   };
 126   static const unsigned int aAscii[4] = {
 127     0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
 128   };
 129
 130   if( (unsigned int)c<128 ){
 131     return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 );
 132   }else if( (unsigned int)c<(1<<22) ){
 133     unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
 134     int iRes = 0;
 135     int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
 136     int iLo = 0;
 137     while( iHi>=iLo ){
 138       int iTest = (iHi + iLo) / 2;
 139       if( key >= aEntry[iTest] ){
 140         iRes = iTest;
 141         iLo = iTest+1;
 142       }else{
 143         iHi = iTest-1;
 144       }
 145     }
 146     assert( aEntry[0]<key );
 147     assert( key>=aEntry[iRes] );
 148     return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
 149   }
 150   return 1;
 151 }
 152
 153
 154 /*
 155 ** If the argument is a codepoint corresponding to a lowercase letter
 156 ** in the ASCII range with a diacritic added, return the codepoint
 157 ** of the ASCII letter only. For example, if passed 235 - "LATIN
 158 ** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
 159 ** E"). The resuls of passing a codepoint that corresponds to an
 160 ** uppercase letter are undefined.
 161 */
 162 static int remove_diacritic(int c){
 163   unsigned short aDia[] = {
 164         0,  1797,  1848,  1859,  1891,  1928,  1940,  1995,
 165      2024,  2040,  2060,  2110,  2168,  2206,  2264,  2286,
 166      2344,  2383,  2472,  2488,  2516,  2596,  2668,  2732,
 167      2782,  2842,  2894,  2954,  2984,  3000,  3028,  3336,
 168      3456,  3696,  3712,  3728,  3744,  3896,  3912,  3928,
 169      3968,  4008,  4040,  4106,  4138,  4170,  4202,  4234,
 170      4266,  4296,  4312,  4344,  4408,  4424,  4472,  4504,
 171      6148,  6198,  6264,  6280,  6360,  6429,  6505,  6529,
 172     61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
 173     61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
 174     62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
 175     62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
 176     62924, 63050, 63082, 63274, 63390,
 177   };
 178   char aChar[] = {
 179     '\0', 'a',  'c',  'e',  'i',  'n',  'o',  'u',  'y',  'y',  'a',  'c',
 180     'd',  'e',  'e',  'g',  'h',  'i',  'j',  'k',  'l',  'n',  'o',  'r',
 181     's',  't',  'u',  'u',  'w',  'y',  'z',  'o',  'u',  'a',  'i',  'o',
 182     'u',  'g',  'k',  'o',  'j',  'g',  'n',  'a',  'e',  'i',  'o',  'r',
 183     'u',  's',  't',  'h',  'a',  'e',  'o',  'y',  '\0', '\0', '\0', '\0',
 184     '\0', '\0', '\0', '\0', 'a',  'b',  'd',  'd',  'e',  'f',  'g',  'h',
 185     'h',  'i',  'k',  'l',  'l',  'm',  'n',  'p',  'r',  'r',  's',  't',
 186     'u',  'v',  'w',  'w',  'x',  'y',  'z',  'h',  't',  'w',  'y',  'a',
 187     'e',  'i',  'o',  'u',  'y',
 188   };
 189
 190   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
 191   int iRes = 0;
 192   int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
 193   int iLo = 0;
 194   while( iHi>=iLo ){
 195     int iTest = (iHi + iLo) / 2;
 196     if( key >= aDia[iTest] ){
 197       iRes = iTest;
 198       iLo = iTest+1;
 199     }else{
 200       iHi = iTest-1;
 201     }
 202   }
 203   assert( key>=aDia[iRes] );
 204   return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
 205 }
 206
 207
 208 /*
 209 ** Return true if the argument interpreted as a unicode codepoint
 210 ** is a diacritical modifier character.
 211 */
 212 int sqlite3FtsUnicodeIsdiacritic(int c){
 213   unsigned int mask0 = 0x08029FDF;
 214   unsigned int mask1 = 0x000361F8;
 215   if( c<768 || c>817 ) return 0;
 216   return (c < 768+32) ?
 217       (mask0 & (1 << (c-768))) :
 218       (mask1 & (1 << (c-768-32)));
 219 }
 220
 221
 222 /*
 223 ** Interpret the argument as a unicode codepoint. If the codepoint
 224 ** is an upper case character that has a lower case equivalent,
 225 ** return the codepoint corresponding to the lower case version.
 226 ** Otherwise, return a copy of the argument.
 227 **
 228 ** The results are undefined if the value passed to this function
 229 ** is less than zero.
 230 */
 231 int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
 232   /* Each entry in the following array defines a rule for folding a range
 233   ** of codepoints to lower case. The rule applies to a range of nRange
 234   ** codepoints starting at codepoint iCode.
 235   **
 236   ** If the least significant bit in flags is clear, then the rule applies
 237   ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
 238   ** need to be folded). Or, if it is set, then the rule only applies to
 239   ** every second codepoint in the range, starting with codepoint C.
 240   **
 241   ** The 7 most significant bits in flags are an index into the aiOff[]
 242   ** array. If a specific codepoint C does require folding, then its lower
 243   ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
 244   **
 245   ** The contents of this array are generated by parsing the CaseFolding.txt
 246   ** file distributed as part of the "Unicode Character Database". See
 247   ** http://www.unicode.org for details.
 248   */
 249   static const struct TableEntry {
 250     unsigned short iCode;
 251     unsigned char flags;
 252     unsigned char nRange;
 253   } aEntry[] = {
 254     {65, 14, 26},          {181, 64, 1},          {192, 14, 23},
 255     {216, 14, 7},          {256, 1, 48},          {306, 1, 6},
 256     {313, 1, 16},          {330, 1, 46},          {376, 116, 1},
 257     {377, 1, 6},           {383, 104, 1},         {385, 50, 1},
 258     {386, 1, 4},           {390, 44, 1},          {391, 0, 1},
 259     {393, 42, 2},          {395, 0, 1},           {398, 32, 1},
 260     {399, 38, 1},          {400, 40, 1},          {401, 0, 1},
 261     {403, 42, 1},          {404, 46, 1},          {406, 52, 1},
 262     {407, 48, 1},          {408, 0, 1},           {412, 52, 1},
 263     {413, 54, 1},          {415, 56, 1},          {416, 1, 6},
 264     {422, 60, 1},          {423, 0, 1},           {425, 60, 1},
 265     {428, 0, 1},           {430, 60, 1},          {431, 0, 1},
 266     {433, 58, 2},          {435, 1, 4},           {439, 62, 1},
 267     {440, 0, 1},           {444, 0, 1},           {452, 2, 1},
 268     {453, 0, 1},           {455, 2, 1},           {456, 0, 1},
 269     {458, 2, 1},           {459, 1, 18},          {478, 1, 18},
 270     {497, 2, 1},           {498, 1, 4},           {502, 122, 1},
 271     {503, 134, 1},         {504, 1, 40},          {544, 110, 1},
 272     {546, 1, 18},          {570, 70, 1},          {571, 0, 1},
 273     {573, 108, 1},         {574, 68, 1},          {577, 0, 1},
 274     {579, 106, 1},         {580, 28, 1},          {581, 30, 1},
 275     {582, 1, 10},          {837, 36, 1},          {880, 1, 4},
 276     {886, 0, 1},           {902, 18, 1},          {904, 16, 3},
 277     {908, 26, 1},          {910, 24, 2},          {913, 14, 17},
 278     {931, 14, 9},          {962, 0, 1},           {975, 4, 1},
 279     {976, 140, 1},         {977, 142, 1},         {981, 146, 1},
 280     {982, 144, 1},         {984, 1, 24},          {1008, 136, 1},
 281     {1009, 138, 1},        {1012, 130, 1},        {1013, 128, 1},
 282     {1015, 0, 1},          {1017, 152, 1},        {1018, 0, 1},
 283     {1021, 110, 3},        {1024, 34, 16},        {1040, 14, 32},
 284     {1120, 1, 34},         {1162, 1, 54},         {1216, 6, 1},
 285     {1217, 1, 14},         {1232, 1, 88},         {1329, 22, 38},
 286     {4256, 66, 38},        {4295, 66, 1},         {4301, 66, 1},
 287     {7680, 1, 150},        {7835, 132, 1},        {7838, 96, 1},
 288     {7840, 1, 96},         {7944, 150, 8},        {7960, 150, 6},
 289     {7976, 150, 8},        {7992, 150, 8},        {8008, 150, 6},
 290     {8025, 151, 8},        {8040, 150, 8},        {8072, 150, 8},
 291     {8088, 150, 8},        {8104, 150, 8},        {8120, 150, 2},
 292     {8122, 126, 2},        {8124, 148, 1},        {8126, 100, 1},
 293     {8136, 124, 4},        {8140, 148, 1},        {8152, 150, 2},
 294     {8154, 120, 2},        {8168, 150, 2},        {8170, 118, 2},
 295     {8172, 152, 1},        {8184, 112, 2},        {8186, 114, 2},
 296     {8188, 148, 1},        {8486, 98, 1},         {8490, 92, 1},
 297     {8491, 94, 1},         {8498, 12, 1},         {8544, 8, 16},
 298     {8579, 0, 1},          {9398, 10, 26},        {11264, 22, 47},
 299     {11360, 0, 1},         {11362, 88, 1},        {11363, 102, 1},
 300     {11364, 90, 1},        {11367, 1, 6},         {11373, 84, 1},
 301     {11374, 86, 1},        {11375, 80, 1},        {11376, 82, 1},
 302     {11378, 0, 1},         {11381, 0, 1},         {11390, 78, 2},
 303     {11392, 1, 100},       {11499, 1, 4},         {11506, 0, 1},
 304     {42560, 1, 46},        {42624, 1, 24},        {42786, 1, 14},
 305     {42802, 1, 62},        {42873, 1, 4},         {42877, 76, 1},
 306     {42878, 1, 10},        {42891, 0, 1},         {42893, 74, 1},
 307     {42896, 1, 4},         {42912, 1, 10},        {42922, 72, 1},
 308     {65313, 14, 26},
 309   };
 310   static const unsigned short aiOff[] = {
 311    1,     2,     8,     15,    16,    26,    28,    32,
 312    37,    38,    40,    48,    63,    64,    69,    71,
 313    79,    80,    116,   202,   203,   205,   206,   207,
 314    209,   210,   211,   213,   214,   217,   218,   219,
 315    775,   7264,  10792, 10795, 23228, 23256, 30204, 54721,
 316    54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
 317    57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
 318    65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
 319    65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
 320    65514, 65521, 65527, 65528, 65529,
 321   };
 322
 323   int ret = c;
 324
 325   assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
 326
 327   if( c<128 ){
 328     if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
 329   }else if( c<65536 ){
 330     const struct TableEntry *p;
 331     int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
 332     int iLo = 0;
 333     int iRes = -1;
 334
 335     assert( c>aEntry[0].iCode );
 336     while( iHi>=iLo ){
 337       int iTest = (iHi + iLo) / 2;
 338       int cmp = (c - aEntry[iTest].iCode);
 339       if( cmp>=0 ){
 340         iRes = iTest;
 341         iLo = iTest+1;
 342       }else{
 343         iHi = iTest-1;
 344       }
 345     }
 346
 347     assert( iRes>=0 && c>=aEntry[iRes].iCode );
 348     p = &aEntry[iRes];
 349     if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
 350       ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
 351       assert( ret>0 );
 352     }
 353
 354     if( bRemoveDiacritic ) ret = remove_diacritic(ret);
 355   }
 356
 357   else if( c>=66560 && c<66600 ){
 358     ret = c + 40;
 359   }
 360
 361   return ret;
 362 }
 363 #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */
 364 #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */