Modules/expat/xmltok.c

   1 /* Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd
   2    See the file COPYING for copying permission.
   3 */
   4
   5 #ifdef COMPILED_FROM_DSP
   6 #include "winconfig.h"
   7 #elif defined(MACOS_CLASSIC)
   8 #include "macconfig.h"
   9 #elif defined(__amigaos4__)
  10 #include "amigaconfig.h"
  11 #else
  12 #ifdef HAVE_EXPAT_CONFIG_H
  13 #include <expat_config.h>
  14 #endif
  15 #endif /* ndef COMPILED_FROM_DSP */
  16
  17 #include <stddef.h>
  18
  19 #include "expat_external.h"
  20 #include "internal.h"
  21 #include "xmltok.h"
  22 #include "nametab.h"
  23
  24 #ifdef XML_DTD
  25 #define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
  26 #else
  27 #define IGNORE_SECTION_TOK_VTABLE /* as nothing */
  28 #endif
  29
  30 #define VTABLE1 \
  31   { PREFIX(prologTok), PREFIX(contentTok), \
  32     PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
  33   { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
  34   PREFIX(sameName), \
  35   PREFIX(nameMatchesAscii), \
  36   PREFIX(nameLength), \
  37   PREFIX(skipS), \
  38   PREFIX(getAtts), \
  39   PREFIX(charRefNumber), \
  40   PREFIX(predefinedEntityName), \
  41   PREFIX(updatePosition), \
  42   PREFIX(isPublicId)
  43
  44 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
  45
  46 #define UCS2_GET_NAMING(pages, hi, lo) \
  47    (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1 << ((lo) & 0x1F)))
  48
  49 /* A 2 byte UTF-8 representation splits the characters 11 bits between
  50    the bottom 5 and 6 bits of the bytes.  We need 8 bits to index into
  51    pages, 3 bits to add to that index and 5 bits to generate the mask.
  52 */
  53 #define UTF8_GET_NAMING2(pages, byte) \
  54     (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
  55                       + ((((byte)[0]) & 3) << 1) \
  56                       + ((((byte)[1]) >> 5) & 1)] \
  57          & (1 << (((byte)[1]) & 0x1F)))
  58
  59 /* A 3 byte UTF-8 representation splits the characters 16 bits between
  60    the bottom 4, 6 and 6 bits of the bytes.  We need 8 bits to index
  61    into pages, 3 bits to add to that index and 5 bits to generate the
  62    mask.
  63 */
  64 #define UTF8_GET_NAMING3(pages, byte) \
  65   (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
  66                              + ((((byte)[1]) >> 2) & 0xF)] \
  67                        << 3) \
  68                       + ((((byte)[1]) & 3) << 1) \
  69                       + ((((byte)[2]) >> 5) & 1)] \
  70          & (1 << (((byte)[2]) & 0x1F)))
  71
  72 #define UTF8_GET_NAMING(pages, p, n) \
  73   ((n) == 2 \
  74   ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
  75   : ((n) == 3 \
  76      ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
  77      : 0))
  78
  79 /* Detection of invalid UTF-8 sequences is based on Table 3.1B
  80    of Unicode 3.2: http://www.unicode.org/unicode/reports/tr28/
  81    with the additional restriction of not allowing the Unicode
  82    code points 0xFFFF and 0xFFFE (sequences EF,BF,BF and EF,BF,BE).
  83    Implementation details:
  84      (A & 0x80) == 0     means A < 0x80
  85    and
  86      (A & 0xC0) == 0xC0  means A > 0xBF
  87 */
  88
  89 #define UTF8_INVALID2(p) \
  90   ((*p) < 0xC2 || ((p)[1] & 0x80) == 0 || ((p)[1] & 0xC0) == 0xC0)
  91
  92 #define UTF8_INVALID3(p) \
  93   (((p)[2] & 0x80) == 0 \
  94   || \
  95   ((*p) == 0xEF && (p)[1] == 0xBF \
  96     ? \
  97     (p)[2] > 0xBD \
  98     : \
  99     ((p)[2] & 0xC0) == 0xC0) \
 100   || \
 101   ((*p) == 0xE0 \
 102     ? \
 103     (p)[1] < 0xA0 || ((p)[1] & 0xC0) == 0xC0 \
 104     : \
 105     ((p)[1] & 0x80) == 0 \
 106     || \
 107     ((*p) == 0xED ? (p)[1] > 0x9F : ((p)[1] & 0xC0) == 0xC0)))
 108
 109 #define UTF8_INVALID4(p) \
 110   (((p)[3] & 0x80) == 0 || ((p)[3] & 0xC0) == 0xC0 \
 111   || \
 112   ((p)[2] & 0x80) == 0 || ((p)[2] & 0xC0) == 0xC0 \
 113   || \
 114   ((*p) == 0xF0 \
 115     ? \
 116     (p)[1] < 0x90 || ((p)[1] & 0xC0) == 0xC0 \
 117     : \
 118     ((p)[1] & 0x80) == 0 \
 119     || \
 120     ((*p) == 0xF4 ? (p)[1] > 0x8F : ((p)[1] & 0xC0) == 0xC0)))
 121
 122 static int PTRFASTCALL
 123 isNever(const ENCODING *enc, const char *p)
 124 {
 125   return 0;
 126 }
 127
 128 static int PTRFASTCALL
 129 utf8_isName2(const ENCODING *enc, const char *p)
 130 {
 131   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
 132 }
 133
 134 static int PTRFASTCALL
 135 utf8_isName3(const ENCODING *enc, const char *p)
 136 {
 137   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
 138 }
 139
 140 #define utf8_isName4 isNever
 141
 142 static int PTRFASTCALL
 143 utf8_isNmstrt2(const ENCODING *enc, const char *p)
 144 {
 145   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 146 }
 147
 148 static int PTRFASTCALL
 149 utf8_isNmstrt3(const ENCODING *enc, const char *p)
 150 {
 151   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 152 }
 153
 154 #define utf8_isNmstrt4 isNever
 155
 156 static int PTRFASTCALL
 157 utf8_isInvalid2(const ENCODING *enc, const char *p)
 158 {
 159   return UTF8_INVALID2((const unsigned char *)p);
 160 }
 161
 162 static int PTRFASTCALL
 163 utf8_isInvalid3(const ENCODING *enc, const char *p)
 164 {
 165   return UTF8_INVALID3((const unsigned char *)p);
 166 }
 167
 168 static int PTRFASTCALL
 169 utf8_isInvalid4(const ENCODING *enc, const char *p)
 170 {
 171   return UTF8_INVALID4((const unsigned char *)p);
 172 }
 173
 174 struct normal_encoding {
 175   ENCODING enc;
 176   unsigned char type[256];
 177 #ifdef XML_MIN_SIZE
 178   int (PTRFASTCALL *byteType)(const ENCODING *, const char *);
 179   int (PTRFASTCALL *isNameMin)(const ENCODING *, const char *);
 180   int (PTRFASTCALL *isNmstrtMin)(const ENCODING *, const char *);
 181   int (PTRFASTCALL *byteToAscii)(const ENCODING *, const char *);
 182   int (PTRCALL *charMatches)(const ENCODING *, const char *, int);
 183 #endif /* XML_MIN_SIZE */
 184   int (PTRFASTCALL *isName2)(const ENCODING *, const char *);
 185   int (PTRFASTCALL *isName3)(const ENCODING *, const char *);
 186   int (PTRFASTCALL *isName4)(const ENCODING *, const char *);
 187   int (PTRFASTCALL *isNmstrt2)(const ENCODING *, const char *);
 188   int (PTRFASTCALL *isNmstrt3)(const ENCODING *, const char *);
 189   int (PTRFASTCALL *isNmstrt4)(const ENCODING *, const char *);
 190   int (PTRFASTCALL *isInvalid2)(const ENCODING *, const char *);
 191   int (PTRFASTCALL *isInvalid3)(const ENCODING *, const char *);
 192   int (PTRFASTCALL *isInvalid4)(const ENCODING *, const char *);
 193 };
 194
 195 #define AS_NORMAL_ENCODING(enc)   ((const struct normal_encoding *) (enc))
 196
 197 #ifdef XML_MIN_SIZE
 198
 199 #define STANDARD_VTABLE(E) \
 200  E ## byteType, \
 201  E ## isNameMin, \
 202  E ## isNmstrtMin, \
 203  E ## byteToAscii, \
 204  E ## charMatches,
 205
 206 #else
 207
 208 #define STANDARD_VTABLE(E) /* as nothing */
 209
 210 #endif
 211
 212 #define NORMAL_VTABLE(E) \
 213  E ## isName2, \
 214  E ## isName3, \
 215  E ## isName4, \
 216  E ## isNmstrt2, \
 217  E ## isNmstrt3, \
 218  E ## isNmstrt4, \
 219  E ## isInvalid2, \
 220  E ## isInvalid3, \
 221  E ## isInvalid4
 222
 223 static int FASTCALL checkCharRefNumber(int);
 224
 225 #include "xmltok_impl.h"
 226 #include "ascii.h"
 227
 228 #ifdef XML_MIN_SIZE
 229 #define sb_isNameMin isNever
 230 #define sb_isNmstrtMin isNever
 231 #endif
 232
 233 #ifdef XML_MIN_SIZE
 234 #define MINBPC(enc) ((enc)->minBytesPerChar)
 235 #else
 236 /* minimum bytes per character */
 237 #define MINBPC(enc) 1
 238 #endif
 239
 240 #define SB_BYTE_TYPE(enc, p) \
 241   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 242
 243 #ifdef XML_MIN_SIZE
 244 static int PTRFASTCALL
 245 sb_byteType(const ENCODING *enc, const char *p)
 246 {
 247   return SB_BYTE_TYPE(enc, p);
 248 }
 249 #define BYTE_TYPE(enc, p) \
 250  (AS_NORMAL_ENCODING(enc)->byteType(enc, p))
 251 #else
 252 #define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 253 #endif
 254
 255 #ifdef XML_MIN_SIZE
 256 #define BYTE_TO_ASCII(enc, p) \
 257  (AS_NORMAL_ENCODING(enc)->byteToAscii(enc, p))
 258 static int PTRFASTCALL
 259 sb_byteToAscii(const ENCODING *enc, const char *p)
 260 {
 261   return *p;
 262 }
 263 #else
 264 #define BYTE_TO_ASCII(enc, p) (*(p))
 265 #endif
 266
 267 #define IS_NAME_CHAR(enc, p, n) \
 268  (AS_NORMAL_ENCODING(enc)->isName ## n(enc, p))
 269 #define IS_NMSTRT_CHAR(enc, p, n) \
 270  (AS_NORMAL_ENCODING(enc)->isNmstrt ## n(enc, p))
 271 #define IS_INVALID_CHAR(enc, p, n) \
 272  (AS_NORMAL_ENCODING(enc)->isInvalid ## n(enc, p))
 273
 274 #ifdef XML_MIN_SIZE
 275 #define IS_NAME_CHAR_MINBPC(enc, p) \
 276  (AS_NORMAL_ENCODING(enc)->isNameMin(enc, p))
 277 #define IS_NMSTRT_CHAR_MINBPC(enc, p) \
 278  (AS_NORMAL_ENCODING(enc)->isNmstrtMin(enc, p))
 279 #else
 280 #define IS_NAME_CHAR_MINBPC(enc, p) (0)
 281 #define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 282 #endif
 283
 284 #ifdef XML_MIN_SIZE
 285 #define CHAR_MATCHES(enc, p, c) \
 286  (AS_NORMAL_ENCODING(enc)->charMatches(enc, p, c))
 287 static int PTRCALL
 288 sb_charMatches(const ENCODING *enc, const char *p, int c)
 289 {
 290   return *p == c;
 291 }
 292 #else
 293 /* c is an ASCII character */
 294 #define CHAR_MATCHES(enc, p, c) (*(p) == c)
 295 #endif
 296
 297 #define PREFIX(ident) normal_ ## ident
 298 #include "xmltok_impl.c"
 299
 300 #undef MINBPC
 301 #undef BYTE_TYPE
 302 #undef BYTE_TO_ASCII
 303 #undef CHAR_MATCHES
 304 #undef IS_NAME_CHAR
 305 #undef IS_NAME_CHAR_MINBPC
 306 #undef IS_NMSTRT_CHAR
 307 #undef IS_NMSTRT_CHAR_MINBPC
 308 #undef IS_INVALID_CHAR
 309
 310 enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
 311   UTF8_cval1 = 0x00,
 312   UTF8_cval2 = 0xc0,
 313   UTF8_cval3 = 0xe0,
 314   UTF8_cval4 = 0xf0
 315 };
 316
 317 static void PTRCALL
 318 utf8_toUtf8(const ENCODING *enc,
 319             const char **fromP, const char *fromLim,
 320             char **toP, const char *toLim)
 321 {
 322   char *to;
 323   const char *from;
 324   if (fromLim - *fromP > toLim - *toP) {
 325     /* Avoid copying partial characters. */
 326     for (fromLim = *fromP + (toLim - *toP); fromLim > *fromP; fromLim--)
 327       if (((unsigned char)fromLim[-1] & 0xc0) != 0x80)
 328         break;
 329   }
 330   for (to = *toP, from = *fromP; from != fromLim; from++, to++)
 331     *to = *from;
 332   *fromP = from;
 333   *toP = to;
 334 }
 335
 336 static void PTRCALL
 337 utf8_toUtf16(const ENCODING *enc,
 338              const char **fromP, const char *fromLim,
 339              unsigned short **toP, const unsigned short *toLim)
 340 {
 341   unsigned short *to = *toP;
 342   const char *from = *fromP;
 343   while (from != fromLim && to != toLim) {
 344     switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) {
 345     case BT_LEAD2:
 346       *to++ = (unsigned short)(((from[0] & 0x1f) << 6) | (from[1] & 0x3f));
 347       from += 2;
 348       break;
 349     case BT_LEAD3:
 350       *to++ = (unsigned short)(((from[0] & 0xf) << 12)
 351                                | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f));
 352       from += 3;
 353       break;
 354     case BT_LEAD4:
 355       {
 356         unsigned long n;
 357         if (to + 1 == toLim)
 358           goto after;
 359         n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12)
 360             | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
 361         n -= 0x10000;
 362         to[0] = (unsigned short)((n >> 10) | 0xD800);
 363         to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
 364         to += 2;
 365         from += 4;
 366       }
 367       break;
 368     default:
 369       *to++ = *from++;
 370       break;
 371     }
 372   }
 373 after:
 374   *fromP = from;
 375   *toP = to;
 376 }
 377
 378 #ifdef XML_NS
 379 static const struct normal_encoding utf8_encoding_ns = {
 380   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 381   {
 382 #include "asciitab.h"
 383 #include "utf8tab.h"
 384   },
 385   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 386 };
 387 #endif
 388
 389 static const struct normal_encoding utf8_encoding = {
 390   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 391   {
 392 #define BT_COLON BT_NMSTRT
 393 #include "asciitab.h"
 394 #undef BT_COLON
 395 #include "utf8tab.h"
 396   },
 397   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 398 };
 399
 400 #ifdef XML_NS
 401
 402 static const struct normal_encoding internal_utf8_encoding_ns = {
 403   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 404   {
 405 #include "iasciitab.h"
 406 #include "utf8tab.h"
 407   },
 408   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 409 };
 410
 411 #endif
 412
 413 static const struct normal_encoding internal_utf8_encoding = {
 414   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 415   {
 416 #define BT_COLON BT_NMSTRT
 417 #include "iasciitab.h"
 418 #undef BT_COLON
 419 #include "utf8tab.h"
 420   },
 421   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 422 };
 423
 424 static void PTRCALL
 425 latin1_toUtf8(const ENCODING *enc,
 426               const char **fromP, const char *fromLim,
 427               char **toP, const char *toLim)
 428 {
 429   for (;;) {
 430     unsigned char c;
 431     if (*fromP == fromLim)
 432       break;
 433     c = (unsigned char)**fromP;
 434     if (c & 0x80) {
 435       if (toLim - *toP < 2)
 436         break;
 437       *(*toP)++ = (char)((c >> 6) | UTF8_cval2);
 438       *(*toP)++ = (char)((c & 0x3f) | 0x80);
 439       (*fromP)++;
 440     }
 441     else {
 442       if (*toP == toLim)
 443         break;
 444       *(*toP)++ = *(*fromP)++;
 445     }
 446   }
 447 }
 448
 449 static void PTRCALL
 450 latin1_toUtf16(const ENCODING *enc,
 451                const char **fromP, const char *fromLim,
 452                unsigned short **toP, const unsigned short *toLim)
 453 {
 454   while (*fromP != fromLim && *toP != toLim)
 455     *(*toP)++ = (unsigned char)*(*fromP)++;
 456 }
 457
 458 #ifdef XML_NS
 459
 460 static const struct normal_encoding latin1_encoding_ns = {
 461   { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
 462   {
 463 #include "asciitab.h"
 464 #include "latin1tab.h"
 465   },
 466   STANDARD_VTABLE(sb_)
 467 };
 468
 469 #endif
 470
 471 static const struct normal_encoding latin1_encoding = {
 472   { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
 473   {
 474 #define BT_COLON BT_NMSTRT
 475 #include "asciitab.h"
 476 #undef BT_COLON
 477 #include "latin1tab.h"
 478   },
 479   STANDARD_VTABLE(sb_)
 480 };
 481
 482 static void PTRCALL
 483 ascii_toUtf8(const ENCODING *enc,
 484              const char **fromP, const char *fromLim,
 485              char **toP, const char *toLim)
 486 {
 487   while (*fromP != fromLim && *toP != toLim)
 488     *(*toP)++ = *(*fromP)++;
 489 }
 490
 491 #ifdef XML_NS
 492
 493 static const struct normal_encoding ascii_encoding_ns = {
 494   { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
 495   {
 496 #include "asciitab.h"
 497 /* BT_NONXML == 0 */
 498   },
 499   STANDARD_VTABLE(sb_)
 500 };
 501
 502 #endif
 503
 504 static const struct normal_encoding ascii_encoding = {
 505   { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
 506   {
 507 #define BT_COLON BT_NMSTRT
 508 #include "asciitab.h"
 509 #undef BT_COLON
 510 /* BT_NONXML == 0 */
 511   },
 512   STANDARD_VTABLE(sb_)
 513 };
 514
 515 static int PTRFASTCALL
 516 unicode_byte_type(char hi, char lo)
 517 {
 518   switch ((unsigned char)hi) {
 519   case 0xD8: case 0xD9: case 0xDA: case 0xDB:
 520     return BT_LEAD4;
 521   case 0xDC: case 0xDD: case 0xDE: case 0xDF:
 522     return BT_TRAIL;
 523   case 0xFF:
 524     switch ((unsigned char)lo) {
 525     case 0xFF:
 526     case 0xFE:
 527       return BT_NONXML;
 528     }
 529     break;
 530   }
 531   return BT_NONASCII;
 532 }
 533
 534 #define DEFINE_UTF16_TO_UTF8(E) \
 535 static void  PTRCALL \
 536 E ## toUtf8(const ENCODING *enc, \
 537             const char **fromP, const char *fromLim, \
 538             char **toP, const char *toLim) \
 539 { \
 540   const char *from; \
 541   for (from = *fromP; from != fromLim; from += 2) { \
 542     int plane; \
 543     unsigned char lo2; \
 544     unsigned char lo = GET_LO(from); \
 545     unsigned char hi = GET_HI(from); \
 546     switch (hi) { \
 547     case 0: \
 548       if (lo < 0x80) { \
 549         if (*toP == toLim) { \
 550           *fromP = from; \
 551           return; \
 552         } \
 553         *(*toP)++ = lo; \
 554         break; \
 555       } \
 556       /* fall through */ \
 557     case 0x1: case 0x2: case 0x3: \
 558     case 0x4: case 0x5: case 0x6: case 0x7: \
 559       if (toLim -  *toP < 2) { \
 560         *fromP = from; \
 561         return; \
 562       } \
 563       *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
 564       *(*toP)++ = ((lo & 0x3f) | 0x80); \
 565       break; \
 566     default: \
 567       if (toLim -  *toP < 3)  { \
 568         *fromP = from; \
 569         return; \
 570       } \
 571       /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
 572       *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
 573       *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
 574       *(*toP)++ = ((lo & 0x3f) | 0x80); \
 575       break; \
 576     case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
 577       if (toLim -  *toP < 4) { \
 578         *fromP = from; \
 579         return; \
 580       } \
 581       plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
 582       *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
 583       *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
 584       from += 2; \
 585       lo2 = GET_LO(from); \
 586       *(*toP)++ = (((lo & 0x3) << 4) \
 587                    | ((GET_HI(from) & 0x3) << 2) \
 588                    | (lo2 >> 6) \
 589                    | 0x80); \
 590       *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
 591       break; \
 592     } \
 593   } \
 594   *fromP = from; \
 595 }
 596
 597 #define DEFINE_UTF16_TO_UTF16(E) \
 598 static void  PTRCALL \
 599 E ## toUtf16(const ENCODING *enc, \
 600              const char **fromP, const char *fromLim, \
 601              unsigned short **toP, const unsigned short *toLim) \
 602 { \
 603   /* Avoid copying first half only of surrogate */ \
 604   if (fromLim - *fromP > ((toLim - *toP) << 1) \
 605       && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) \
 606     fromLim -= 2; \
 607   for (; *fromP != fromLim && *toP != toLim; *fromP += 2) \
 608     *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
 609 }
 610
 611 #define SET2(ptr, ch) \
 612   (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
 613 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 614 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 615
 616 DEFINE_UTF16_TO_UTF8(little2_)
 617 DEFINE_UTF16_TO_UTF16(little2_)
 618
 619 #undef SET2
 620 #undef GET_LO
 621 #undef GET_HI
 622
 623 #define SET2(ptr, ch) \
 624   (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
 625 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 626 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 627
 628 DEFINE_UTF16_TO_UTF8(big2_)
 629 DEFINE_UTF16_TO_UTF16(big2_)
 630
 631 #undef SET2
 632 #undef GET_LO
 633 #undef GET_HI
 634
 635 #define LITTLE2_BYTE_TYPE(enc, p) \
 636  ((p)[1] == 0 \
 637   ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
 638   : unicode_byte_type((p)[1], (p)[0]))
 639 #define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
 640 #define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
 641 #define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
 642   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
 643 #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
 644   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 645
 646 #ifdef XML_MIN_SIZE
 647
 648 static int PTRFASTCALL
 649 little2_byteType(const ENCODING *enc, const char *p)
 650 {
 651   return LITTLE2_BYTE_TYPE(enc, p);
 652 }
 653
 654 static int PTRFASTCALL
 655 little2_byteToAscii(const ENCODING *enc, const char *p)
 656 {
 657   return LITTLE2_BYTE_TO_ASCII(enc, p);
 658 }
 659
 660 static int PTRCALL
 661 little2_charMatches(const ENCODING *enc, const char *p, int c)
 662 {
 663   return LITTLE2_CHAR_MATCHES(enc, p, c);
 664 }
 665
 666 static int PTRFASTCALL
 667 little2_isNameMin(const ENCODING *enc, const char *p)
 668 {
 669   return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
 670 }
 671
 672 static int PTRFASTCALL
 673 little2_isNmstrtMin(const ENCODING *enc, const char *p)
 674 {
 675   return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
 676 }
 677
 678 #undef VTABLE
 679 #define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 680
 681 #else /* not XML_MIN_SIZE */
 682
 683 #undef PREFIX
 684 #define PREFIX(ident) little2_ ## ident
 685 #define MINBPC(enc) 2
 686 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
 687 #define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
 688 #define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
 689 #define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
 690 #define IS_NAME_CHAR(enc, p, n) 0
 691 #define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
 692 #define IS_NMSTRT_CHAR(enc, p, n) (0)
 693 #define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
 694
 695 #include "xmltok_impl.c"
 696
 697 #undef MINBPC
 698 #undef BYTE_TYPE
 699 #undef BYTE_TO_ASCII
 700 #undef CHAR_MATCHES
 701 #undef IS_NAME_CHAR
 702 #undef IS_NAME_CHAR_MINBPC
 703 #undef IS_NMSTRT_CHAR
 704 #undef IS_NMSTRT_CHAR_MINBPC
 705 #undef IS_INVALID_CHAR
 706
 707 #endif /* not XML_MIN_SIZE */
 708
 709 #ifdef XML_NS
 710
 711 static const struct normal_encoding little2_encoding_ns = {
 712   { VTABLE, 2, 0,
 713 #if BYTEORDER == 1234
 714     1
 715 #else
 716     0
 717 #endif
 718   },
 719   {
 720 #include "asciitab.h"
 721 #include "latin1tab.h"
 722   },
 723   STANDARD_VTABLE(little2_)
 724 };
 725
 726 #endif
 727
 728 static const struct normal_encoding little2_encoding = {
 729   { VTABLE, 2, 0,
 730 #if BYTEORDER == 1234
 731     1
 732 #else
 733     0
 734 #endif
 735   },
 736   {
 737 #define BT_COLON BT_NMSTRT
 738 #include "asciitab.h"
 739 #undef BT_COLON
 740 #include "latin1tab.h"
 741   },
 742   STANDARD_VTABLE(little2_)
 743 };
 744
 745 #if BYTEORDER != 4321
 746
 747 #ifdef XML_NS
 748
 749 static const struct normal_encoding internal_little2_encoding_ns = {
 750   { VTABLE, 2, 0, 1 },
 751   {
 752 #include "iasciitab.h"
 753 #include "latin1tab.h"
 754   },
 755   STANDARD_VTABLE(little2_)
 756 };
 757
 758 #endif
 759
 760 static const struct normal_encoding internal_little2_encoding = {
 761   { VTABLE, 2, 0, 1 },
 762   {
 763 #define BT_COLON BT_NMSTRT
 764 #include "iasciitab.h"
 765 #undef BT_COLON
 766 #include "latin1tab.h"
 767   },
 768   STANDARD_VTABLE(little2_)
 769 };
 770
 771 #endif
 772
 773
 774 #define BIG2_BYTE_TYPE(enc, p) \
 775  ((p)[0] == 0 \
 776   ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
 777   : unicode_byte_type((p)[0], (p)[1]))
 778 #define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
 779 #define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
 780 #define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
 781   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
 782 #define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
 783   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 784
 785 #ifdef XML_MIN_SIZE
 786
 787 static int PTRFASTCALL
 788 big2_byteType(const ENCODING *enc, const char *p)
 789 {
 790   return BIG2_BYTE_TYPE(enc, p);
 791 }
 792
 793 static int PTRFASTCALL
 794 big2_byteToAscii(const ENCODING *enc, const char *p)
 795 {
 796   return BIG2_BYTE_TO_ASCII(enc, p);
 797 }
 798
 799 static int PTRCALL
 800 big2_charMatches(const ENCODING *enc, const char *p, int c)
 801 {
 802   return BIG2_CHAR_MATCHES(enc, p, c);
 803 }
 804
 805 static int PTRFASTCALL
 806 big2_isNameMin(const ENCODING *enc, const char *p)
 807 {
 808   return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
 809 }
 810
 811 static int PTRFASTCALL
 812 big2_isNmstrtMin(const ENCODING *enc, const char *p)
 813 {
 814   return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
 815 }
 816
 817 #undef VTABLE
 818 #define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 819
 820 #else /* not XML_MIN_SIZE */
 821
 822 #undef PREFIX
 823 #define PREFIX(ident) big2_ ## ident
 824 #define MINBPC(enc) 2
 825 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
 826 #define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
 827 #define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
 828 #define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
 829 #define IS_NAME_CHAR(enc, p, n) 0
 830 #define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
 831 #define IS_NMSTRT_CHAR(enc, p, n) (0)
 832 #define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
 833
 834 #include "xmltok_impl.c"
 835
 836 #undef MINBPC
 837 #undef BYTE_TYPE
 838 #undef BYTE_TO_ASCII
 839 #undef CHAR_MATCHES
 840 #undef IS_NAME_CHAR
 841 #undef IS_NAME_CHAR_MINBPC
 842 #undef IS_NMSTRT_CHAR
 843 #undef IS_NMSTRT_CHAR_MINBPC
 844 #undef IS_INVALID_CHAR
 845
 846 #endif /* not XML_MIN_SIZE */
 847
 848 #ifdef XML_NS
 849
 850 static const struct normal_encoding big2_encoding_ns = {
 851   { VTABLE, 2, 0,
 852 #if BYTEORDER == 4321
 853   1
 854 #else
 855   0
 856 #endif
 857   },
 858   {
 859 #include "asciitab.h"
 860 #include "latin1tab.h"
 861   },
 862   STANDARD_VTABLE(big2_)
 863 };
 864
 865 #endif
 866
 867 static const struct normal_encoding big2_encoding = {
 868   { VTABLE, 2, 0,
 869 #if BYTEORDER == 4321
 870   1
 871 #else
 872   0
 873 #endif
 874   },
 875   {
 876 #define BT_COLON BT_NMSTRT
 877 #include "asciitab.h"
 878 #undef BT_COLON
 879 #include "latin1tab.h"
 880   },
 881   STANDARD_VTABLE(big2_)
 882 };
 883
 884 #if BYTEORDER != 1234
 885
 886 #ifdef XML_NS
 887
 888 static const struct normal_encoding internal_big2_encoding_ns = {
 889   { VTABLE, 2, 0, 1 },
 890   {
 891 #include "iasciitab.h"
 892 #include "latin1tab.h"
 893   },
 894   STANDARD_VTABLE(big2_)
 895 };
 896
 897 #endif
 898
 899 static const struct normal_encoding internal_big2_encoding = {
 900   { VTABLE, 2, 0, 1 },
 901   {
 902 #define BT_COLON BT_NMSTRT
 903 #include "iasciitab.h"
 904 #undef BT_COLON
 905 #include "latin1tab.h"
 906   },
 907   STANDARD_VTABLE(big2_)
 908 };
 909
 910 #endif
 911
 912 #undef PREFIX
 913
 914 static int FASTCALL
 915 streqci(const char *s1, const char *s2)
 916 {
 917   for (;;) {
 918     char c1 = *s1++;
 919     char c2 = *s2++;
 920     if (ASCII_a <= c1 && c1 <= ASCII_z)
 921       c1 += ASCII_A - ASCII_a;
 922     if (ASCII_a <= c2 && c2 <= ASCII_z)
 923       c2 += ASCII_A - ASCII_a;
 924     if (c1 != c2)
 925       return 0;
 926     if (!c1)
 927       break;
 928   }
 929   return 1;
 930 }
 931
 932 static void PTRCALL
 933 initUpdatePosition(const ENCODING *enc, const char *ptr,
 934                    const char *end, POSITION *pos)
 935 {
 936   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 937 }
 938
 939 static int
 940 toAscii(const ENCODING *enc, const char *ptr, const char *end)
 941 {
 942   char buf[1];
 943   char *p = buf;
 944   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
 945   if (p == buf)
 946     return -1;
 947   else
 948     return buf[0];
 949 }
 950
 951 static int FASTCALL
 952 isSpace(int c)
 953 {
 954   switch (c) {
 955   case 0x20:
 956   case 0xD:
 957   case 0xA:
 958   case 0x9:
 959     return 1;
 960   }
 961   return 0;
 962 }
 963
 964 /* Return 1 if there's just optional white space or there's an S
 965    followed by name=val.
 966 */
 967 static int
 968 parsePseudoAttribute(const ENCODING *enc,
 969                      const char *ptr,
 970                      const char *end,
 971                      const char **namePtr,
 972                      const char **nameEndPtr,
 973                      const char **valPtr,
 974                      const char **nextTokPtr)
 975 {
 976   int c;
 977   char open;
 978   if (ptr == end) {
 979     *namePtr = NULL;
 980     return 1;
 981   }
 982   if (!isSpace(toAscii(enc, ptr, end))) {
 983     *nextTokPtr = ptr;
 984     return 0;
 985   }
 986   do {
 987     ptr += enc->minBytesPerChar;
 988   } while (isSpace(toAscii(enc, ptr, end)));
 989   if (ptr == end) {
 990     *namePtr = NULL;
 991     return 1;
 992   }
 993   *namePtr = ptr;
 994   for (;;) {
 995     c = toAscii(enc, ptr, end);
 996     if (c == -1) {
 997       *nextTokPtr = ptr;
 998       return 0;
 999     }
1000     if (c == ASCII_EQUALS) {
1001       *nameEndPtr = ptr;
1002       break;
1003     }
1004     if (isSpace(c)) {
1005       *nameEndPtr = ptr;
1006       do {
1007         ptr += enc->minBytesPerChar;
1008       } while (isSpace(c = toAscii(enc, ptr, end)));
1009       if (c != ASCII_EQUALS) {
1010         *nextTokPtr = ptr;
1011         return 0;
1012       }
1013       break;
1014     }
1015     ptr += enc->minBytesPerChar;
1016   }
1017   if (ptr == *namePtr) {
1018     *nextTokPtr = ptr;
1019     return 0;
1020   }
1021   ptr += enc->minBytesPerChar;
1022   c = toAscii(enc, ptr, end);
1023   while (isSpace(c)) {
1024     ptr += enc->minBytesPerChar;
1025     c = toAscii(enc, ptr, end);
1026   }
1027   if (c != ASCII_QUOT && c != ASCII_APOS) {
1028     *nextTokPtr = ptr;
1029     return 0;
1030   }
1031   open = (char)c;
1032   ptr += enc->minBytesPerChar;
1033   *valPtr = ptr;
1034   for (;; ptr += enc->minBytesPerChar) {
1035     c = toAscii(enc, ptr, end);
1036     if (c == open)
1037       break;
1038     if (!(ASCII_a <= c && c <= ASCII_z)
1039         && !(ASCII_A <= c && c <= ASCII_Z)
1040         && !(ASCII_0 <= c && c <= ASCII_9)
1041         && c != ASCII_PERIOD
1042         && c != ASCII_MINUS
1043         && c != ASCII_UNDERSCORE) {
1044       *nextTokPtr = ptr;
1045       return 0;
1046     }
1047   }
1048   *nextTokPtr = ptr + enc->minBytesPerChar;
1049   return 1;
1050 }
1051
1052 static const char KW_version[] = {
1053   ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
1054 };
1055
1056 static const char KW_encoding[] = {
1057   ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
1058 };
1059
1060 static const char KW_standalone[] = {
1061   ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o,
1062   ASCII_n, ASCII_e, '\0'
1063 };
1064
1065 static const char KW_yes[] = {
1066   ASCII_y, ASCII_e, ASCII_s,  '\0'
1067 };
1068
1069 static const char KW_no[] = {
1070   ASCII_n, ASCII_o,  '\0'
1071 };
1072
1073 static int
1074 doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
1075                                                  const char *,
1076                                                  const char *),
1077                int isGeneralTextEntity,
1078                const ENCODING *enc,
1079                const char *ptr,
1080                const char *end,
1081                const char **badPtr,
1082                const char **versionPtr,
1083                const char **versionEndPtr,
1084                const char **encodingName,
1085                const ENCODING **encoding,
1086                int *standalone)
1087 {
1088   const char *val = NULL;
1089   const char *name = NULL;
1090   const char *nameEnd = NULL;
1091   ptr += 5 * enc->minBytesPerChar;
1092   end -= 2 * enc->minBytesPerChar;
1093   if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)
1094       || !name) {
1095     *badPtr = ptr;
1096     return 0;
1097   }
1098   if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
1099     if (!isGeneralTextEntity) {
1100       *badPtr = name;
1101       return 0;
1102     }
1103   }
1104   else {
1105     if (versionPtr)
1106       *versionPtr = val;
1107     if (versionEndPtr)
1108       *versionEndPtr = ptr;
1109     if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
1110       *badPtr = ptr;
1111       return 0;
1112     }
1113     if (!name) {
1114       if (isGeneralTextEntity) {
1115         /* a TextDecl must have an EncodingDecl */
1116         *badPtr = ptr;
1117         return 0;
1118       }
1119       return 1;
1120     }
1121   }
1122   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
1123     int c = toAscii(enc, val, end);
1124     if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
1125       *badPtr = val;
1126       return 0;
1127     }
1128     if (encodingName)
1129       *encodingName = val;
1130     if (encoding)
1131       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
1132     if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
1133       *badPtr = ptr;
1134       return 0;
1135     }
1136     if (!name)
1137       return 1;
1138   }
1139   if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone)
1140       || isGeneralTextEntity) {
1141     *badPtr = name;
1142     return 0;
1143   }
1144   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
1145     if (standalone)
1146       *standalone = 1;
1147   }
1148   else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
1149     if (standalone)
1150       *standalone = 0;
1151   }
1152   else {
1153     *badPtr = val;
1154     return 0;
1155   }
1156   while (isSpace(toAscii(enc, ptr, end)))
1157     ptr += enc->minBytesPerChar;
1158   if (ptr != end) {
1159     *badPtr = ptr;
1160     return 0;
1161   }
1162   return 1;
1163 }
1164
1165 static int FASTCALL
1166 checkCharRefNumber(int result)
1167 {
1168   switch (result >> 8) {
1169   case 0xD8: case 0xD9: case 0xDA: case 0xDB:
1170   case 0xDC: case 0xDD: case 0xDE: case 0xDF:
1171     return -1;
1172   case 0:
1173     if (latin1_encoding.type[result] == BT_NONXML)
1174       return -1;
1175     break;
1176   case 0xFF:
1177     if (result == 0xFFFE || result == 0xFFFF)
1178       return -1;
1179     break;
1180   }
1181   return result;
1182 }
1183
1184 int FASTCALL
1185 XmlUtf8Encode(int c, char *buf)
1186 {
1187   enum {
1188     /* minN is minimum legal resulting value for N byte sequence */
1189     min2 = 0x80,
1190     min3 = 0x800,
1191     min4 = 0x10000
1192   };
1193
1194   if (c < 0)
1195     return 0;
1196   if (c < min2) {
1197     buf[0] = (char)(c | UTF8_cval1);
1198     return 1;
1199   }
1200   if (c < min3) {
1201     buf[0] = (char)((c >> 6) | UTF8_cval2);
1202     buf[1] = (char)((c & 0x3f) | 0x80);
1203     return 2;
1204   }
1205   if (c < min4) {
1206     buf[0] = (char)((c >> 12) | UTF8_cval3);
1207     buf[1] = (char)(((c >> 6) & 0x3f) | 0x80);
1208     buf[2] = (char)((c & 0x3f) | 0x80);
1209     return 3;
1210   }
1211   if (c < 0x110000) {
1212     buf[0] = (char)((c >> 18) | UTF8_cval4);
1213     buf[1] = (char)(((c >> 12) & 0x3f) | 0x80);
1214     buf[2] = (char)(((c >> 6) & 0x3f) | 0x80);
1215     buf[3] = (char)((c & 0x3f) | 0x80);
1216     return 4;
1217   }
1218   return 0;
1219 }
1220
1221 int FASTCALL
1222 XmlUtf16Encode(int charNum, unsigned short *buf)
1223 {
1224   if (charNum < 0)
1225     return 0;
1226   if (charNum < 0x10000) {
1227     buf[0] = (unsigned short)charNum;
1228     return 1;
1229   }
1230   if (charNum < 0x110000) {
1231     charNum -= 0x10000;
1232     buf[0] = (unsigned short)((charNum >> 10) + 0xD800);
1233     buf[1] = (unsigned short)((charNum & 0x3FF) + 0xDC00);
1234     return 2;
1235   }
1236   return 0;
1237 }
1238
1239 struct unknown_encoding {
1240   struct normal_encoding normal;
1241   CONVERTER convert;
1242   void *userData;
1243   unsigned short utf16[256];
1244   char utf8[256][4];
1245 };
1246
1247 #define AS_UNKNOWN_ENCODING(enc)  ((const struct unknown_encoding *) (enc))
1248
1249 int
1250 XmlSizeOfUnknownEncoding(void)
1251 {
1252   return sizeof(struct unknown_encoding);
1253 }
1254
1255 static int PTRFASTCALL
1256 unknown_isName(const ENCODING *enc, const char *p)
1257 {
1258   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
1259   int c = uenc->convert(uenc->userData, p);
1260   if (c & ~0xFFFF)
1261     return 0;
1262   return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF);
1263 }
1264
1265 static int PTRFASTCALL
1266 unknown_isNmstrt(const ENCODING *enc, const char *p)
1267 {
1268   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
1269   int c = uenc->convert(uenc->userData, p);
1270   if (c & ~0xFFFF)
1271     return 0;
1272   return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF);
1273 }
1274
1275 static int PTRFASTCALL
1276 unknown_isInvalid(const ENCODING *enc, const char *p)
1277 {
1278   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
1279   int c = uenc->convert(uenc->userData, p);
1280   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
1281 }
1282
1283 static void PTRCALL
1284 unknown_toUtf8(const ENCODING *enc,
1285                const char **fromP, const char *fromLim,
1286                char **toP, const char *toLim)
1287 {
1288   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
1289   char buf[XML_UTF8_ENCODE_MAX];
1290   for (;;) {
1291     const char *utf8;
1292     int n;
1293     if (*fromP == fromLim)
1294       break;
1295     utf8 = uenc->utf8[(unsigned char)**fromP];
1296     n = *utf8++;
1297     if (n == 0) {
1298       int c = uenc->convert(uenc->userData, *fromP);
1299       n = XmlUtf8Encode(c, buf);
1300       if (n > toLim - *toP)
1301         break;
1302       utf8 = buf;
1303       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
1304                  - (BT_LEAD2 - 2));
1305     }
1306     else {
1307       if (n > toLim - *toP)
1308         break;
1309       (*fromP)++;
1310     }
1311     do {
1312       *(*toP)++ = *utf8++;
1313     } while (--n != 0);
1314   }
1315 }
1316
1317 static void PTRCALL
1318 unknown_toUtf16(const ENCODING *enc,
1319                 const char **fromP, const char *fromLim,
1320                 unsigned short **toP, const unsigned short *toLim)
1321 {
1322   const struct unknown_encoding *uenc = AS_UNKNOWN_ENCODING(enc);
1323   while (*fromP != fromLim && *toP != toLim) {
1324     unsigned short c = uenc->utf16[(unsigned char)**fromP];
1325     if (c == 0) {
1326       c = (unsigned short)
1327           uenc->convert(uenc->userData, *fromP);
1328       *fromP += (AS_NORMAL_ENCODING(enc)->type[(unsigned char)**fromP]
1329                  - (BT_LEAD2 - 2));
1330     }
1331     else
1332       (*fromP)++;
1333     *(*toP)++ = c;
1334   }
1335 }
1336
1337 ENCODING *
1338 XmlInitUnknownEncoding(void *mem,
1339                        int *table,
1340                        CONVERTER convert,
1341                        void *userData)
1342 {
1343   int i;
1344   struct unknown_encoding *e = (struct unknown_encoding *)mem;
1345   for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
1346     ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
1347   for (i = 0; i < 128; i++)
1348     if (latin1_encoding.type[i] != BT_OTHER
1349         && latin1_encoding.type[i] != BT_NONXML
1350         && table[i] != i)
1351       return 0;
1352   for (i = 0; i < 256; i++) {
1353     int c = table[i];
1354     if (c == -1) {
1355       e->normal.type[i] = BT_MALFORM;
1356       /* This shouldn't really get used. */
1357       e->utf16[i] = 0xFFFF;
1358       e->utf8[i][0] = 1;
1359       e->utf8[i][1] = 0;
1360     }
1361     else if (c < 0) {
1362       if (c < -4)
1363         return 0;
1364       e->normal.type[i] = (unsigned char)(BT_LEAD2 - (c + 2));
1365       e->utf8[i][0] = 0;
1366       e->utf16[i] = 0;
1367     }
1368     else if (c < 0x80) {
1369       if (latin1_encoding.type[c] != BT_OTHER
1370           && latin1_encoding.type[c] != BT_NONXML
1371           && c != i)
1372         return 0;
1373       e->normal.type[i] = latin1_encoding.type[c];
1374       e->utf8[i][0] = 1;
1375       e->utf8[i][1] = (char)c;
1376       e->utf16[i] = (unsigned short)(c == 0 ? 0xFFFF : c);
1377     }
1378     else if (checkCharRefNumber(c) < 0) {
1379       e->normal.type[i] = BT_NONXML;
1380       /* This shouldn't really get used. */
1381       e->utf16[i] = 0xFFFF;
1382       e->utf8[i][0] = 1;
1383       e->utf8[i][1] = 0;
1384     }
1385     else {
1386       if (c > 0xFFFF)
1387         return 0;
1388       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
1389         e->normal.type[i] = BT_NMSTRT;
1390       else if (UCS2_GET_NAMING(namePages, c >> 8, c & 0xff))
1391         e->normal.type[i] = BT_NAME;
1392       else
1393         e->normal.type[i] = BT_OTHER;
1394       e->utf8[i][0] = (char)XmlUtf8Encode(c, e->utf8[i] + 1);
1395       e->utf16[i] = (unsigned short)c;
1396     }
1397   }
1398   e->userData = userData;
1399   e->convert = convert;
1400   if (convert) {
1401     e->normal.isName2 = unknown_isName;
1402     e->normal.isName3 = unknown_isName;
1403     e->normal.isName4 = unknown_isName;
1404     e->normal.isNmstrt2 = unknown_isNmstrt;
1405     e->normal.isNmstrt3 = unknown_isNmstrt;
1406     e->normal.isNmstrt4 = unknown_isNmstrt;
1407     e->normal.isInvalid2 = unknown_isInvalid;
1408     e->normal.isInvalid3 = unknown_isInvalid;
1409     e->normal.isInvalid4 = unknown_isInvalid;
1410   }
1411   e->normal.enc.utf8Convert = unknown_toUtf8;
1412   e->normal.enc.utf16Convert = unknown_toUtf16;
1413   return &(e->normal.enc);
1414 }
1415
1416 /* If this enumeration is changed, getEncodingIndex and encodings
1417 must also be changed. */
1418 enum {
1419   UNKNOWN_ENC = -1,
1420   ISO_8859_1_ENC = 0,
1421   US_ASCII_ENC,
1422   UTF_8_ENC,
1423   UTF_16_ENC,
1424   UTF_16BE_ENC,
1425   UTF_16LE_ENC,
1426   /* must match encodingNames up to here */
1427   NO_ENC
1428 };
1429
1430 static const char KW_ISO_8859_1[] = {
1431   ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9,
1432   ASCII_MINUS, ASCII_1, '\0'
1433 };
1434 static const char KW_US_ASCII[] = {
1435   ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I,
1436   '\0'
1437 };
1438 static const char KW_UTF_8[] =  {
1439   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
1440 };
1441 static const char KW_UTF_16[] = {
1442   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
1443 };
1444 static const char KW_UTF_16BE[] = {
1445   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E,
1446   '\0'
1447 };
1448 static const char KW_UTF_16LE[] = {
1449   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E,
1450   '\0'
1451 };
1452
1453 static int FASTCALL
1454 getEncodingIndex(const char *name)
1455 {
1456   static const char * const encodingNames[] = {
1457     KW_ISO_8859_1,
1458     KW_US_ASCII,
1459     KW_UTF_8,
1460     KW_UTF_16,
1461     KW_UTF_16BE,
1462     KW_UTF_16LE,
1463   };
1464   int i;
1465   if (name == NULL)
1466     return NO_ENC;
1467   for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
1468     if (streqci(name, encodingNames[i]))
1469       return i;
1470   return UNKNOWN_ENC;
1471 }
1472
1473 /* For binary compatibility, we store the index of the encoding
1474    specified at initialization in the isUtf16 member.
1475 */
1476
1477 #define INIT_ENC_INDEX(enc) ((int)(enc)->initEnc.isUtf16)
1478 #define SET_INIT_ENC_INDEX(enc, i) ((enc)->initEnc.isUtf16 = (char)i)
1479
1480 /* This is what detects the encoding.  encodingTable maps from
1481    encoding indices to encodings; INIT_ENC_INDEX(enc) is the index of
1482    the external (protocol) specified encoding; state is
1483    XML_CONTENT_STATE if we're parsing an external text entity, and
1484    XML_PROLOG_STATE otherwise.
1485 */
1486
1487
1488 static int
1489 initScan(const ENCODING * const *encodingTable,
1490          const INIT_ENCODING *enc,
1491          int state,
1492          const char *ptr,
1493          const char *end,
1494          const char **nextTokPtr)
1495 {
1496   const ENCODING **encPtr;
1497
1498   if (ptr == end)
1499     return XML_TOK_NONE;
1500   encPtr = enc->encPtr;
1501   if (ptr + 1 == end) {
1502     /* only a single byte available for auto-detection */
1503 #ifndef XML_DTD /* FIXME */
1504     /* a well-formed document entity must have more than one byte */
1505     if (state != XML_CONTENT_STATE)
1506       return XML_TOK_PARTIAL;
1507 #endif
1508     /* so we're parsing an external text entity... */
1509     /* if UTF-16 was externally specified, then we need at least 2 bytes */
1510     switch (INIT_ENC_INDEX(enc)) {
1511     case UTF_16_ENC:
1512     case UTF_16LE_ENC:
1513     case UTF_16BE_ENC:
1514       return XML_TOK_PARTIAL;
1515     }
1516     switch ((unsigned char)*ptr) {
1517     case 0xFE:
1518     case 0xFF:
1519     case 0xEF: /* possibly first byte of UTF-8 BOM */
1520       if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
1521           && state == XML_CONTENT_STATE)
1522         break;
1523       /* fall through */
1524     case 0x00:
1525     case 0x3C:
1526       return XML_TOK_PARTIAL;
1527     }
1528   }
1529   else {
1530     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
1531     case 0xFEFF:
1532       if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
1533           && state == XML_CONTENT_STATE)
1534         break;
1535       *nextTokPtr = ptr + 2;
1536       *encPtr = encodingTable[UTF_16BE_ENC];
1537       return XML_TOK_BOM;
1538     /* 00 3C is handled in the default case */
1539     case 0x3C00:
1540       if ((INIT_ENC_INDEX(enc) == UTF_16BE_ENC
1541            || INIT_ENC_INDEX(enc) == UTF_16_ENC)
1542           && state == XML_CONTENT_STATE)
1543         break;
1544       *encPtr = encodingTable[UTF_16LE_ENC];
1545       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1546     case 0xFFFE:
1547       if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
1548           && state == XML_CONTENT_STATE)
1549         break;
1550       *nextTokPtr = ptr + 2;
1551       *encPtr = encodingTable[UTF_16LE_ENC];
1552       return XML_TOK_BOM;
1553     case 0xEFBB:
1554       /* Maybe a UTF-8 BOM (EF BB BF) */
1555       /* If there's an explicitly specified (external) encoding
1556          of ISO-8859-1 or some flavour of UTF-16
1557          and this is an external text entity,
1558          don't look for the BOM,
1559          because it might be a legal data.
1560       */
1561       if (state == XML_CONTENT_STATE) {
1562         int e = INIT_ENC_INDEX(enc);
1563         if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC
1564             || e == UTF_16LE_ENC || e == UTF_16_ENC)
1565           break;
1566       }
1567       if (ptr + 2 == end)
1568         return XML_TOK_PARTIAL;
1569       if ((unsigned char)ptr[2] == 0xBF) {
1570         *nextTokPtr = ptr + 3;
1571         *encPtr = encodingTable[UTF_8_ENC];
1572         return XML_TOK_BOM;
1573       }
1574       break;
1575     default:
1576       if (ptr[0] == '\0') {
1577         /* 0 isn't a legal data character. Furthermore a document
1578            entity can only start with ASCII characters.  So the only
1579            way this can fail to be big-endian UTF-16 if it it's an
1580            external parsed general entity that's labelled as
1581            UTF-16LE.
1582         */
1583         if (state == XML_CONTENT_STATE && INIT_ENC_INDEX(enc) == UTF_16LE_ENC)
1584           break;
1585         *encPtr = encodingTable[UTF_16BE_ENC];
1586         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1587       }
1588       else if (ptr[1] == '\0') {
1589         /* We could recover here in the case:
1590             - parsing an external entity
1591             - second byte is 0
1592             - no externally specified encoding
1593             - no encoding declaration
1594            by assuming UTF-16LE.  But we don't, because this would mean when
1595            presented just with a single byte, we couldn't reliably determine
1596            whether we needed further bytes.
1597         */
1598         if (state == XML_CONTENT_STATE)
1599           break;
1600         *encPtr = encodingTable[UTF_16LE_ENC];
1601         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1602       }
1603       break;
1604     }
1605   }
1606   *encPtr = encodingTable[INIT_ENC_INDEX(enc)];
1607   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1608 }
1609
1610
1611 #define NS(x) x
1612 #define ns(x) x
1613 #include "xmltok_ns.c"
1614 #undef NS
1615 #undef ns
1616
1617 #ifdef XML_NS
1618
1619 #define NS(x) x ## NS
1620 #define ns(x) x ## _ns
1621
1622 #include "xmltok_ns.c"
1623
1624 #undef NS
1625 #undef ns
1626
1627 ENCODING *
1628 XmlInitUnknownEncodingNS(void *mem,
1629                          int *table,
1630                          CONVERTER convert,
1631                          void *userData)
1632 {
1633   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
1634   if (enc)
1635     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
1636   return enc;
1637 }
1638
1639 #endif /* XML_NS */