main.c

   1 /* Alexey's C compiler. */
   2 #include <stdlib.h>
   3 #include <sys/types.h>
   4 #include <sys/stat.h>
   5 #include <fcntl.h>
   6 #include <stdio.h>
   7 #include <errno.h>
   8 #include <unistd.h>
   9 #include <stdarg.h>
  10 #include <stdint.h>
  11 #include <inttypes.h>
  12 #include <string.h>
  13 #include <limits.h>
  14
  15 struct pos {
  16         unsigned int line, column;
  17 };
  18
  19 #ifdef __GNUC__
  20 #define __noreturn      __attribute((noreturn))
  21 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  22 #else
  23 #define __noreturn
  24 #define __printf(a, b)
  25 #endif
  26
  27 static void warning(struct pos *pos, const char *fmt, ...) __printf(2, 3);
  28 static void warning(struct pos *pos, const char *fmt, ...)
  29 {
  30         va_list args;
  31
  32         fprintf(stderr, "%u:%u: warning: ", pos->line, pos->column);
  33         va_start(args, fmt);
  34         vfprintf(stderr, fmt, args);
  35         va_end(args);
  36         fputc('\n', stderr);
  37 }
  38
  39 static void error_exit(struct pos *pos, const char *fmt, ...) __printf(2, 3) __noreturn;
  40 static void error_exit(struct pos *pos, const char *fmt, ...)
  41 {
  42         va_list args;
  43
  44         fprintf(stderr, "%u:%u: error: ", pos->line, pos->column);
  45         va_start(args, fmt);
  46         vfprintf(stderr, fmt, args);
  47         va_end(args);
  48         fputc('\n', stderr);
  49         exit(EXIT_FAILURE);
  50 }
  51
  52 static void perror_exit(const char *fmt, ...) __printf(1, 2) __noreturn;
  53 static void perror_exit(const char *fmt, ...)
  54 {
  55         int old_errno = errno;
  56         va_list args;
  57
  58         fputs("acc: ", stderr);
  59         va_start(args, fmt);
  60         vfprintf(stderr, fmt, args);
  61         va_end(args);
  62         fputs(": ", stderr);
  63         errno = old_errno;
  64         perror(NULL);
  65         exit(EXIT_FAILURE);
  66 }
  67
  68 static void _error_exit(const char *fmt, ...) __printf(1, 2) __noreturn;
  69 static void _error_exit(const char *fmt, ...)
  70 {
  71         va_list args;
  72
  73         fputs("acc: error: ", stderr);
  74         va_start(args, fmt);
  75         vfprintf(stderr, fmt, args);
  76         va_end(args);
  77         fputc('\n', stderr);
  78         exit(EXIT_FAILURE);
  79 }
  80
  81 static void *xmalloc(size_t size)
  82 {
  83         void *p;
  84
  85         p = malloc(size);
  86         if (!p)
  87                 perror_exit("%s: size %zu", __func__, size);
  88         return p;
  89 }
  90
  91 static void *xmemdup(const void *src, size_t n)
  92 {
  93         void *dst;
  94
  95         dst = xmalloc(n);
  96         memcpy(dst, src, n);
  97         return dst;
  98 }
  99
 100 static ssize_t _xread(int fd, void *buf, size_t count)
 101 {
 102         ssize_t rv;
 103
 104         do {
 105                 rv = read(fd, buf, count);
 106         } while (rv == -1 && (errno == EAGAIN || errno == EINTR));
 107         return rv;
 108 }
 109
 110 static void xread(int fd, void *buf, size_t count)
 111 {
 112         while (count > 0) {
 113                 ssize_t rv;
 114
 115                 rv = _xread(fd, buf, count);
 116                 if (rv == -1)
 117                         perror_exit("read fd %d, buf %p, count %zu", fd, buf, count);
 118                 if (rv == 0)
 119                         _error_exit("fd %d truncated, buf %p, count %zu", fd, buf, count);
 120
 121                 buf = (char *)buf + rv;
 122                 count -= rv;
 123         }
 124 }
 125
 126 static void convert_from_utf8(uint8_t *_c, unsigned int _nr_c, uint32_t **c, unsigned int *nr_c)
 127 {
 128         unsigned int i;
 129
 130         if (_nr_c >= 0xffffffff / sizeof(uint32_t))
 131                 _error_exit("integer overflow _nr_c %"PRIu32, _nr_c);
 132
 133         /* At worse all data is ASCII. */
 134         *c = xmalloc(_nr_c * sizeof(uint32_t));
 135         *nr_c = 0;
 136
 137         i = 0;
 138         while (i < _nr_c) {
 139                 static const struct {
 140                         uint8_t mask1, res1;
 141                         uint32_t min;
 142                 } _mask[] = {
 143                         { 0x80, 0x00, 0         }, /* 0xxxxxxx */
 144                         { 0xe0, 0xc0, 0x80      }, /* 110xxxxx 10xxxxxx */
 145                         { 0xf0, 0xe0, 0x800     }, /* 1110xxxx 10xxxxxx 10xxxxxx */
 146                         { 0xf8, 0xf0, 0x10000   }, /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
 147                 };
 148                 unsigned int level, j;
 149                 uint32_t ch;
 150
 151                 level = 0;
 152                 while (level < sizeof(_mask) / sizeof(_mask[0])) {
 153                         if ((_c[i] & _mask[level].mask1) == _mask[level].res1)
 154                                 break;
 155                         level++;
 156                 }
 157                 if (level == sizeof(_mask) / sizeof(_mask[0]))
 158                         _error_exit("invalid UTF-8 octet sequence at %u: %02"PRIx8, i, _c[i]);
 159                 if (i + level >= _nr_c)
 160                         _error_exit("truncated UTF-8 octet sequence at %u: %02"PRIx8, i, _c[i]);
 161                 for (j = 0; j < level; j++) {
 162                         if ((_c[i + j + 1] & 0xc0) != 0x80)
 163                                 _error_exit("invalid UTF-8 octet sequence at %u: %02"PRIx8" ... %02"PRIx8, i + j + 1, _c[i], _c[i + j + 1]);
 164                 }
 165
 166                 ch = _c[i] & ~_mask[level].mask1;
 167                 for (j = 0; j < level; j++)
 168                         ch = (ch << 6) | (_c[i + j + 1] & ~0xc0);
 169
 170                 if (ch < _mask[level].min)
 171                         _error_exit("invalid UTF-8 octet sequence at %u: %02"PRIx8, i, _c[i]);
 172
 173                 i += level + 1;
 174
 175                 (*c)[*nr_c] = ch;
 176                 (*nr_c)++;
 177         }
 178 }
 179
 180 /* LINE SEPARATOR to catch \n. */
 181 #define LS ((uint32_t)0x2028)
 182
 183 static void fix_newline(uint32_t *c, unsigned int *nr_c)
 184 {
 185         unsigned int i;
 186
 187         i = 0;
 188         while (i < *nr_c) {
 189                 if (c[i] == 0x0d && i + 1 < *nr_c && c[i + 1] == 0x0a) {
 190                         memmove(&c[i], &c[i + 1], *nr_c - i - 1);
 191                         (*nr_c)--;
 192                 }
 193                 switch (c[i]) {
 194                 case 0x0d:
 195                 case 0x0a:
 196                 case 0x85:
 197                 case 0x0b:
 198                 case 0x0c:
 199                 case 0x2028:
 200                 case 0x2029:
 201                         c[i] = LS;
 202                 }
 203                 i++;
 204         }
 205 }
 206
 207 static struct pos *line_column(const uint32_t *c, unsigned int nr_c)
 208 {
 209         struct pos *pos;
 210         unsigned int line, column;
 211         unsigned int i;
 212
 213         if (nr_c >= 0xffffffff / sizeof(struct pos))
 214                 _error_exit("integer overflow nr_c %u", nr_c);
 215         pos = xmalloc(nr_c * sizeof(struct pos));
 216
 217         line = 1;
 218         column = 1;
 219         for (i = 0; i < nr_c; i++) {
 220                 pos[i].line = line;
 221                 pos[i].column = column;
 222
 223                 if (c[i] == LS) {
 224                         line++;
 225                         column = 1;
 226                 } else
 227                         column++;
 228         }
 229         return pos;
 230 }
 231
 232 static void warn_trigraph(const uint32_t *c, unsigned int nr_c, struct pos *pos)
 233 {
 234         unsigned int i;
 235
 236         i = 0;
 237         while (i + 2 < nr_c) {
 238                 if (c[i] == '?' && c[i + 1] == '?') {
 239                         switch (c[i + 2]) {
 240                         case '=':case ')':case '!':
 241                         case '(':case '\'':case '>':
 242                         case '/':case '<':case '-':
 243                                 warning(&pos[i], "trigraph sequence ??%c, ignoring", c[i + 2]);
 244                                 i += 3;
 245                                 break;
 246                         default:
 247                                 i++;
 248                         }
 249                 } else
 250                         i++;
 251         }
 252 }
 253
 254 static void delete_backslash_newline(uint32_t *c, unsigned int *nr_c, struct pos *pos)
 255 {
 256         unsigned int i;
 257
 258         i = 0;
 259         while (i + 1 < *nr_c) {
 260                 if (c[i] == '\\' && c[i + 1] == LS) {
 261                         unsigned int nr_to_move = *nr_c - i - 2;
 262
 263                         memmove(&c[i], &c[i + 2], nr_to_move * sizeof(uint32_t));
 264                         memmove(&pos[i], &pos[i + 2], nr_to_move * sizeof(struct pos));
 265                         (*nr_c) -= 2;
 266                 } else
 267                         i++;
 268         }
 269 }
 270
 271 struct pp_token {
 272         struct pp_token *next;
 273         enum pp_token_type {
 274                 PP_TOKEN_IDENTIFIER = UCHAR_MAX + 1,
 275                 PP_TOKEN_NUMBER,
 276                 PP_TOKEN_STRING,
 277                 PP_TOKEN_CHAR,
 278
 279 #define _2(c1, c2)      ((((uint32_t)c1) << 8) | ((uint32_t)c2))
 280 #define _3(c1, c2, c3)  ((((uint32_t)c1) << 16)| (((uint32_t)c2) << 8) | ((uint32_t)c3))
 281                 PP_TOKEN_DOTDOTDOT      = _3('.', '.', '.'),
 282                 PP_TOKEN_DEREFERENCE    = _2('-', '>'),
 283                 PP_TOKEN_SUB_EQ         = _2('-', '='),
 284                 PP_TOKEN_DEC            = _2('-', '-'),
 285                 PP_TOKEN_ADD_EQ         = _2('+', '='),
 286                 PP_TOKEN_INC            = _2('+', '+'),
 287                 PP_TOKEN_AND_EQ         = _2('&', '='),
 288                 PP_TOKEN_AND            = _2('&', '&'),
 289                 PP_TOKEN_MUL_EQ         = _2('*', '='),
 290                 PP_TOKEN_NOT_EQ         = _2('!', '='),
 291                 PP_TOKEN_DIV_EQ         = _2('/', '='),
 292                 PP_TOKEN_REM_EQ         = _2('%', '='),
 293                 PP_TOKEN_LSHIFT_EQ      = _3('<', '<', '='),
 294                 PP_TOKEN_LSHIFT         = _2('<', '<'),
 295                 PP_TOKEN_LEQ            = _2('<', '='),
 296                 PP_TOKEN_RSHIFT_EQ      = _3('>', '>', '='),
 297                 PP_TOKEN_RSHIFT         = _2('>', '>'),
 298                 PP_TOKEN_GEQ            = _2('>', '='),
 299                 PP_TOKEN_EQ             = _2('=', '='),
 300                 PP_TOKEN_XOR_EQ         = _2('^', '='),
 301                 PP_TOKEN_OR_EQ          = _2('|', '='),
 302                 PP_TOKEN_OR             = _2('|', '|'),
 303                 PP_TOKEN_SHARPSHARP     = _2('#', '#'),
 304 #undef _2
 305 #undef _3
 306         } type;
 307         uint32_t *id;   /* string representation, if type is not enough */
 308         struct pos pos;
 309 };
 310
 311 static struct pp_token *pp_token_create(enum pp_token_type type, struct pos *pos)
 312 {
 313         struct pp_token *ppt;
 314
 315         ppt = xmalloc(sizeof(struct pp_token));
 316         ppt->next = NULL;
 317         ppt->type = type;
 318         ppt->id = NULL;
 319         ppt->pos = *pos;
 320         return ppt;
 321 }
 322
 323 /* [start, end) */
 324 static void pp_token_add(struct pp_token *ppt, const uint32_t *c, unsigned int start, unsigned int end)
 325 {
 326         ppt->id = xmemdup(&c[start], (end - start) * sizeof(uint32_t));
 327 }
 328
 329 static void pp_token_free(struct pp_token *ppt_head)
 330 {
 331         struct pp_token *ppt;
 332
 333         ppt = ppt_head;
 334         while (ppt) {
 335                 struct pp_token *next;
 336
 337                 next = ppt->next;
 338                 if (ppt->id)
 339                         free(ppt->id);
 340                 free(ppt);
 341                 ppt = next;
 342         }
 343 }
 344
 345 static void pp_token_print(struct pp_token *ppt)
 346 {
 347         printf("%u:%u:\t", ppt->pos.line, ppt->pos.column);
 348         switch (ppt->type) {
 349         case LS:
 350                 printf("\\n");
 351                 break;
 352         case ' ':
 353                 printf("' '");
 354                 break;
 355         case PP_TOKEN_IDENTIFIER:
 356                 printf("pp-identifier");
 357                 break;
 358         case PP_TOKEN_NUMBER:
 359                 printf("pp-number");
 360                 break;
 361         case PP_TOKEN_STRING:
 362                 printf("pp-string");
 363                 break;
 364         case PP_TOKEN_CHAR:
 365                 printf("pp-char");
 366                 break;
 367         case PP_TOKEN_DOTDOTDOT:
 368         case PP_TOKEN_LSHIFT_EQ:
 369         case PP_TOKEN_RSHIFT_EQ:
 370                 printf("%c%c%c", (ppt->type >> 16) & 0xff, (ppt->type >> 8) & 0xff, ppt->type & 0xff);
 371                 break;
 372         case PP_TOKEN_DEREFERENCE:
 373         case PP_TOKEN_SUB_EQ:
 374         case PP_TOKEN_DEC:
 375         case PP_TOKEN_ADD_EQ:
 376         case PP_TOKEN_INC:
 377         case PP_TOKEN_AND_EQ:
 378         case PP_TOKEN_AND:
 379         case PP_TOKEN_MUL_EQ:
 380         case PP_TOKEN_NOT_EQ:
 381         case PP_TOKEN_DIV_EQ:
 382         case PP_TOKEN_REM_EQ:
 383         case PP_TOKEN_LSHIFT:
 384         case PP_TOKEN_LEQ:
 385         case PP_TOKEN_RSHIFT:
 386         case PP_TOKEN_GEQ:
 387         case PP_TOKEN_EQ:
 388         case PP_TOKEN_XOR_EQ:
 389         case PP_TOKEN_OR_EQ:
 390         case PP_TOKEN_OR:
 391         case PP_TOKEN_SHARPSHARP:
 392                 printf("%c%c", (ppt->type >> 8) & 0xff, ppt->type & 0xff);
 393                 break;
 394         default:
 395                 printf("%c", ppt->type);
 396         }
 397         putc('\n', stdout);
 398 }
 399
 400 static int pp_nondigit(const uint32_t c)
 401 {
 402         return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '_';
 403 }
 404
 405 static int pp_octdigit(const uint32_t c)
 406 {
 407         return '0' <= c && c <= '7';
 408 }
 409
 410 static int pp_digit(const uint32_t c)
 411 {
 412         return '0' <= c && c <= '9';
 413 }
 414
 415 static int pp_hexdigit(const uint32_t c)
 416 {
 417         return pp_digit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F');
 418 }
 419
 420 static unsigned int _pp_identifier_end(const uint32_t *c, unsigned int nr_c, unsigned int start)
 421 {
 422         unsigned int i;
 423
 424         i = start;
 425         while (i < nr_c) {
 426                 if (pp_nondigit(c[i]) || pp_digit(c[i])) {
 427                         i++;
 428                 } else if (i + 5 < nr_c && c[i] == '\\' && c[i + 1] == 'u' &&
 429                            pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) && pp_hexdigit(c[i + 5])) {
 430                         i += 2 + 4;
 431                 } else if (i + 9 < nr_c && c[i] == '\\' && c[i + 1] == 'U' &&
 432                            pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) && pp_hexdigit(c[i + 5]) &&
 433                            pp_hexdigit(c[i + 6]) && pp_hexdigit(c[i + 7]) && pp_hexdigit(c[i + 8]) && pp_hexdigit(c[i + 9])) {
 434                         i += 2 + 4 + 4;
 435                 } else
 436                         return i;
 437         }
 438         return i;
 439 }
 440
 441 static unsigned int pp_number_end(const uint32_t *c, unsigned int nr_c, unsigned int start)
 442 {
 443         unsigned int i;
 444
 445         i = start + 1;
 446         while (i < nr_c) {
 447                 if ((c[i] == 'e' || c[i] == 'E' || c[i] == 'p' || c[i] == 'P') &&
 448                     i + 1 < nr_c && (c[i + 1] == '+' || c[i + 1] == '-')) {
 449                         i += 2;
 450                 } else if (pp_digit(c[i]) || pp_nondigit(c[i]) || c[i] == '.') {
 451                         i++;
 452                 } else if (c[i] == '\\' && i + 5 < nr_c && c[i + 1] == 'u' &&
 453                            pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) && pp_hexdigit(c[i + 5])) {
 454                         i += 2 + 4;
 455                 } else if (c[i] == '\\' && i + 9 < nr_c && c[i + 1] == 'U' &&
 456                            pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) && pp_hexdigit(c[i + 5]) &&
 457                            pp_hexdigit(c[i + 6]) && pp_hexdigit(c[i + 7]) && pp_hexdigit(c[i + 8]) && pp_hexdigit(c[i + 9])) {
 458                         i += 2 + 4 + 4;
 459                 } else
 460                         return i;
 461         }
 462         return i;
 463 }
 464
 465 static unsigned int c_comment_end(const uint32_t *c, unsigned int nr_c, unsigned int start)
 466 {
 467         unsigned int i;
 468
 469         i = start + 2;
 470         while (i + 1 < nr_c) {
 471                 if (c[i] == '*' && c[i + 1] == '/')
 472                         return i + 2;
 473                 i++;
 474         }
 475         return nr_c;
 476 }
 477
 478 static unsigned int cpp_comment_end(const uint32_t *c, unsigned int nr_c, unsigned int start)
 479 {
 480         unsigned int i;
 481
 482         i = start + 2;
 483         while (i < nr_c && c[i] != LS)
 484                 i++;
 485         return i;
 486 }
 487
 488 static unsigned int escape_sequence_end(const uint32_t *c, unsigned int nr_c, unsigned int start, struct pos *_pos)
 489 {
 490         struct pos *pos = &_pos[start];
 491         unsigned int i;
 492
 493         i = start + 1;
 494         if (i >= nr_c)
 495                 error_exit(pos, "incomplete escape sequence");
 496         switch (c[i]) {
 497         case '\'':case '"':case '?':case '\\':
 498         case 'a':case 'b':case 'f':case 'n':case 'r':case 't':case 'v':
 499                 return i + 1;
 500         case '0':case '1':case '2':case '3':case '4':case '6':case '7':
 501                 if (i + 2 < nr_c && pp_octdigit(c[i + 1]) && pp_octdigit(c[i + 2]))
 502                         return i + 3;
 503                 if (i + 1 < nr_c && pp_octdigit(c[i + 1]))
 504                         return i + 2;
 505                 return i + 1;
 506         case 'x':
 507                 i++;
 508                 while (i < nr_c && pp_hexdigit(c[i]))
 509                         i++;
 510                 if (i == start + 2)
 511                         error_exit(pos, "invalid hexadecimal escape sequence");
 512                 return i;
 513         case 'u':
 514                 if (i + 4 < nr_c &&
 515                     pp_hexdigit(c[i + 1]) && pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]))
 516                         return i + 5;
 517                 error_exit(pos, "invalid universal character name");
 518         case 'U':
 519                 if (i + 8 < nr_c &&
 520                     pp_hexdigit(c[i + 1]) && pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) &&
 521                     pp_hexdigit(c[i + 5]) && pp_hexdigit(c[i + 6]) && pp_hexdigit(c[i + 7]) && pp_hexdigit(c[i + 8]))
 522                         return i + 9;
 523                 error_exit(pos, "invalid universal character name");
 524         default:
 525                 error_exit(pos, "invalid escape sequence");
 526         }
 527 }
 528
 529 static unsigned int pp_string_end(const uint32_t *c, unsigned int nr_c, unsigned int start, struct pos *_pos)
 530 {
 531         struct pos *pos = &_pos[start];
 532         unsigned int i;
 533
 534         i = start + 1;
 535         while (i < nr_c && c[i] != '"') {
 536                 switch (c[i]) {
 537                 case LS:
 538                         goto incomplete;
 539                 case '\\':
 540                         i = escape_sequence_end(c, nr_c, i, _pos);
 541                         break;
 542                 default:
 543                         i++;
 544                 }
 545         }
 546         if (i >= nr_c)
 547                 goto incomplete;
 548         return i + 1;
 549
 550 incomplete:
 551         error_exit(pos, "incomplete string literal");
 552 }
 553
 554 static unsigned int pp_char_end(const uint32_t *c, unsigned int nr_c, unsigned int start, struct pos *_pos)
 555 {
 556         struct pos *pos = &_pos[start];
 557         unsigned int i;
 558
 559         i = start + 1;
 560         while (i < nr_c && c[i] != '\'') {
 561                 switch (c[i]) {
 562                 case LS:
 563                         goto incomplete;
 564                 case '\\':
 565                         i = escape_sequence_end(c, nr_c, i, _pos);
 566                         break;
 567                 default:
 568                         i++;
 569                 }
 570         }
 571         if (i >= nr_c)
 572                 goto incomplete;
 573         if (i == start + 1)
 574                 goto empty;
 575         return i + 1;
 576
 577 incomplete:
 578         error_exit(pos, "incomplete character constant");
 579 empty:
 580         error_exit(pos, "empty character constant");
 581 }
 582
 583 static struct pp_token *pp_tokenize(const uint32_t *c, unsigned int nr_c, struct pos *_pos)
 584 {
 585         struct pp_token *ppt_head, *ppt_tail;
 586         unsigned int i;
 587
 588         ppt_head = NULL;
 589         i = 0;
 590         while (i < nr_c) {
 591                 struct pos *pos = &_pos[i];
 592                 struct pp_token *ppt;
 593
 594                 switch (c[i]) {
 595                         unsigned int j;
 596
 597                 case '\t':
 598                 case ' ':
 599                         ppt = pp_token_create(' ', pos);
 600                         i++;
 601                         break;
 602                 case LS:
 603                         ppt = pp_token_create(LS, pos);
 604                         i++;
 605                         break;
 606                 case '[':case ']':
 607                 case '(':case ')':
 608                 case '{':case '}':
 609                 case '~':
 610                 case '?':
 611                 case ':':
 612                 case ';':
 613                 case ',':
 614 pp_token_simple:
 615                         ppt = pp_token_create(c[i], pos);
 616                         i++;
 617                         break;
 618                 case 'a':case 'b':case 'c':case 'd':case 'e':case 'f':case 'g':
 619                 case 'h':case 'i':case 'j':case 'k':case 'l':case 'm':case 'n':
 620                 case 'o':case 'p':case 'q':case 'r':case 's':case 't':case 'u':
 621                 case 'v':case 'w':case 'x':case 'y':case 'z':
 622                 case 'A':case 'B':case 'C':case 'D':case 'E':case 'F':case 'G':
 623                 case 'H':case 'I':case 'J':case 'K':case 'L':case 'M':case 'N':
 624                 case 'O':case 'P':case 'Q':case 'R':case 'S':case 'T':case 'U':
 625                 case 'V':case 'W':case 'X':case 'Y':case 'Z':
 626                 case '_':
 627                         ppt = pp_token_create(PP_TOKEN_IDENTIFIER, pos);
 628                         j = _pp_identifier_end(c, nr_c, i + 1);
 629                         pp_token_add(ppt, c, i, j);
 630                         i = j;
 631                         break;
 632                 case '\\':
 633                         if (i + 5 < nr_c && c[i + 1] == 'u' &&
 634                             pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) && pp_hexdigit(c[i + 5])) {
 635                                 ppt = pp_token_create(PP_TOKEN_IDENTIFIER, pos);
 636                                 j = _pp_identifier_end(c, nr_c, i + 2 + 4);
 637                                 pp_token_add(ppt, c, i, j);
 638                                 i = j;
 639                         } else if (i + 9 < nr_c && c[i + 1] == 'U' &&
 640                                    pp_hexdigit(c[i + 2]) && pp_hexdigit(c[i + 3]) && pp_hexdigit(c[i + 4]) && pp_hexdigit(c[i + 5]) &&
 641                                    pp_hexdigit(c[i + 6]) && pp_hexdigit(c[i + 7]) && pp_hexdigit(c[i + 8]) && pp_hexdigit(c[i + 9])) {
 642                                 ppt = pp_token_create(PP_TOKEN_IDENTIFIER, pos);
 643                                 j = _pp_identifier_end(c, nr_c, i + 2 + 4 + 4);
 644                                 pp_token_add(ppt, c, i, j);
 645                                 i = j;
 646                         } else
 647                                 error_exit(pos, "unknown character %08"PRIx32, c[i]);
 648                         break;
 649                 case '0':case '1':case '2':case '3':case '4':
 650                 case '5':case '6':case '7':case '8':case '9':
 651                         ppt = pp_token_create(PP_TOKEN_NUMBER, pos);
 652                         j = pp_number_end(c, nr_c, i);
 653                         pp_token_add(ppt, c, i, j);
 654                         i = j;
 655                         break;
 656                 case '.':
 657                         if (i + 2 < nr_c && c[i + 1] == '.' && c[i + 2] == '.') {
 658                                 ppt = pp_token_create(PP_TOKEN_DOTDOTDOT, pos);
 659                                 i += 3;
 660                         } else if (i + 1 < nr_c && pp_digit(c[i + 1])) {
 661                                 ppt = pp_token_create(PP_TOKEN_NUMBER, pos);
 662                                 j = pp_number_end(c, nr_c, i + 1);
 663                                 pp_token_add(ppt, c, i, j);
 664                                 i = j;
 665                         } else
 666                                 goto pp_token_simple;
 667                         break;
 668                 case '"':
 669                         ppt = pp_token_create(PP_TOKEN_STRING, pos);
 670                         j = pp_string_end(c, nr_c, i, _pos);
 671                         pp_token_add(ppt, c, i + 1, j - 1);
 672                         i = j;
 673                         break;
 674                 case '\'':
 675                         ppt = pp_token_create(PP_TOKEN_CHAR, pos);
 676                         j = pp_char_end(c, nr_c, i, _pos);
 677                         pp_token_add(ppt, c, i + 1, j - 1);
 678                         i = j;
 679                         break;
 680                 case '/':
 681                         if (i + 1 < nr_c && c[i + 1] == '*') {
 682                                 ppt = pp_token_create(' ', pos);
 683                                 i = c_comment_end(c, nr_c, i);
 684                         } else if (i + 1 < nr_c && c[i + 1] == '/') {
 685                                 warning(pos, "C++ comment");
 686                                 ppt = pp_token_create(' ', pos);
 687                                 i = cpp_comment_end(c, nr_c, i);
 688                         } else if (i + 1 < nr_c && c[i + 1] == '=') {
 689                                 ppt = pp_token_create((c[i] << 8) | c[i + 1], pos);
 690                                 i += 2;
 691                         } else
 692                                 goto pp_token_simple;
 693                         break;
 694                 case '-':
 695                         if (i + 1 < nr_c && (c[i + 1] == '>' || c[i + 1] == '=' || c[i + 1] == '-')) {
 696                                 ppt = pp_token_create((c[i] << 8) | c[i + 1], pos);
 697                                 i += 2;
 698                         } else
 699                                 goto pp_token_simple;
 700                         break;
 701                 case '+':
 702                 case '&':
 703                 case '|':
 704                         if (i + 1 < nr_c && (c[i + 1] == '=' || c[i + 1] == c[i])) {
 705                                 ppt = pp_token_create((c[i] << 8) | c[i + 1], pos);
 706                                 i += 2;
 707                         } else
 708                                 goto pp_token_simple;
 709                         break;
 710                 case '*':
 711                 case '!':
 712                 case '%':
 713                 case '=':
 714                 case '^':
 715                         if (i + 1 < nr_c && c[i + 1] == '=') {
 716                                 ppt = pp_token_create((c[i] << 8) | c[i + 1], pos);
 717                                 i += 2;
 718                         } else
 719                                 goto pp_token_simple;
 720                         break;
 721                 case '<':
 722                 case '>':
 723                         if (i + 2 < nr_c && c[i + 1] == c[i] && c[i + 2] == '=') {
 724                                 ppt = pp_token_create((c[i] << 16) | (c[i + 1] << 8) | c[i + 2], pos);
 725                                 i += 3;
 726                         } else if (i + 1 < nr_c && (c[i + 1] == c[i] || c[i + 1] == '=')) {
 727                                 ppt = pp_token_create((c[i] << 8) | c[i + 1], pos);
 728                                 i += 2;
 729                         } else
 730                                 goto pp_token_simple;
 731                         break;
 732                 case '#':
 733                         if (i + 1 < nr_c && c[i + 1] == '#') {
 734                                 ppt = pp_token_create((c[i] << 8) | c[i + 1], pos);
 735                                 i += 2;
 736                         } else
 737                                 goto pp_token_simple;
 738                         break;
 739                 default:
 740                         error_exit(pos, "unknown character %08"PRIx32, c[i]);
 741                 }
 742
 743                 if (!ppt_head)
 744                         ppt_head = ppt;
 745                 else
 746                         ppt_tail->next = ppt;
 747                 ppt_tail = ppt;
 748         }
 749         return ppt_head;
 750 }
 751
 752 int main(int argc, char *argv[])
 753 {
 754         int fd;
 755         struct stat st;
 756         unsigned int st_size;
 757         void *buf;
 758         uint8_t *_c;
 759         unsigned int _nr_c;
 760         uint32_t *c;
 761         unsigned int nr_c;
 762         struct pos *pos;
 763         struct pp_token *ppt_head;
 764
 765         if (argc < 2)
 766                 return EXIT_FAILURE;
 767
 768         fd = open(argv[1], O_RDONLY);
 769         if (fd == -1)
 770                 perror_exit("open %s", argv[1]);
 771         if (fstat(fd, &st) == -1)
 772                 perror_exit("fstat %s", argv[1]);
 773         if (st.st_size < 0)
 774                 _error_exit("%s: negative st_size %"PRIdMAX, argv[1], (intmax_t)st.st_size);
 775         st_size = (unsigned int)(uintmax_t)(intmax_t)st.st_size;
 776         if ((uintmax_t)(intmax_t)st.st_size != (uintmax_t)st_size)
 777                 _error_exit("%s: too big st_size %"PRIdMAX, argv[1], (intmax_t)st.st_size);
 778
 779         buf = xmalloc(st_size);
 780         xread(fd, buf, st_size);
 781         close(fd);
 782
 783         _c = buf;
 784         _nr_c = st_size;
 785         /* Skip UTF-8 "BOM" if any. */
 786         if (st_size >= 3 && _c[0] == 0xef && _c[1] == 0xbb && _c[2] == 0xbf) {
 787                 _c += 3;
 788                 _nr_c -= 3;
 789         }
 790         convert_from_utf8(_c, _nr_c, &c, &nr_c);
 791         free(buf);
 792
 793         fix_newline(c, &nr_c);
 794         pos = line_column(c, nr_c);
 795         warn_trigraph(c, nr_c, pos);
 796         delete_backslash_newline(c, &nr_c, pos);
 797
 798         ppt_head = pp_tokenize(c, nr_c, pos);
 799         free(c);
 800         free(pos);
 801
 802         {
 803                 struct pp_token *ppt;
 804
 805                 for (ppt = ppt_head; ppt; ppt = ppt->next)
 806                         pp_token_print(ppt);
 807         }
 808
 809         pp_token_free(ppt_head);
 810
 811         return EXIT_SUCCESS;
 812 }