boost/libs/wave/src/cpplexer/re2clex/cpp_re.cpp

   1 /*=============================================================================
   2     Boost.Wave: A Standard compliant C++ preprocessor library
   3
   4     Copyright (c) 2001 Daniel C. Nuffer
   5     Copyright (c) 2001-2007 Hartmut Kaiser.
   6     Distributed under the Boost Software License, Version 1.0. (See accompanying
   7     file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   8
   9     TODO:
  10         It also may be necessary to add $ to identifiers, for asm.
  11         handle errors better.
  12         have some easier way to parse strings instead of files (done)
  13 =============================================================================*/
  14
  15 #define BOOST_WAVE_SOURCE 1
  16 #include <ctime>
  17 #include <cstdlib>
  18 #include <cstdio>
  19 #include <cstring>
  20 #include <sys/types.h>
  21 #include <sys/stat.h>
  22 #include <fcntl.h>
  23
  24 #include <boost/wave/wave_config.hpp>          // configuration data
  25
  26 #if defined(BOOST_HAS_UNISTD_H)
  27 #include <unistd.h>
  28 #else
  29 #include <io.h>
  30 #endif
  31
  32 #include <boost/assert.hpp>
  33 #include <boost/detail/workaround.hpp>
  34
  35 #include <boost/wave/token_ids.hpp>
  36 #include <boost/wave/cpplexer/re2clex/aq.hpp>
  37 #include <boost/wave/cpplexer/re2clex/scanner.hpp>
  38 #include <boost/wave/cpplexer/re2clex/cpp_re.hpp>
  39
  40 // this must occur after all of the includes and before any code appears
  41 #ifdef BOOST_HAS_ABI_HEADERS
  42 #include BOOST_ABI_PREFIX
  43 #endif
  44
  45 ///////////////////////////////////////////////////////////////////////////////
  46 #if defined(BOOST_MSVC)
  47 #pragma warning (disable: 4101)     // 'foo' : unreferenced local variable
  48 #pragma warning (disable: 4102)     // 'foo' : unreferenced label
  49 #endif
  50
  51 ///////////////////////////////////////////////////////////////////////////////
  52 #define BOOST_WAVE_BSIZE     196608
  53
  54 #define YYCTYPE   uchar
  55 #define YYCURSOR  cursor
  56 #define YYLIMIT   limit
  57 #define YYMARKER  marker
  58 #define YYFILL(n)                                                             \
  59     {                                                                         \
  60         cursor = uchar_wrapper(fill(s, cursor), cursor.column);               \
  61         limit = uchar_wrapper (s->lim);                                       \
  62     }                                                                         \
  63     /**/
  64
  65 #include <iostream>
  66
  67 ///////////////////////////////////////////////////////////////////////////////
  68 #define BOOST_WAVE_UPDATE_CURSOR()                                            \
  69     {                                                                         \
  70         s->line += count_backslash_newlines(s, cursor);                       \
  71         s->curr_column = cursor.column;                                       \
  72         s->cur = cursor;                                                      \
  73         s->lim = limit;                                                       \
  74         s->ptr = marker;                                                      \
  75     }                                                                         \
  76     /**/
  77
  78 ///////////////////////////////////////////////////////////////////////////////
  79 #define BOOST_WAVE_RET(i)                                                     \
  80     {                                                                         \
  81         BOOST_WAVE_UPDATE_CURSOR()                                            \
  82         if (s->cur > s->lim)                                                  \
  83             return T_EOF;     /* may happen for empty files */                \
  84         return (i);                                                           \
  85     }                                                                         \
  86     /**/
  87
  88 ///////////////////////////////////////////////////////////////////////////////
  89 namespace boost {
  90 namespace wave {
  91 namespace cpplexer {
  92 namespace re2clex {
  93
  94 #define RE2C_ASSERT BOOST_ASSERT
  95
  96 int get_one_char(Scanner *s)
  97 {
  98     if (0 != s->act) {
  99         RE2C_ASSERT(s->first != 0 && s->last != 0);
 100         RE2C_ASSERT(s->first <= s->act && s->act <= s->last);
 101         if (s->act < s->last)
 102             return *(s->act)++;
 103     }
 104     return -1;
 105 }
 106
 107 std::ptrdiff_t rewind_stream (Scanner *s, int cnt)
 108 {
 109     if (0 != s->act) {
 110         RE2C_ASSERT(s->first != 0 && s->last != 0);
 111         s->act += cnt;
 112         RE2C_ASSERT(s->first <= s->act && s->act <= s->last);
 113         return s->act - s->first;
 114     }
 115     return 0;
 116 }
 117
 118 std::size_t get_first_eol_offset(Scanner* s)
 119 {
 120     if (!AQ_EMPTY(s->eol_offsets))
 121     {
 122         return s->eol_offsets->queue[s->eol_offsets->head];
 123     }
 124     else
 125     {
 126         return (unsigned int)-1;
 127     }
 128 }
 129
 130 void adjust_eol_offsets(Scanner* s, std::size_t adjustment)
 131 {
 132     aq_queue q;
 133     std::size_t i;
 134
 135     if (!s->eol_offsets)
 136         s->eol_offsets = aq_create();
 137
 138     q = s->eol_offsets;
 139
 140     if (AQ_EMPTY(q))
 141         return;
 142
 143     i = q->head;
 144     while (i != q->tail)
 145     {
 146         if (adjustment > q->queue[i])
 147             q->queue[i] = 0;
 148         else
 149             q->queue[i] -= adjustment;
 150         ++i;
 151         if (i == q->max_size)
 152             i = 0;
 153     }
 154     if (adjustment > q->queue[i])
 155         q->queue[i] = 0;
 156     else
 157         q->queue[i] -= adjustment;
 158 }
 159
 160 int count_backslash_newlines(Scanner *s, uchar *cursor)
 161 {
 162     std::size_t diff, offset;
 163     int skipped = 0;
 164
 165     /* figure out how many backslash-newlines skipped over unknowingly. */
 166     diff = cursor - s->bot;
 167     offset = get_first_eol_offset(s);
 168     while (offset <= diff && offset != (unsigned int)-1)
 169     {
 170         skipped++;
 171         aq_pop(s->eol_offsets);
 172         offset = get_first_eol_offset(s);
 173     }
 174     return skipped;
 175 }
 176
 177 bool is_backslash(uchar *p, uchar *end, int &len)
 178 {
 179     if (*p == '\\') {
 180         len = 1;
 181         return true;
 182     }
 183     else if (*p == '?' && *(p+1) == '?' && (p+2 < end && *(p+2) == '/')) {
 184         len = 3;
 185         return true;
 186     }
 187     return false;
 188 }
 189
 190 uchar *fill(Scanner *s, uchar *cursor)
 191 {
 192     using namespace std;    // some systems have memcpy etc. in namespace std
 193     if(!s->eof)
 194     {
 195         uchar* p;
 196         std::ptrdiff_t cnt = s->tok - s->bot;
 197         if(cnt)
 198         {
 199             if (NULL == s->lim)
 200                 s->lim = s->top;
 201             memmove(s->bot, s->tok, s->lim - s->tok);
 202             s->tok = s->cur = s->bot;
 203             s->ptr -= cnt;
 204             cursor -= cnt;
 205             s->lim -= cnt;
 206             adjust_eol_offsets(s, cnt);
 207         }
 208
 209         if((s->top - s->lim) < BOOST_WAVE_BSIZE)
 210         {
 211             uchar *buf = (uchar*) malloc(((s->lim - s->bot) + BOOST_WAVE_BSIZE)*sizeof(uchar));
 212             if (buf == 0)
 213             {
 214                 using namespace std;      // some systems have printf in std
 215                 if (0 != s->error_proc)
 216                     (*s->error_proc)(s, "Out of memory!");
 217                 else
 218                     printf("Out of memory!\n");
 219
 220                 /* get the scanner to stop */
 221                 *cursor = 0;
 222                 return cursor;
 223             }
 224
 225             memmove(buf, s->tok, s->lim - s->tok);
 226             s->tok = s->cur = buf;
 227             s->ptr = &buf[s->ptr - s->bot];
 228             cursor = &buf[cursor - s->bot];
 229             s->lim = &buf[s->lim - s->bot];
 230             s->top = &s->lim[BOOST_WAVE_BSIZE];
 231             free(s->bot);
 232             s->bot = buf;
 233         }
 234
 235         if (s->act != 0) {
 236             cnt = s->last - s->act;
 237             if (cnt > BOOST_WAVE_BSIZE)
 238                 cnt = BOOST_WAVE_BSIZE;
 239             memmove(s->lim, s->act, cnt);
 240             s->act += cnt;
 241             if (cnt != BOOST_WAVE_BSIZE)
 242             {
 243                 s->eof = &s->lim[cnt]; *(s->eof)++ = '\0';
 244             }
 245         }
 246
 247         /* backslash-newline erasing time */
 248
 249         /* first scan for backslash-newline and erase them */
 250         for (p = s->lim; p < s->lim + cnt - 2; ++p)
 251         {
 252             int len = 0;
 253             if (is_backslash(p, s->lim + cnt, len))
 254             {
 255                 if (*(p+len) == '\n')
 256                 {
 257                     int offset = len + 1;
 258                     memmove(p, p + offset, s->lim + cnt - p - offset);
 259                     cnt -= offset;
 260                     --p;
 261                     aq_enqueue(s->eol_offsets, p - s->bot + 1);
 262                 }
 263                 else if (*(p+len) == '\r')
 264                 {
 265                     if (*(p+len+1) == '\n')
 266                     {
 267                         int offset = len + 2;
 268                         memmove(p, p + offset, s->lim + cnt - p - offset);
 269                         cnt -= offset;
 270                         --p;
 271                     }
 272                     else
 273                     {
 274                         int offset = len + 1;
 275                         memmove(p, p + offset, s->lim + cnt - p - offset);
 276                         cnt -= offset;
 277                         --p;
 278                     }
 279                     aq_enqueue(s->eol_offsets, p - s->bot + 1);
 280                 }
 281             }
 282         }
 283
 284         /* FIXME: the following code should be fixed to recognize correctly the
 285                   trigraph backslash token */
 286
 287         /* check to see if what we just read ends in a backslash */
 288         if (cnt >= 2)
 289         {
 290             uchar last = s->lim[cnt-1];
 291             uchar last2 = s->lim[cnt-2];
 292             /* check \ EOB */
 293             if (last == '\\')
 294             {
 295                 int next = get_one_char(s);
 296                 /* check for \ \n or \ \r or \ \r \n straddling the border */
 297                 if (next == '\n')
 298                 {
 299                     --cnt; /* chop the final \, we've already read the \n. */
 300                     aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
 301                 }
 302                 else if (next == '\r')
 303                 {
 304                     int next2 = get_one_char(s);
 305                     if (next2 == '\n')
 306                     {
 307                         --cnt; /* skip the backslash */
 308                     }
 309                     else
 310                     {
 311                         /* rewind one, and skip one char */
 312                         rewind_stream(s, -1);
 313                         --cnt;
 314                     }
 315                     aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
 316                 }
 317                 else if (next != -1) /* -1 means end of file */
 318                 {
 319                     /* next was something else, so rewind the stream */
 320                     rewind_stream(s, -1);
 321                 }
 322             }
 323             /* check \ \r EOB */
 324             else if (last == '\r' && last2 == '\\')
 325             {
 326                 int next = get_one_char(s);
 327                 if (next == '\n')
 328                 {
 329                     cnt -= 2; /* skip the \ \r */
 330                 }
 331                 else
 332                 {
 333                     /* rewind one, and skip two chars */
 334                     rewind_stream(s, -1);
 335                     cnt -= 2;
 336                 }
 337                 aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
 338             }
 339             /* check \ \n EOB */
 340             else if (last == '\n' && last2 == '\\')
 341             {
 342                 cnt -= 2;
 343                 aq_enqueue(s->eol_offsets, cnt + (s->lim - s->bot));
 344             }
 345         }
 346
 347         s->lim += cnt;
 348         if (s->eof) /* eof needs adjusting if we erased backslash-newlines */
 349         {
 350             s->eof = s->lim;
 351             *(s->eof)++ = '\0';
 352         }
 353     }
 354     return cursor;
 355 }
 356
 357 ///////////////////////////////////////////////////////////////////////////////
 358 //  Special wrapper class holding the current cursor position
 359 struct uchar_wrapper
 360 {
 361     uchar_wrapper (uchar *base_cursor, unsigned int column = 1)
 362     :   base_cursor(base_cursor), column(column)
 363     {}
 364
 365     uchar_wrapper& operator++()
 366     {
 367         ++base_cursor;
 368         ++column;
 369         return *this;
 370     }
 371
 372     uchar_wrapper& operator--()
 373     {
 374         --base_cursor;
 375         --column;
 376         return *this;
 377     }
 378
 379     uchar operator* () const
 380     {
 381         return *base_cursor;
 382     }
 383
 384     operator uchar *() const
 385     {
 386         return base_cursor;
 387     }
 388
 389     friend int operator- (uchar_wrapper const& lhs, uchar_wrapper const& rhs)
 390     {
 391         return lhs.base_cursor - rhs.base_cursor;
 392     }
 393
 394     uchar *base_cursor;
 395     unsigned int column;
 396 };
 397
 398 ///////////////////////////////////////////////////////////////////////////////
 399 boost::wave::token_id scan(Scanner *s)
 400 {
 401     BOOST_ASSERT(0 != s->error_proc);     // error handler must be given
 402
 403     uchar_wrapper cursor (s->tok = s->cur, s->column = s->curr_column);
 404     uchar_wrapper marker (s->ptr);
 405     uchar_wrapper limit (s->lim);
 406
 407 // include the correct Re2C token definition rules
 408 #if BOOST_WAVE_USE_STRICT_LEXER != 0
 409 #include "strict_cpp_re.inc"
 410 #else
 411 #include "cpp_re.inc"
 412 #endif
 413
 414 } /* end of scan */
 415
 416 ///////////////////////////////////////////////////////////////////////////////
 417 }   // namespace re2clex
 418 }   // namespace cpplexer
 419 }   // namespace wave
 420 }   // namespace boost
 421
 422 #undef BOOST_WAVE_RET
 423 #undef BOOST_WAVE_BSIZE
 424 #undef YYCTYPE
 425 #undef YYCURSOR
 426 #undef YYLIMIT
 427 #undef YYMARKER
 428 #undef YYFILL
 429
 430 // the suffix header occurs after all of the code
 431 #ifdef BOOST_HAS_ABI_HEADERS
 432 #include BOOST_ABI_SUFFIX
 433 #endif
 434