ctags/main/entry.c

   1 /*
   2 *   Copyright (c) 1996-2002, Darren Hiebert
   3 *
   4 *   This source code is released for free distribution under the terms of the
   5 *   GNU General Public License version 2 or (at your option) any later version.
   6 *
   7 *   This module contains functions for creating tag entries.
   8 */
   9
  10 /*
  11 *   INCLUDE FILES
  12 */
  13 #include "general.h"  /* must always come first */
  14
  15 #include <string.h>
  16 #include <ctype.h>        /* to define isspace () */
  17 #include <errno.h>
  18
  19 #if defined (HAVE_SYS_TYPES_H)
  20 # include <sys/types.h>   /* to declare off_t on some hosts */
  21 #endif
  22 #if defined (HAVE_TYPES_H)
  23 # include <types.h>       /* to declare off_t on some hosts */
  24 #endif
  25 #if defined (HAVE_UNISTD_H)
  26 # include <unistd.h>      /* to declare close (), ftruncate (), truncate () */
  27 #endif
  28
  29 /*  These header files provide for the functions necessary to do file
  30  *  truncation.
  31  */
  32 #ifdef HAVE_FCNTL_H
  33 # include <fcntl.h>
  34 #endif
  35 #ifdef HAVE_IO_H
  36 # include <io.h>
  37 #endif
  38
  39 #include <stdint.h>
  40 #include <limits.h>  /* to define INT_MAX */
  41
  42 #include "debug.h"
  43 #include "entry_p.h"
  44 #include "field.h"
  45 #include "fmt_p.h"
  46 #include "kind.h"
  47 #include "nestlevel.h"
  48 #include "options_p.h"
  49 #include "ptag_p.h"
  50 #include "rbtree.h"
  51 #include "read.h"
  52 #include "read_p.h"
  53 #include "routines.h"
  54 #include "routines_p.h"
  55 #include "parse_p.h"
  56 #include "ptrarray.h"
  57 #include "sort_p.h"
  58 #include "strlist.h"
  59 #include "subparser_p.h"
  60 #include "trashbox.h"
  61 #include "writer_p.h"
  62 #include "xtag_p.h"
  63
  64 /*
  65 *   MACROS
  66 */
  67
  68 /*
  69  *  Portability defines
  70  */
  71 #if !defined(HAVE_TRUNCATE) && !defined(HAVE_FTRUNCATE) && !defined(HAVE_CHSIZE)
  72 # define USE_REPLACEMENT_TRUNCATE
  73 #endif
  74
  75 /*  Hack for ridiculous practice of Microsoft Visual C++.
  76  */
  77 #if defined (WIN32) && defined (_MSC_VER)
  78 # define chsize         _chsize
  79 # define open           _open
  80 # define close          _close
  81 # define O_RDWR         _O_RDWR
  82 #endif
  83
  84
  85 /*  Maintains the state of the tag file.
  86  */
  87 typedef struct eTagFile {
  88         char *name;
  89         char *directory;
  90         MIO *mio;
  91         struct sNumTags { unsigned long added, prev; } numTags;
  92         struct sMax { size_t line, tag; } max;
  93         vString *vLine;
  94
  95         int cork;
  96         unsigned int corkFlags;
  97         ptrArray *corkQueue;
  98
  99         bool patternCacheValid;
 100 } tagFile;
 101
 102 typedef struct sTagEntryInfoX  {
 103         tagEntryInfo slot;
 104         int corkIndex;
 105         struct rb_root symtab;
 106         struct rb_node symnode;
 107 } tagEntryInfoX;
 108
 109 /*
 110 *   DATA DEFINITIONS
 111 */
 112
 113 static tagFile TagFile = {
 114     NULL,               /* tag file name */
 115     NULL,               /* tag file directory (absolute) */
 116     NULL,               /* file pointer */
 117     { 0, 0 },           /* numTags */
 118     { 0, 0 },        /* max */
 119     NULL,                /* vLine */
 120     .cork = false,
 121     .corkQueue = NULL,
 122     .patternCacheValid = false,
 123 };
 124
 125 static bool TagsToStdout = false;
 126
 127 /*
 128 *   FUNCTION PROTOTYPES
 129 */
 130 #ifdef NEED_PROTO_TRUNCATE
 131 extern int truncate (const char *path, off_t length);
 132 #endif
 133
 134 #ifdef NEED_PROTO_FTRUNCATE
 135 extern int ftruncate (int fd, off_t length);
 136 #endif
 137
 138 /*
 139 *   FUNCTION DEFINITIONS
 140 */
 141
 142 extern void freeTagFileResources (void)
 143 {
 144         if (TagFile.directory != NULL)
 145                 eFree (TagFile.directory);
 146         vStringDelete (TagFile.vLine);
 147 }
 148
 149 extern const char *tagFileName (void)
 150 {
 151         return TagFile.name;
 152 }
 153
 154 /*
 155 *   Pseudo tag support
 156 */
 157
 158 extern void abort_if_ferror(MIO *const mio)
 159 {
 160         if (mio != NULL && mio_error (mio))
 161                 error (FATAL | PERROR, "cannot write tag file");
 162 }
 163
 164 static void rememberMaxLengths (const size_t nameLength, const size_t lineLength)
 165 {
 166         if (nameLength > TagFile.max.tag)
 167                 TagFile.max.tag = nameLength;
 168
 169         if (lineLength > TagFile.max.line)
 170                 TagFile.max.line = lineLength;
 171 }
 172
 173 static void addCommonPseudoTags (void)
 174 {
 175         for (int i = 0; i < PTAG_COUNT; i++)
 176         {
 177                 if (isPtagCommonInParsers (i))
 178                         makePtagIfEnabled (i, LANG_IGNORE, &Option);
 179         }
 180 }
 181
 182 extern void makeFileTag (const char *const fileName)
 183 {
 184         tagEntryInfo tag;
 185
 186         if (!isXtagEnabled(XTAG_FILE_NAMES))
 187                 return;
 188
 189         initTagEntry (&tag, baseFilename (fileName), KIND_FILE_INDEX);
 190
 191         tag.isFileEntry     = true;
 192         tag.lineNumberEntry = true;
 193         markTagExtraBit (&tag, XTAG_FILE_NAMES);
 194
 195         tag.lineNumber = 1;
 196         if (isFieldEnabled (FIELD_END_LINE))
 197         {
 198                 /* isFieldEnabled is called again in the rendering
 199                    stage. However, it is called here for avoiding
 200                    unnecessary read line loop. */
 201                 while (readLineFromInputFile () != NULL)
 202                         ; /* Do nothing */
 203                 tag.extensionFields.endLine = getInputLineNumber ();
 204         }
 205
 206         if (isFieldEnabled (FIELD_EPOCH))
 207                 tag.extensionFields.epoch = getInputFileMtime ();
 208
 209         makeTagEntry (&tag);
 210 }
 211
 212 static void updateSortedFlag (
 213                 const char *const line, MIO *const mio, MIOPos startOfLine)
 214 {
 215         const char *const tab = strchr (line, '\t');
 216
 217         if (tab != NULL)
 218         {
 219                 const long boolOffset = tab - line + 1;  /* where it should be */
 220
 221                 if (line [boolOffset] == '0'  ||  line [boolOffset] == '1')
 222                 {
 223                         MIOPos nextLine;
 224
 225                         if (mio_getpos (mio, &nextLine) == -1 || mio_setpos (mio, &startOfLine) == -1)
 226                                 error (WARNING, "Failed to update 'sorted' pseudo-tag");
 227                         else
 228                         {
 229                                 MIOPos flagLocation;
 230                                 int c, d;
 231
 232                                 do
 233                                         c = mio_getc (mio);
 234                                 while (c != '\t'  &&  c != '\n');
 235                                 mio_getpos (mio, &flagLocation);
 236                                 d = mio_getc (mio);
 237                                 if (c == '\t'  &&  (d == '0'  ||  d == '1')  &&
 238                                         d != (int) Option.sorted)
 239                                 {
 240                                         mio_setpos (mio, &flagLocation);
 241                                         mio_putc (mio, Option.sorted == SO_FOLDSORTED ? '2' :
 242                                                 (Option.sorted == SO_SORTED ? '1' : '0'));
 243                                 }
 244                                 mio_setpos (mio, &nextLine);
 245                         }
 246                 }
 247         }
 248 }
 249
 250 /*  Look through all line beginning with "!_TAG_FILE", and update those which
 251  *  require it.
 252  */
 253 static long unsigned int updatePseudoTags (MIO *const mio)
 254 {
 255         enum { maxEntryLength = 20 };
 256         char entry [maxEntryLength + 1];
 257         unsigned long linesRead = 0;
 258         MIOPos startOfLine;
 259         size_t entryLength;
 260         const char *line;
 261
 262         sprintf (entry, "%sTAG_FILE", PSEUDO_TAG_PREFIX);
 263         entryLength = strlen (entry);
 264         Assert (entryLength < maxEntryLength);
 265
 266         mio_getpos (mio, &startOfLine);
 267         line = readLineRaw (TagFile.vLine, mio);
 268         while (line != NULL  &&  line [0] == entry [0])
 269         {
 270                 ++linesRead;
 271                 if (strncmp (line, entry, entryLength) == 0)
 272                 {
 273                         char tab, classType [16];
 274
 275                         if (sscanf (line + entryLength, "%15s%c", classType, &tab) == 2  &&
 276                                 tab == '\t')
 277                         {
 278                                 if (strcmp (classType, "_SORTED") == 0)
 279                                         updateSortedFlag (line, mio, startOfLine);
 280                         }
 281                         mio_getpos (mio, &startOfLine);
 282                 }
 283                 line = readLineRaw (TagFile.vLine, mio);
 284         }
 285         while (line != NULL)  /* skip to end of file */
 286         {
 287                 ++linesRead;
 288                 line = readLineRaw (TagFile.vLine, mio);
 289         }
 290         return linesRead;
 291 }
 292
 293 /*
 294  *  Tag file management
 295  */
 296
 297 static bool isValidTagAddress (const char *const excmd)
 298 {
 299         bool isValid = false;
 300
 301         if (strchr ("/?", excmd [0]) != NULL)
 302                 isValid = true;
 303         else
 304         {
 305                 char *address = xMalloc (strlen (excmd) + 1, char);
 306                 if (sscanf (excmd, "%[^;\n]", address) == 1  &&
 307                         strspn (address,"0123456789") == strlen (address))
 308                                 isValid = true;
 309                 eFree (address);
 310         }
 311         return isValid;
 312 }
 313
 314 static bool isCtagsLine (const char *const line)
 315 {
 316         enum fieldList { TAG, TAB1, SRC_FILE, TAB2, EXCMD, NUM_FIELDS };
 317         bool ok = false;  /* we assume not unless confirmed */
 318         const size_t fieldLength = strlen (line) + 1;
 319         char *const fields = xMalloc (NUM_FIELDS * fieldLength, char);
 320
 321         if (fields == NULL)
 322                 error (FATAL, "Cannot analyze tag file");
 323         else
 324         {
 325 #define field(x)                (fields + ((size_t) (x) * fieldLength))
 326
 327                 const int numFields = sscanf (
 328                         line, "%[^\t]%[\t]%[^\t]%[\t]%[^\r\n]",
 329                         field (TAG), field (TAB1), field (SRC_FILE),
 330                         field (TAB2), field (EXCMD));
 331
 332                 /*  There must be exactly five fields: two tab fields containing
 333                  *  exactly one tab each, the tag must not begin with "#", and the
 334                  *  file name should not end with ";", and the excmd must be
 335                  *  acceptable.
 336                  *
 337                  *  These conditions will reject tag-looking lines like:
 338                  *      int a;        <C-comment>
 339                  *      #define LABEL <C-comment>
 340                  */
 341                 if (numFields == NUM_FIELDS   &&
 342                         strlen (field (TAB1)) == 1  &&
 343                         strlen (field (TAB2)) == 1  &&
 344                         field (TAG) [0] != '#'      &&
 345                         field (SRC_FILE) [strlen (field (SRC_FILE)) - 1] != ';'  &&
 346                         isValidTagAddress (field (EXCMD)))
 347                                 ok = true;
 348
 349                 eFree (fields);
 350         }
 351         return ok;
 352 }
 353
 354 static bool isEtagsLine (const char *const line)
 355 {
 356         bool result = false;
 357         if (line [0] == '\f')
 358                 result = (bool) (line [1] == '\n'  ||  line [1] == '\r');
 359         return result;
 360 }
 361
 362 static bool isTagFile (const char *const filename)
 363 {
 364         bool ok = false;  /* we assume not unless confirmed */
 365         MIO *const mio = mio_new_file (filename, "rb");
 366
 367         if (mio == NULL  &&  errno == ENOENT)
 368                 ok = true;
 369         else if (mio != NULL)
 370         {
 371                 const char *line = readLineRaw (TagFile.vLine, mio);
 372
 373                 if (line == NULL)
 374                         ok = true;
 375                 else
 376                         ok = (bool) (isCtagsLine (line) || isEtagsLine (line));
 377                 mio_unref (mio);
 378         }
 379         return ok;
 380 }
 381
 382 extern void openTagFile (void)
 383 {
 384         setDefaultTagFileName ();
 385         TagsToStdout = isDestinationStdout ();
 386
 387         if (TagFile.vLine == NULL)
 388                 TagFile.vLine = vStringNew ();
 389
 390         /*  Open the tags file.
 391          */
 392         if (TagsToStdout)
 393         {
 394                 if (Option.interactive == INTERACTIVE_SANDBOX)
 395                 {
 396                         TagFile.mio = mio_new_memory (NULL, 0, eRealloc, eFreeNoNullCheck);
 397                         TagFile.name = NULL;
 398                 }
 399                 else
 400                         TagFile.mio = tempFile ("w+", &TagFile.name);
 401                 if (isXtagEnabled (XTAG_PSEUDO_TAGS))
 402                         addCommonPseudoTags ();
 403         }
 404         else
 405         {
 406                 bool fileExists;
 407
 408                 TagFile.name = eStrdup (Option.tagFileName);
 409                 fileExists = doesFileExist (TagFile.name);
 410                 if (fileExists  &&  ! isTagFile (TagFile.name))
 411                         error (FATAL,
 412                           "\"%s\" doesn't look like a tag file; I refuse to overwrite it.",
 413                                   TagFile.name);
 414
 415                 if (Option.etags)
 416                 {
 417                         if (Option.append  &&  fileExists)
 418                                 TagFile.mio = mio_new_file (TagFile.name, "a+b");
 419                         else
 420                                 TagFile.mio = mio_new_file (TagFile.name, "w+b");
 421                 }
 422                 else
 423                 {
 424                         if (Option.append  &&  fileExists)
 425                         {
 426                                 TagFile.mio = mio_new_file (TagFile.name, "r+");
 427                                 if (TagFile.mio != NULL)
 428                                 {
 429                                         TagFile.numTags.prev = updatePseudoTags (TagFile.mio);
 430                                         mio_unref (TagFile.mio);
 431                                         TagFile.mio = mio_new_file (TagFile.name, "a+");
 432                                 }
 433                         }
 434                         else
 435                         {
 436                                 TagFile.mio = mio_new_file (TagFile.name, "w");
 437                                 if (TagFile.mio != NULL && isXtagEnabled (XTAG_PSEUDO_TAGS))
 438                                         addCommonPseudoTags ();
 439                         }
 440                 }
 441                 if (TagFile.mio == NULL)
 442                         error (FATAL | PERROR, "cannot open tag file");
 443         }
 444
 445         if (TagFile.directory == NULL)
 446         {
 447                 if (TagsToStdout)
 448                         TagFile.directory = eStrdup (CurrentDirectory);
 449                 else
 450                         TagFile.directory = absoluteDirname (TagFile.name);
 451         }
 452 }
 453
 454 #ifdef USE_REPLACEMENT_TRUNCATE
 455
 456 static void copyBytes (MIO* const fromMio, MIO* const toMio, const long size)
 457 {
 458         enum { BufferSize = 1000 };
 459         long toRead, numRead;
 460         char* buffer = xMalloc (BufferSize, char);
 461         long remaining = size;
 462         do
 463         {
 464                 toRead = (0 < remaining && remaining < BufferSize) ?
 465                                         remaining : (long) BufferSize;
 466                 numRead = mio_read (fromMio, buffer, (size_t) 1, (size_t) toRead);
 467                 if (mio_write (toMio, buffer, (size_t)1, (size_t)numRead) < (size_t)numRead)
 468                         error (FATAL | PERROR, "cannot complete write");
 469                 if (remaining > 0)
 470                         remaining -= numRead;
 471         } while (numRead == toRead  &&  remaining != 0);
 472         eFree (buffer);
 473 }
 474
 475 static void copyFile (const char *const from, const char *const to, const long size)
 476 {
 477         MIO* const fromMio = mio_new_file (from, "rb");
 478         if (fromMio == NULL)
 479                 error (FATAL | PERROR, "cannot open file to copy");
 480         else
 481         {
 482                 MIO* const toMio = mio_new_file (to, "wb");
 483                 if (toMio == NULL)
 484                         error (FATAL | PERROR, "cannot open copy destination");
 485                 else
 486                 {
 487                         copyBytes (fromMio, toMio, size);
 488                         mio_unref (toMio);
 489                 }
 490                 mio_unref (fromMio);
 491         }
 492 }
 493
 494 /*  Replacement for missing library function.
 495  */
 496 static int replacementTruncate (const char *const name, const long size)
 497 {
 498 #define WHOLE_FILE  -1L
 499         char *tempName = NULL;
 500         MIO *mio = tempFile ("w", &tempName);
 501         mio_unref (mio);
 502         copyFile (name, tempName, size);
 503         copyFile (tempName, name, WHOLE_FILE);
 504         remove (tempName);
 505         eFree (tempName);
 506
 507         return 0;
 508 }
 509
 510 #endif
 511
 512 #ifndef EXTERNAL_SORT
 513 static void internalSortTagFile (void)
 514 {
 515         MIO *mio;
 516
 517         /*  Open/Prepare the tag file and place its lines into allocated buffers.
 518          */
 519         if (TagsToStdout)
 520         {
 521                 mio = TagFile.mio;
 522                 mio_seek (mio, 0, SEEK_SET);
 523         }
 524         else
 525         {
 526                 mio = mio_new_file (tagFileName (), "r");
 527                 if (mio == NULL)
 528                         failedSort (mio, NULL);
 529         }
 530
 531         internalSortTags (TagsToStdout,
 532                           mio,
 533                           TagFile.numTags.added + TagFile.numTags.prev);
 534
 535         if (! TagsToStdout)
 536                 mio_unref (mio);
 537 }
 538 #endif
 539
 540 static void sortTagFile (void)
 541 {
 542         if (TagFile.numTags.added > 0L)
 543         {
 544                 if (Option.sorted != SO_UNSORTED)
 545                 {
 546                         verbose ("sorting tag file\n");
 547 #ifdef EXTERNAL_SORT
 548                         externalSortTags (TagsToStdout, TagFile.mio);
 549 #else
 550                         internalSortTagFile ();
 551 #endif
 552                 }
 553                 else if (TagsToStdout)
 554                         catFile (TagFile.mio);
 555         }
 556 }
 557
 558 static void resizeTagFile (const long newSize)
 559 {
 560         int result;
 561
 562         if (!TagFile.name)
 563         {
 564                 mio_try_resize (TagFile.mio, newSize);
 565                 return;
 566         }
 567
 568 #ifdef USE_REPLACEMENT_TRUNCATE
 569         result = replacementTruncate (TagFile.name, newSize);
 570 #else
 571 # ifdef HAVE_TRUNCATE
 572         result = truncate (TagFile.name, (off_t) newSize);
 573 # else
 574         const int fd = open (TagFile.name, O_RDWR);
 575
 576         if (fd == -1)
 577                 result = -1;
 578         else
 579         {
 580 #  ifdef HAVE_FTRUNCATE
 581                 result = ftruncate (fd, (off_t) newSize);
 582 #  else
 583 #   ifdef HAVE_CHSIZE
 584                 result = chsize (fd, newSize);
 585 #   endif
 586 #  endif
 587                 close (fd);
 588         }
 589 # endif
 590 #endif
 591         if (result == -1)
 592                 fprintf (stderr, "Cannot shorten tag file: errno = %d\n", errno);
 593 }
 594
 595 static void writeEtagsIncludes (MIO *const mio)
 596 {
 597         if (Option.etagsInclude)
 598         {
 599                 unsigned int i;
 600                 for (i = 0  ;  i < stringListCount (Option.etagsInclude)  ;  ++i)
 601                 {
 602                         vString *item = stringListItem (Option.etagsInclude, i);
 603                         mio_printf (mio, "\f\n%s,include\n", vStringValue (item));
 604                 }
 605         }
 606 }
 607
 608 extern void closeTagFile (const bool resize)
 609 {
 610         long desiredSize, size;
 611
 612         if (Option.etags)
 613                 writeEtagsIncludes (TagFile.mio);
 614         mio_flush (TagFile.mio);
 615
 616         abort_if_ferror (TagFile.mio);
 617         desiredSize = mio_tell (TagFile.mio);
 618         mio_seek (TagFile.mio, 0L, SEEK_END);
 619         size = mio_tell (TagFile.mio);
 620         if (! TagsToStdout)
 621                 /* The tag file should be closed before resizing. */
 622                 if (mio_unref (TagFile.mio) != 0)
 623                         error (FATAL | PERROR, "cannot close tag file");
 624
 625         if (resize  &&  desiredSize < size)
 626         {
 627                 DebugStatement (
 628                         debugPrintf (DEBUG_STATUS, "shrinking %s from %ld to %ld bytes\n",
 629                                 TagFile.name? TagFile.name: "<mio>", size, desiredSize); )
 630                 resizeTagFile (desiredSize);
 631         }
 632         sortTagFile ();
 633         if (TagsToStdout)
 634         {
 635                 if (mio_unref (TagFile.mio) != 0)
 636                         error (FATAL | PERROR, "cannot close tag file");
 637                 if (TagFile.name)
 638                         remove (TagFile.name);  /* remove temporary file */
 639         }
 640
 641         TagFile.mio = NULL;
 642         if (TagFile.name)
 643                 eFree (TagFile.name);
 644         TagFile.name = NULL;
 645 }
 646
 647 /*
 648  *  Tag entry management
 649  */
 650
 651 /*  This function copies the current line out to a specified file. It has no
 652  *  effect on the fileGetc () function.  During copying, any '\' characters
 653  *  are doubled and a leading '^' or trailing '$' is also quoted. End of line
 654  *  characters (line feed or carriage return) are dropped.
 655  */
 656 static size_t appendInputLine (int putc_func (char , void *), const char *const line,
 657                                                            unsigned int patternLengthLimit,
 658                                                            void * data, bool *omitted)
 659 {
 660         size_t length = 0;
 661         const char *p;
 662         int extraLength = 0;
 663
 664         /*  Write everything up to, but not including, a line end character.
 665          */
 666         *omitted = false;
 667         for (p = line  ;  *p != '\0'  ;  ++p)
 668         {
 669                 const int next = *(p + 1);
 670                 const int c = *p;
 671
 672                 if (c == CRETURN  ||  c == NEWLINE)
 673                         break;
 674
 675                 if (patternLengthLimit != 0 && length >= patternLengthLimit &&
 676                         /* Do not cut inside a multi-byte UTF-8 character, but safe-guard it not to
 677                          * allow more than one extra valid UTF-8 character in case it's not actually
 678                          * UTF-8.  To do that, limit to an extra 3 UTF-8 sub-bytes (0b10xxxxxx). */
 679                         ((((unsigned char) c) & 0xc0) != 0x80 || ++extraLength > 3))
 680                 {
 681                         *omitted = true;
 682                         break;
 683                 }
 684                 /*  If character is '\', or a terminal '$', then quote it.
 685                  */
 686                 if (c == BACKSLASH  ||  c == (Option.backward ? '?' : '/')  ||
 687                         (c == '$'  &&  (next == NEWLINE  ||  next == CRETURN)))
 688                 {
 689                         putc_func (BACKSLASH, data);
 690                         ++length;
 691                 }
 692                 putc_func (c, data);
 693                 ++length;
 694         }
 695
 696         return length;
 697 }
 698
 699 static int vstring_putc (char c, void *data)
 700 {
 701         vString *str = data;
 702         vStringPut (str, c);
 703         return 1;
 704 }
 705
 706 static int vstring_puts (const char* s, void *data)
 707 {
 708         vString *str = data;
 709         size_t len = vStringLength (str);
 710         vStringCatS (str, s);
 711         return (int) (vStringLength (str) - len);
 712 }
 713
 714 #ifdef DEBUG
 715 static bool isPosSet(MIOPos pos)
 716 {
 717         char * p = (char *)&pos;
 718         bool r = false;
 719         unsigned int i;
 720
 721         for (i = 0; i < sizeof(pos); i++)
 722                 r |= p[i];
 723         return r;
 724 }
 725 #endif
 726
 727 extern char *readLineFromBypassForTag (vString *const vLine, const tagEntryInfo *const tag,
 728                                    long *const pSeekValue)
 729 {
 730         Assert (isPosSet (tag->filePosition) || (tag->pattern == NULL));
 731         return readLineFromBypass (vLine, tag->filePosition, pSeekValue);
 732 }
 733
 734 /*  Truncates the text line containing the tag at the character following the
 735  *  tag, providing a character which designates the end of the tag.
 736  *  Returns the length of the truncated line (or 0 if it doesn't truncate).
 737  */
 738 extern size_t truncateTagLineAfterTag (
 739                 char *const line, const char *const token, const bool discardNewline)
 740 {
 741         size_t len = 0;
 742         char *p = strstr (line, token);
 743
 744         if (p != NULL)
 745         {
 746                 p += strlen (token);
 747                 if (*p != '\0'  &&  ! (*p == '\n'  &&  discardNewline))
 748                         ++p;    /* skip past character terminating character */
 749                 *p = '\0';
 750                 len = p - line;
 751         }
 752
 753         return len;
 754 }
 755
 756 static char* getFullQualifiedScopeNameFromCorkQueue (const tagEntryInfo * inner_scope)
 757 {
 758
 759         int kindIndex = KIND_GHOST_INDEX;
 760         langType lang;
 761         const tagEntryInfo *scope = inner_scope;
 762         const tagEntryInfo *root_scope = NULL;
 763         stringList *queue = stringListNew ();
 764         vString *v;
 765         vString *n;
 766         unsigned int c;
 767         const char *sep;
 768
 769         while (scope)
 770         {
 771                 if (!scope->placeholder)
 772                 {
 773                         if (kindIndex != KIND_GHOST_INDEX)
 774                         {
 775                                 sep = scopeSeparatorFor (lang, kindIndex, scope->kindIndex);
 776                                 v = vStringNewInit (sep);
 777                                 stringListAdd (queue, v);
 778                         }
 779                         /* TODO: scope field of SCOPE can be used for optimization. */
 780                         v = vStringNewInit (scope->name);
 781                         stringListAdd (queue, v);
 782                         kindIndex = scope->kindIndex;
 783                         lang = scope->langType;
 784                         root_scope = scope;
 785                 }
 786                 scope =  getEntryInCorkQueue (scope->extensionFields.scopeIndex);
 787         }
 788
 789         n = vStringNew ();
 790         sep = root_scope? scopeSeparatorFor (root_scope->langType, root_scope->kindIndex, KIND_GHOST_INDEX): NULL;
 791         if (sep)
 792                 vStringCatS(n, sep);
 793
 794         while ((c = stringListCount (queue)) > 0)
 795         {
 796                 v = stringListLast (queue);
 797                 vStringCat (n, v);
 798                 vStringDelete (v);
 799                 stringListRemoveLast (queue);
 800         }
 801         stringListDelete (queue);
 802
 803         return vStringDeleteUnwrap (n);
 804 }
 805
 806 extern void getTagScopeInformation (tagEntryInfo *const tag,
 807                                     const char **kind, const char **name)
 808 {
 809         if (kind)
 810                 *kind = NULL;
 811         if (name)
 812                 *name = NULL;
 813
 814         const tagEntryInfo * scope = getEntryInCorkQueue (tag->extensionFields.scopeIndex);
 815         if (tag->extensionFields.scopeKindIndex == KIND_GHOST_INDEX
 816             && tag->extensionFields.scopeName == NULL
 817             && scope
 818             && ptrArrayCount (TagFile.corkQueue) > 0)
 819         {
 820                 char *full_qualified_scope_name = getFullQualifiedScopeNameFromCorkQueue(scope);
 821                 Assert (full_qualified_scope_name);
 822
 823                 /* Make the information reusable to generate full qualified entry, and xformat output*/
 824                 tag->extensionFields.scopeLangType = scope->langType;
 825                 tag->extensionFields.scopeKindIndex = scope->kindIndex;
 826                 tag->extensionFields.scopeName = full_qualified_scope_name;
 827         }
 828
 829         if (tag->extensionFields.scopeKindIndex != KIND_GHOST_INDEX  &&
 830             tag->extensionFields.scopeName != NULL)
 831         {
 832                 if (kind)
 833                 {
 834                         langType lang = (tag->extensionFields.scopeLangType == LANG_AUTO)
 835                                 ? tag->langType
 836                                 : tag->extensionFields.scopeLangType;
 837                         kindDefinition *kdef = getLanguageKind (lang,
 838                                                                                                         tag->extensionFields.scopeKindIndex);
 839                         *kind = kdef->name;
 840                 }
 841                 if (name)
 842                         *name = tag->extensionFields.scopeName;
 843         }
 844 }
 845
 846
 847 static int   makePatternStringCommon (const tagEntryInfo *const tag,
 848                                       int (* putc_func) (char , void *),
 849                                       int (* puts_func) (const char* , void *),
 850                                       void *output)
 851 {
 852         int length = 0;
 853
 854         char *line;
 855         int searchChar;
 856         const char *terminator;
 857         bool  omitted;
 858         size_t line_len;
 859
 860         bool making_cache = false;
 861         int (* puts_o_func)(const char* , void *);
 862         void * o_output;
 863
 864         static vString *cached_pattern;
 865         static MIOPos   cached_location;
 866         if (TagFile.patternCacheValid
 867             && (! tag->truncateLineAfterTag)
 868             && (memcmp (&tag->filePosition, &cached_location, sizeof(MIOPos)) == 0))
 869                 return puts_func (vStringValue (cached_pattern), output);
 870
 871         line = readLineFromBypassForTag (TagFile.vLine, tag, NULL);
 872         if (line == NULL)
 873         {
 874                 /* This can be occurs if the size of input file is zero, and
 875                    an empty regex pattern (//) matches to the input. */
 876                 line = "";
 877                 line_len = 0;
 878         }
 879         else
 880                 line_len = vStringLength (TagFile.vLine);
 881
 882         if (tag->truncateLineAfterTag)
 883         {
 884                 size_t truncted_len;
 885
 886                 truncted_len = truncateTagLineAfterTag (line, tag->name, false);
 887                 if (truncted_len > 0)
 888                         line_len = truncted_len;
 889         }
 890
 891         searchChar = Option.backward ? '?' : '/';
 892         terminator = (line_len > 0 && (line [line_len - 1] == '\n')) ? "$": "";
 893
 894         if (!tag->truncateLineAfterTag)
 895         {
 896                 making_cache = true;
 897                 cached_pattern = vStringNewOrClearWithAutoRelease (cached_pattern);
 898
 899                 puts_o_func = puts_func;
 900                 o_output    = output;
 901                 putc_func   = vstring_putc;
 902                 puts_func   = vstring_puts;
 903                 output      = cached_pattern;
 904         }
 905
 906         length += putc_func(searchChar, output);
 907         if ((tag->boundaryInfo & BOUNDARY_START) == 0)
 908                 length += putc_func('^', output);
 909         length += appendInputLine (putc_func, line, Option.patternLengthLimit,
 910                                                            output, &omitted);
 911         length += puts_func (omitted? "": terminator, output);
 912         length += putc_func (searchChar, output);
 913
 914         if (making_cache)
 915         {
 916                 puts_o_func (vStringValue (cached_pattern), o_output);
 917                 cached_location = tag->filePosition;
 918                 TagFile.patternCacheValid = true;
 919         }
 920
 921         return length;
 922 }
 923
 924 extern char* makePatternString (const tagEntryInfo *const tag)
 925 {
 926         vString* pattern = vStringNew ();
 927         makePatternStringCommon (tag, vstring_putc, vstring_puts, pattern);
 928         return vStringDeleteUnwrap (pattern);
 929 }
 930
 931 static tagField * tagFieldNew(fieldType ftype, const char *value, bool valueOwner)
 932 {
 933         tagField *f = xMalloc (1, tagField);
 934
 935         f->ftype = ftype;
 936         f->value = value;
 937         f->valueOwner = valueOwner;
 938         return f;
 939 }
 940
 941 static void tagFieldDelete (tagField * f)
 942 {
 943         if (f->valueOwner)
 944                 eFree((void *)f->value);
 945         eFree (f);
 946 }
 947
 948 static void attachParserFieldGeneric (tagEntryInfo *const tag, fieldType ftype, const char * value,
 949                                                                           bool valueOwner)
 950 {
 951         if (tag->usedParserFields < PRE_ALLOCATED_PARSER_FIELDS)
 952         {
 953                 tag->parserFields [tag->usedParserFields].ftype = ftype;
 954                 tag->parserFields [tag->usedParserFields].value = value;
 955                 tag->parserFields [tag->usedParserFields].valueOwner = valueOwner;
 956                 tag->usedParserFields++;
 957         }
 958         else if (tag->parserFieldsDynamic == NULL)
 959         {
 960                 tag->parserFieldsDynamic = ptrArrayNew((ptrArrayDeleteFunc)tagFieldDelete);
 961                 PARSER_TRASH_BOX(tag->parserFieldsDynamic, ptrArrayDelete);
 962                 attachParserFieldGeneric (tag, ftype, value, valueOwner);
 963         }
 964         else
 965         {
 966                 tagField *f = tagFieldNew (ftype, value, valueOwner);
 967                 ptrArrayAdd(tag->parserFieldsDynamic, f);
 968                 tag->usedParserFields++;
 969         }
 970 }
 971
 972 extern void attachParserField (tagEntryInfo *const tag, bool inCorkQueue, fieldType ftype, const char * value)
 973 {
 974         Assert (tag != NULL);
 975
 976         if (inCorkQueue)
 977         {
 978                 const char * v;
 979                 v = eStrdup (value);
 980
 981                 bool dynfields_allocated = tag->parserFieldsDynamic? true: false;
 982                 attachParserFieldGeneric (tag, ftype, v, true);
 983                 if (!dynfields_allocated && tag->parserFieldsDynamic)
 984                         PARSER_TRASH_BOX_TAKE_BACK(tag->parserFieldsDynamic);
 985         }
 986         else
 987                 attachParserFieldGeneric (tag, ftype, value, false);
 988 }
 989
 990 extern void attachParserFieldToCorkEntry (int index,
 991                                          fieldType ftype,
 992                                          const char *value)
 993 {
 994         tagEntryInfo * tag = getEntryInCorkQueue (index);
 995         if (tag)
 996                 attachParserField (tag, true, ftype, value);
 997 }
 998
 999 extern const tagField* getParserFieldForIndex (const tagEntryInfo * tag, int index)
1000 {
1001         if (index < 0
1002                 || tag->usedParserFields <= ((unsigned int)index) )
1003                 return NULL;
1004         else if (index < PRE_ALLOCATED_PARSER_FIELDS)
1005                 return tag->parserFields + index;
1006         else
1007         {
1008                 unsigned int n = index - PRE_ALLOCATED_PARSER_FIELDS;
1009                 return ptrArrayItem(tag->parserFieldsDynamic, n);
1010         }
1011 }
1012
1013 extern const char* getParserFieldValueForType (tagEntryInfo *const tag, fieldType ftype)
1014 {
1015         for (int i = 0; i < tag->usedParserFields; i++)
1016         {
1017                 const tagField *f = getParserFieldForIndex (tag, i);
1018                 if (f && f->ftype == ftype)
1019                         return f->value;
1020         }
1021         return NULL;
1022 }
1023
1024 static void copyParserFields (const tagEntryInfo *const tag, tagEntryInfo* slot)
1025 {
1026         unsigned int i;
1027         const char* value;
1028
1029         for (i = 0; i < tag->usedParserFields; i++)
1030         {
1031                 const tagField *f = getParserFieldForIndex (tag, i);
1032                 Assert(f);
1033
1034                 value = f->value;
1035                 if (value)
1036                         value = eStrdup (value);
1037
1038                 attachParserFieldGeneric (slot,
1039                                                                   f->ftype,
1040                                                                   value,
1041                                                                   true);
1042         }
1043
1044 }
1045
1046 static tagEntryInfo *newNilTagEntry (unsigned int corkFlags)
1047 {
1048         tagEntryInfoX *x = xCalloc (1, tagEntryInfoX);
1049         x->corkIndex = CORK_NIL;
1050         x->symtab = RB_ROOT;
1051         x->slot.kindIndex = KIND_FILE_INDEX;
1052         return &(x->slot);
1053 }
1054
1055 static tagEntryInfoX *copyTagEntry (const tagEntryInfo *const tag,
1056                                                                    unsigned int corkFlags)
1057 {
1058         tagEntryInfoX *x = xMalloc (1, tagEntryInfoX);
1059         x->symtab = RB_ROOT;
1060         x->corkIndex = CORK_NIL;
1061         tagEntryInfo  *slot = (tagEntryInfo *)x;
1062
1063         *slot = *tag;
1064
1065         if (slot->pattern)
1066                 slot->pattern = eStrdup (slot->pattern);
1067
1068         slot->inputFileName = eStrdup (slot->inputFileName);
1069         slot->name = eStrdup (slot->name);
1070         if (slot->extensionFields.access)
1071                 slot->extensionFields.access = eStrdup (slot->extensionFields.access);
1072         if (slot->extensionFields.implementation)
1073                 slot->extensionFields.implementation = eStrdup (slot->extensionFields.implementation);
1074         if (slot->extensionFields.inheritance)
1075                 slot->extensionFields.inheritance = eStrdup (slot->extensionFields.inheritance);
1076         if (slot->extensionFields.scopeName)
1077                 slot->extensionFields.scopeName = eStrdup (slot->extensionFields.scopeName);
1078         if (slot->extensionFields.signature)
1079                 slot->extensionFields.signature = eStrdup (slot->extensionFields.signature);
1080         if (slot->extensionFields.typeRef[0])
1081                 slot->extensionFields.typeRef[0] = eStrdup (slot->extensionFields.typeRef[0]);
1082         if (slot->extensionFields.typeRef[1])
1083                 slot->extensionFields.typeRef[1] = eStrdup (slot->extensionFields.typeRef[1]);
1084 #ifdef HAVE_LIBXML
1085         if (slot->extensionFields.xpath)
1086                 slot->extensionFields.xpath = eStrdup (slot->extensionFields.xpath);
1087 #endif
1088
1089         if (slot->extraDynamic)
1090         {
1091                 int n = countXtags () - XTAG_COUNT;
1092                 slot->extraDynamic = xCalloc ((n / 8) + 1, uint8_t);
1093                 memcpy (slot->extraDynamic, tag->extraDynamic, (n / 8) + 1);
1094         }
1095
1096         if (slot->sourceFileName)
1097                 slot->sourceFileName = eStrdup (slot->sourceFileName);
1098
1099
1100         slot->usedParserFields = 0;
1101         slot->parserFieldsDynamic = NULL;
1102         copyParserFields (tag, slot);
1103         if (slot->parserFieldsDynamic)
1104                 PARSER_TRASH_BOX_TAKE_BACK(slot->parserFieldsDynamic);
1105
1106         return x;
1107 }
1108
1109 static void clearParserFields (tagEntryInfo *const tag)
1110 {
1111         unsigned int i, n;
1112         const char* value;
1113
1114         if ( tag->usedParserFields < PRE_ALLOCATED_PARSER_FIELDS )
1115                 n = tag->usedParserFields;
1116         else
1117                 n = PRE_ALLOCATED_PARSER_FIELDS;
1118
1119         for (i = 0; i < n; i++)
1120         {
1121                 value = tag->parserFields[i].value;
1122                 if (value && tag->parserFields[i].valueOwner)
1123                         eFree ((char *)value);
1124                 tag->parserFields[i].value = NULL;
1125                 tag->parserFields[i].ftype = FIELD_UNKNOWN;
1126         }
1127         if (tag->parserFieldsDynamic)
1128         {
1129                 ptrArrayDelete (tag->parserFieldsDynamic);
1130                 tag->parserFieldsDynamic = NULL;
1131         }
1132 }
1133
1134 static void deleteTagEnry (void *data)
1135 {
1136         tagEntryInfo *slot = data;
1137
1138         if (slot->kindIndex == KIND_FILE_INDEX)
1139                 goto out;
1140
1141         if (slot->pattern)
1142                 eFree ((char *)slot->pattern);
1143         eFree ((char *)slot->inputFileName);
1144         eFree ((char *)slot->name);
1145
1146         if (slot->extensionFields.access)
1147                 eFree ((char *)slot->extensionFields.access);
1148         if (slot->extensionFields.implementation)
1149                 eFree ((char *)slot->extensionFields.implementation);
1150         if (slot->extensionFields.inheritance)
1151                 eFree ((char *)slot->extensionFields.inheritance);
1152         if (slot->extensionFields.scopeName)
1153                 eFree ((char *)slot->extensionFields.scopeName);
1154         if (slot->extensionFields.signature)
1155                 eFree ((char *)slot->extensionFields.signature);
1156         if (slot->extensionFields.typeRef[0])
1157                 eFree ((char *)slot->extensionFields.typeRef[0]);
1158         if (slot->extensionFields.typeRef[1])
1159                 eFree ((char *)slot->extensionFields.typeRef[1]);
1160 #ifdef HAVE_LIBXML
1161         if (slot->extensionFields.xpath)
1162                 eFree ((char *)slot->extensionFields.xpath);
1163 #endif
1164
1165         if (slot->extraDynamic)
1166                 eFree (slot->extraDynamic);
1167
1168         if (slot->sourceFileName)
1169                 eFree ((char *)slot->sourceFileName);
1170
1171         clearParserFields (slot);
1172
1173  out:
1174         eFree (slot);
1175 }
1176
1177 static void corkSymtabPut (tagEntryInfoX *scope, const char* name, tagEntryInfoX *item)
1178 {
1179         struct rb_root *root = &scope->symtab;
1180
1181         struct rb_node **new = &(root->rb_node), *parent = NULL;
1182
1183         while (*new)
1184         {
1185                 tagEntryInfoX *this = container_of(*new, tagEntryInfoX, symnode);
1186                 int result = strcmp(item->slot.name, this->slot.name);
1187
1188                 parent = *new;
1189
1190                 if (result < 0)
1191                         new = &((*new)->rb_left);
1192                 else if (result > 0)
1193                         new = &((*new)->rb_right);
1194                 else
1195                 {
1196                         unsigned long lthis = this->slot.lineNumber;
1197                         unsigned long litem = item->slot.lineNumber;
1198
1199                         /* Comparing lineNumber */
1200                         if (litem < lthis)
1201                                 new = &((*new)->rb_left);
1202                         else if (litem > lthis)
1203                                 new = &((*new)->rb_right);
1204                         else
1205                         {
1206                                 /* Comparing memory address */
1207                                 if (item < this)
1208                                         new = &((*new)->rb_left);
1209                                 else if (item > this)
1210                                         new = &((*new)->rb_right);
1211                                 else
1212                                 {
1213                                         AssertNotReached(); /* registering the same object twice. */
1214                                         return;
1215                                 }
1216                         }
1217                 }
1218         }
1219
1220         verbose ("symtbl[:=] %s<-%s/%p (line: %lu)\n",
1221                         *new? container_of(*new, tagEntryInfoX, symnode)->slot.name: "*root*",
1222                          item->slot.name, &item->slot, item->slot.lineNumber);
1223         /* Add new node and rebalance tree. */
1224         rb_link_node(&item->symnode, parent, new);
1225         rb_insert_color(&item->symnode, root);
1226 }
1227
1228 extern bool foreachEntriesInScope (int corkIndex,
1229                                                                    const char *name,
1230                                                                    entryForeachFunc func,
1231                                                                    void *data)
1232 {
1233         tagEntryInfoX *x = ptrArrayItem (TagFile.corkQueue, corkIndex);
1234
1235         struct rb_root *root = &x->symtab;
1236         tagEntryInfoX *rep = NULL;
1237
1238         /* More than one tag can have a same name.
1239          * Visit them from the last.
1240          *
1241          * 1. find one of them as the representative,
1242          * 2. find the last one of them from the representative with rb_next,
1243          * 3. call FUNC iteratively from the last to the first.
1244          */
1245         if (name)
1246         {
1247                 struct rb_node *node = root->rb_node;
1248                 while (node)
1249                 {
1250                         tagEntryInfoX *entry = container_of(node, tagEntryInfoX, symnode);
1251                         int result;
1252
1253                         result = strcmp(name, entry->slot.name);
1254
1255                         if (result < 0)
1256                                 node = node->rb_left;
1257                         else if (result > 0)
1258                                 node = node->rb_right;
1259                         else
1260                         {
1261                                 rep = entry;
1262                                 break;
1263                         }
1264                 }
1265                 if (rep == NULL)
1266                         return true;
1267
1268                 verbose("symtbl[<>] %s->%p\n", name, &rep->slot);
1269         }
1270
1271         struct rb_node *last;
1272
1273         if (name)
1274         {
1275                 struct rb_node *tmp = &rep->symnode;
1276                 last = tmp;
1277
1278                 while ((tmp = rb_next (tmp)))
1279                 {
1280                         tagEntryInfoX *entry = container_of(tmp, tagEntryInfoX, symnode);
1281                         if (strcmp(name, entry->slot.name) == 0)
1282                         {
1283                                 verbose ("symtbl[ >] %s->%p\n", name, &container_of(tmp, tagEntryInfoX, symnode)->slot);
1284                                 last = tmp;
1285                         }
1286                         else
1287                                 break;
1288                 }
1289         }
1290         else
1291         {
1292                 last = rb_last(root);
1293                 verbose ("last for %d<%p>: %p\n", corkIndex, root, last);
1294         }
1295
1296         if (!last)
1297         {
1298                 verbose ("symtbl[>V] %s->%p\n", name? name: "(null)", NULL);
1299                 return true;                    /* Nothing here in this node. */
1300         }
1301
1302         verbose ("symtbl[>|] %s->%p\n", name, &container_of(last, tagEntryInfoX, symnode)->slot);
1303
1304         struct rb_node *cursor = last;
1305         bool revisited_rep = false;
1306         do
1307         {
1308                 tagEntryInfoX *entry = container_of(cursor, tagEntryInfoX, symnode);
1309                 if (!revisited_rep || !name || strcmp(name, entry->slot.name))
1310                 {
1311                         verbose ("symtbl[< ] %s->%p\n", name, &entry->slot);
1312                         if (!func (entry->corkIndex, &entry->slot, data))
1313                                 return false;
1314                         if (cursor == &rep->symnode)
1315                                 revisited_rep = true;
1316                 }
1317                 else if (name)
1318                         break;
1319         }
1320         while ((cursor = rb_prev(cursor)));
1321
1322         return true;
1323 }
1324
1325 static bool findName (int corkIndex, tagEntryInfo *entry, void *data)
1326 {
1327         int *index = data;
1328
1329         *index =  corkIndex;
1330         return false;
1331 }
1332
1333 int anyEntryInScope (int corkIndex, const char *name)
1334 {
1335         int index = CORK_NIL;
1336
1337         if (foreachEntriesInScope (corkIndex, name, findName, &index) == false)
1338                 return index;
1339
1340         return CORK_NIL;
1341 }
1342
1343 struct anyKindsEntryInScopeData {
1344         int  index;
1345         const int *kinds;
1346         int  count;
1347 };
1348
1349 static bool findNameOfKinds (int corkIndex, tagEntryInfo *entry, void *data)
1350 {
1351         struct anyKindsEntryInScopeData * kdata = data;
1352
1353         for (int i = 0; i < kdata->count; i++)
1354         {
1355                 int k = kdata->kinds [i];
1356                 if (entry->kindIndex == k)
1357                 {
1358                         kdata->index = corkIndex;
1359                         return false;
1360                 }
1361         }
1362         return true;
1363 }
1364
1365 int anyKindEntryInScope (int corkIndex,
1366                                                  const char *name, int kind)
1367 {
1368         return anyKindsEntryInScope (corkIndex, name, &kind, 1);
1369 }
1370
1371 int anyKindsEntryInScope (int corkIndex,
1372                                                   const char *name,
1373                                                   const int *kinds, int count)
1374 {
1375         struct anyKindsEntryInScopeData data = {
1376                 .index = CORK_NIL,
1377                 .kinds = kinds,
1378                 .count = count,
1379         };
1380
1381         if (foreachEntriesInScope (corkIndex, name, findNameOfKinds, &data) == false)
1382                 return data.index;
1383
1384         return CORK_NIL;
1385 }
1386
1387 int anyKindsEntryInScopeRecursive (int corkIndex,
1388                                                                    const char *name,
1389                                                                    const int *kinds, int count)
1390 {
1391         struct anyKindsEntryInScopeData data = {
1392                 .index = CORK_NIL,
1393                 .kinds = kinds,
1394                 .count = count,
1395         };
1396
1397         tagEntryInfo *e;
1398         do
1399         {
1400                 if (foreachEntriesInScope (corkIndex, name, findNameOfKinds, &data) == false)
1401                         return data.index;
1402
1403                 if (corkIndex == CORK_NIL)
1404                         break;
1405
1406                 e = getEntryInCorkQueue (corkIndex);
1407                 if (!e)
1408                         break;
1409                 corkIndex = e->extensionFields.scopeIndex;
1410         }
1411         while (1);
1412
1413         return CORK_NIL;
1414 }
1415
1416 extern void registerEntry (int corkIndex)
1417 {
1418         Assert (TagFile.corkFlags & CORK_SYMTAB);
1419         Assert (corkIndex != CORK_NIL);
1420
1421         tagEntryInfoX *e = ptrArrayItem (TagFile.corkQueue, corkIndex);
1422         {
1423                 tagEntryInfoX *scope = ptrArrayItem (TagFile.corkQueue, e->slot.extensionFields.scopeIndex);
1424                 corkSymtabPut (scope, e->slot.name, e);
1425         }
1426 }
1427
1428 static int queueTagEntry(const tagEntryInfo *const tag)
1429 {
1430         static bool warned;
1431
1432         int corkIndex;
1433         tagEntryInfoX * entry = copyTagEntry (tag,
1434                                                                                 TagFile.corkFlags);
1435
1436         if (ptrArrayCount (TagFile.corkQueue) == (size_t)INT_MAX)
1437         {
1438                 if (!warned)
1439                 {
1440                         warned = true;
1441                         error (WARNING,
1442                                    "The tag entry queue overflows; drop the tag entry at %lu in %s",
1443                                    tag->lineNumber,
1444                                    tag->inputFileName);
1445                 }
1446                 return CORK_NIL;
1447         }
1448         warned = false;
1449
1450         corkIndex = (int)ptrArrayAdd (TagFile.corkQueue, entry);
1451         entry->corkIndex = corkIndex;
1452         entry->slot.inCorkQueue = 1;
1453
1454         return corkIndex;
1455 }
1456
1457 extern void setupWriter (void *writerClientData)
1458 {
1459         writerSetup (TagFile.mio, writerClientData);
1460 }
1461
1462 extern bool teardownWriter (const char *filename)
1463 {
1464         return writerTeardown (TagFile.mio, filename);
1465 }
1466
1467 static bool isTagWritable(const tagEntryInfo *const tag)
1468 {
1469         if (tag->placeholder)
1470                 return false;
1471
1472         if (! isLanguageKindEnabled(tag->langType, tag->kindIndex))
1473                 return false;
1474
1475         if (tag->extensionFields.roleBits)
1476         {
1477                 size_t available_roles;
1478
1479                 if (!isXtagEnabled (XTAG_REFERENCE_TAGS))
1480                         return false;
1481
1482                 available_roles = countLanguageRoles(tag->langType,
1483                                                                                          tag->kindIndex);
1484                 if (tag->extensionFields.roleBits >=
1485                         (makeRoleBit(available_roles)))
1486                         return false;
1487
1488                 /* TODO: optimization
1489                    A Bitmasks representing all enabled roles can be calculated at the
1490                    end of initializing the parser. Calculating each time when checking
1491                    a tag entry is not needed. */
1492                 for (unsigned int roleIndex = 0; roleIndex < available_roles; roleIndex++)
1493                 {
1494                         if (isRoleAssigned(tag, roleIndex))
1495                         {
1496                                 if (isLanguageRoleEnabled (tag->langType, tag->kindIndex,
1497                                                                                          roleIndex))
1498                                         return true;
1499                         }
1500
1501                 }
1502                 return false;
1503         }
1504         else if (isLanguageKindRefOnly(tag->langType, tag->kindIndex))
1505         {
1506                 error (WARNING, "definition tag for refonly kind(%s) is made: %s",
1507                            getLanguageKind(tag->langType, tag->kindIndex)->name,
1508                            tag->name);
1509                 /* This one is not so critical. */
1510         }
1511
1512         if (!isXtagEnabled(XTAG_ANONYMOUS)
1513                 && isTagExtraBitMarked(tag, XTAG_ANONYMOUS))
1514                 return false;
1515
1516         return true;
1517 }
1518
1519 static void buildFqTagCache (tagEntryInfo *const tag)
1520 {
1521         getTagScopeInformation (tag, NULL, NULL);
1522 }
1523
1524 static void writeTagEntry (const tagEntryInfo *const tag)
1525 {
1526         int length = 0;
1527
1528         Assert (tag->kindIndex != KIND_GHOST_INDEX);
1529
1530         DebugStatement ( debugEntry (tag); )
1531
1532 #ifdef WIN32
1533         if (getFilenameSeparator(Option.useSlashAsFilenameSeparator) == FILENAME_SEP_USE_SLASH)
1534         {
1535                 Assert (((const tagEntryInfo *)tag)->inputFileName);
1536                 char *c = (char *)(((tagEntryInfo *const)tag)->inputFileName);
1537                 while (*c)
1538                 {
1539                         if (*c == PATH_SEPARATOR)
1540                                 *c = OUTPUT_PATH_SEPARATOR;
1541                         c++;
1542                 }
1543         }
1544 #endif
1545
1546         if (includeExtensionFlags ()
1547             && isXtagEnabled (XTAG_QUALIFIED_TAGS)
1548             && doesInputLanguageRequestAutomaticFQTag ()
1549                 && !isTagExtraBitMarked (tag, XTAG_QUALIFIED_TAGS)
1550                 && !tag->skipAutoFQEmission)
1551         {
1552                 /* const is discarded to update the cache field of TAG. */
1553                 buildFqTagCache ( (tagEntryInfo *const)tag);
1554         }
1555
1556         length = writerWriteTag (TagFile.mio, tag);
1557
1558         if (length > 0)
1559         {
1560                 ++TagFile.numTags.added;
1561                 rememberMaxLengths (strlen (tag->name), (size_t) length);
1562         }
1563         DebugStatement ( if (TagFile.mio) mio_flush (TagFile.mio); )
1564
1565         abort_if_ferror (TagFile.mio);
1566 }
1567
1568 extern bool writePseudoTag (const ptagDesc *desc,
1569                                const char *const fileName,
1570                                const char *const pattern,
1571                                const char *const parserName)
1572 {
1573         int length;
1574
1575         length = writerWritePtag (TagFile.mio, desc, fileName,
1576                                                           pattern, parserName);
1577         if (length < 0)
1578                 return false;
1579
1580         abort_if_ferror (TagFile.mio);
1581
1582         ++TagFile.numTags.added;
1583         rememberMaxLengths (strlen (desc->name), (size_t) length);
1584
1585         return true;
1586 }
1587
1588 extern void corkTagFile(unsigned int corkFlags)
1589 {
1590         TagFile.cork++;
1591         if (TagFile.cork == 1)
1592         {
1593                 TagFile.corkFlags = corkFlags;
1594                 TagFile.corkQueue = ptrArrayNew (deleteTagEnry);
1595                 tagEntryInfo *nil = newNilTagEntry (corkFlags);
1596                 ptrArrayAdd (TagFile.corkQueue, nil);
1597         }
1598 }
1599
1600 extern void uncorkTagFile(void)
1601 {
1602         unsigned int i;
1603
1604         TagFile.cork--;
1605
1606         if (TagFile.cork > 0)
1607                 return ;
1608
1609         for (i = 1; i < ptrArrayCount (TagFile.corkQueue); i++)
1610         {
1611                 tagEntryInfo *tag = ptrArrayItem (TagFile.corkQueue, i);
1612
1613                 if (!isTagWritable(tag))
1614                         continue;
1615
1616                 writeTagEntry (tag);
1617
1618                 if (doesInputLanguageRequestAutomaticFQTag ()
1619                     && isXtagEnabled (XTAG_QUALIFIED_TAGS)
1620                         && !isTagExtraBitMarked (tag, XTAG_QUALIFIED_TAGS)
1621                         && !tag->skipAutoFQEmission
1622                         && ((tag->extensionFields.scopeKindIndex != KIND_GHOST_INDEX
1623                                  && tag->extensionFields.scopeName != NULL
1624                                  && tag->extensionFields.scopeIndex != CORK_NIL)
1625                                 || (tag->extensionFields.scopeKindIndex == KIND_GHOST_INDEX
1626                                         && tag->extensionFields.scopeName == NULL
1627                                         && tag->extensionFields.scopeIndex == CORK_NIL)))
1628                         makeQualifiedTagEntry (tag);
1629         }
1630
1631         ptrArrayDelete (TagFile.corkQueue);
1632         TagFile.corkQueue = NULL;
1633 }
1634
1635 extern tagEntryInfo *getEntryInCorkQueue   (int n)
1636 {
1637         if ((CORK_NIL < n) && (((size_t)n) < ptrArrayCount (TagFile.corkQueue)))
1638                 return ptrArrayItem (TagFile.corkQueue, n);
1639         else
1640                 return NULL;
1641 }
1642
1643 extern tagEntryInfo *getEntryOfNestingLevel (const NestingLevel *nl)
1644 {
1645         if (nl == NULL)
1646                 return NULL;
1647         return getEntryInCorkQueue (nl->corkIndex);
1648 }
1649
1650 extern size_t        countEntryInCorkQueue (void)
1651 {
1652         return ptrArrayCount (TagFile.corkQueue);
1653 }
1654
1655 extern void markTagPlaceholder (tagEntryInfo *e, bool placeholder)
1656 {
1657         e->placeholder = placeholder;
1658 }
1659
1660 extern int makePlaceholder (const char *const name)
1661 {
1662         tagEntryInfo e;
1663
1664         initTagEntry (&e, name, KIND_GHOST_INDEX);
1665         markTagPlaceholder(&e, true);
1666
1667         /*
1668          * makePlaceholder may be called even before reading any bytes
1669          * from the input stream. In such case, initTagEntry fills
1670          * the lineNumber field of the placeholder tag with 0.
1671          * This breaks an assertion in makeTagEntry. Following adjustment
1672          * is for avoiding it.
1673          */
1674         if (e.lineNumber == 0)
1675                 e.lineNumber = 1;
1676
1677         return makeTagEntry (&e);
1678 }
1679
1680 extern int makeTagEntry (const tagEntryInfo *const tag)
1681 {
1682         int r = CORK_NIL;
1683         Assert (tag->name != NULL);
1684         Assert(tag->lineNumber > 0);
1685
1686         if (!TagFile.cork)
1687                 if (!isTagWritable (tag))
1688                         goto out;
1689
1690         if (tag->name [0] == '\0' && (!tag->placeholder))
1691         {
1692                 if (!doesInputLanguageAllowNullTag())
1693                         error (WARNING, "ignoring null tag in %s(line: %lu)",
1694                                getInputFileName (), tag->lineNumber);
1695                 goto out;
1696         }
1697
1698         if (TagFile.cork)
1699                 r = queueTagEntry (tag);
1700         else
1701                 writeTagEntry (tag);
1702
1703         if (r != CORK_NIL)
1704                 notifyMakeTagEntry (tag, r);
1705
1706 out:
1707         return r;
1708 }
1709
1710 extern int makeQualifiedTagEntry (const tagEntryInfo *const e)
1711 {
1712         int r = CORK_NIL;
1713         tagEntryInfo x;
1714         int xk;
1715         const char *sep;
1716         static vString *fqn;
1717
1718         if (isXtagEnabled (XTAG_QUALIFIED_TAGS))
1719         {
1720                 x = *e;
1721                 markTagExtraBit (&x, XTAG_QUALIFIED_TAGS);
1722
1723                 fqn = vStringNewOrClearWithAutoRelease (fqn);
1724
1725                 if (e->extensionFields.scopeName)
1726                 {
1727                         vStringCatS (fqn, e->extensionFields.scopeName);
1728                         xk = e->extensionFields.scopeKindIndex;
1729                         sep = scopeSeparatorFor (e->langType, e->kindIndex, xk);
1730                         vStringCatS (fqn, sep);
1731                 }
1732                 else
1733                 {
1734                         /* This is an top level item. prepend a root separator
1735                            if the kind of the item has. */
1736                         sep = scopeSeparatorFor (e->langType, e->kindIndex, KIND_GHOST_INDEX);
1737                         if (sep == NULL)
1738                         {
1739                                 /* No root separator. The name of the
1740                                    optional tag and that of full qualified tag
1741                                    are the same; recording the full qualified tag
1742                                    is meaningless. */
1743                                 return r;
1744                         }
1745                         else
1746                                 vStringCatS (fqn, sep);
1747                 }
1748                 vStringCatS (fqn, e->name);
1749
1750                 x.name = vStringValue (fqn);
1751                 /* makeExtraTagEntry of c.c doesn't clear scope
1752                    related fields. */
1753 #if 0
1754                 x.extensionFields.scopeKind = NULL;
1755                 x.extensionFields.scopeName = NULL;
1756                 x.extensionFields.scopeIndex = CORK_NIL;
1757 #endif
1758
1759                 bool in_subparser
1760                         = isTagExtraBitMarked (&x,
1761                                                                    XTAG_SUBPARSER);
1762
1763                 if (in_subparser)
1764                         pushLanguage(x.langType);
1765
1766                 r = makeTagEntry (&x);
1767
1768                 if (in_subparser)
1769                         popLanguage();
1770         }
1771         return r;
1772 }
1773
1774 extern void setTagPositionFromTag (tagEntryInfo *const dst,
1775                                                                    const tagEntryInfo *const src)
1776 {
1777                 dst->lineNumber = src->lineNumber;
1778                 dst->filePosition = src->filePosition;
1779 }
1780
1781 static void initTagEntryFull (tagEntryInfo *const e, const char *const name,
1782                               unsigned long lineNumber,
1783                               langType langType_,
1784                               MIOPos      filePosition,
1785                               const char *inputFileName,
1786                               int kindIndex,
1787                               roleBitsType roleBits,
1788                               const char *sourceFileName,
1789                               langType sourceLangType,
1790                               long sourceLineNumberDifference)
1791 {
1792         int i;
1793
1794         Assert (getInputFileName() != NULL);
1795
1796         memset (e, 0, sizeof (tagEntryInfo));
1797         e->lineNumberEntry = (bool) (Option.locate == EX_LINENUM);
1798         e->lineNumber      = lineNumber;
1799         e->boundaryInfo    = getNestedInputBoundaryInfo (lineNumber);
1800         e->langType        = langType_;
1801         e->filePosition    = filePosition;
1802         e->inputFileName   = inputFileName;
1803         e->name            = name;
1804         e->extensionFields.scopeLangType = LANG_AUTO;
1805         e->extensionFields.scopeKindIndex = KIND_GHOST_INDEX;
1806         e->extensionFields.scopeIndex     = CORK_NIL;
1807
1808         Assert (kindIndex < 0 || kindIndex < (int)countLanguageKinds(langType_));
1809         e->kindIndex = kindIndex;
1810
1811         Assert (roleBits == 0
1812                         || (roleBits < (makeRoleBit(countLanguageRoles(langType_, kindIndex)))));
1813         e->extensionFields.roleBits = roleBits;
1814         if (roleBits)
1815                 markTagExtraBit (e, XTAG_REFERENCE_TAGS);
1816
1817         e->extensionFields.nth = NO_NTH_FIELD;
1818
1819         if (doesParserRunAsGuest ())
1820                 markTagExtraBit (e, XTAG_GUEST);
1821         if (doesSubparserRun ())
1822                 markTagExtraBit (e, XTAG_SUBPARSER);
1823
1824         e->sourceLangType = sourceLangType;
1825         e->sourceFileName = sourceFileName;
1826         e->sourceLineNumberDifference = sourceLineNumberDifference;
1827
1828         e->usedParserFields = 0;
1829
1830         for ( i = 0; i < PRE_ALLOCATED_PARSER_FIELDS; i++ )
1831                 e->parserFields[i].ftype = FIELD_UNKNOWN;
1832
1833         if (isParserMarkedNoEmission ())
1834                 e->placeholder = 1;
1835 }
1836
1837 extern void initTagEntry (tagEntryInfo *const e, const char *const name,
1838                                                   int kindIndex)
1839 {
1840         initTagEntryFull(e, name,
1841                          getInputLineNumber (),
1842                          getInputLanguage (),
1843                          getInputFilePosition (),
1844                          getInputFileTagPath (),
1845                          kindIndex,
1846                          0,
1847                          getSourceFileTagPath(),
1848                          getSourceLanguage(),
1849                          getSourceLineNumber() - getInputLineNumber ());
1850 }
1851
1852 extern void initRefTagEntry (tagEntryInfo *const e, const char *const name,
1853                              int kindIndex, int roleIndex)
1854 {
1855         initForeignRefTagEntry (e, name, getInputLanguage (), kindIndex, roleIndex);
1856 }
1857
1858 extern void initForeignRefTagEntry (tagEntryInfo *const e, const char *const name,
1859                                                                         langType langType,
1860                                                                         int kindIndex, int roleIndex)
1861 {
1862         initTagEntryFull(e, name,
1863                          getInputLineNumber (),
1864                          langType,
1865                          getInputFilePosition (),
1866                          getInputFileTagPath (),
1867                          kindIndex,
1868                          makeRoleBit(roleIndex),
1869                          getSourceFileTagPath(),
1870                          getSourceLanguage(),
1871                          getSourceLineNumber() - getInputLineNumber ());
1872 }
1873
1874 static void    markTagExtraBitFull     (tagEntryInfo *const tag, xtagType extra, bool mark)
1875 {
1876         unsigned int index;
1877         unsigned int offset;
1878         uint8_t *slot;
1879
1880         Assert (extra != XTAG_UNKNOWN);
1881
1882         if (extra < XTAG_COUNT)
1883         {
1884                 index = (extra / 8);
1885                 offset = (extra % 8);
1886                 slot = tag->extra;
1887         }
1888         else if (tag->extraDynamic)
1889         {
1890                 Assert (extra < countXtags ());
1891
1892                 index = ((extra - XTAG_COUNT) / 8);
1893                 offset = ((extra - XTAG_COUNT) % 8);
1894                 slot = tag->extraDynamic;
1895         }
1896         else
1897         {
1898                 Assert (extra < countXtags ());
1899
1900                 int n = countXtags () - XTAG_COUNT;
1901                 tag->extraDynamic = xCalloc ((n / 8) + 1, uint8_t);
1902                 if (!tag->inCorkQueue)
1903                         PARSER_TRASH_BOX(tag->extraDynamic, eFree);
1904                 markTagExtraBitFull (tag, extra, mark);
1905                 return;
1906         }
1907
1908         if (mark)
1909                 slot [ index ] |= (1 << offset);
1910         else
1911                 slot [ index ] &= ~(1 << offset);
1912 }
1913
1914 extern void    markTagExtraBit     (tagEntryInfo *const tag, xtagType extra)
1915 {
1916         markTagExtraBitFull (tag, extra, true);
1917 }
1918
1919 extern void    unmarkTagExtraBit    (tagEntryInfo *const tag, xtagType extra)
1920 {
1921         markTagExtraBitFull (tag, extra, false);
1922 }
1923
1924 extern bool isTagExtraBitMarked (const tagEntryInfo *const tag, xtagType extra)
1925 {
1926         unsigned int index;
1927         unsigned int offset;
1928         const uint8_t *slot;
1929
1930         Assert (extra != XTAG_UNKNOWN);
1931
1932         if (extra < XTAG_COUNT)
1933         {
1934                 index = (extra / 8);
1935                 offset = (extra % 8);
1936                 slot = tag->extra;
1937         }
1938         else if (!tag->extraDynamic)
1939                 return false;
1940         else
1941         {
1942                 Assert (extra < countXtags ());
1943                 index = ((extra - XTAG_COUNT) / 8);
1944                 offset = ((extra - XTAG_COUNT) % 8);
1945                 slot = tag->extraDynamic;
1946         }
1947         return !! ((slot [ index ]) & (1 << offset));
1948 }
1949
1950 extern bool isTagExtra (const tagEntryInfo *const tag)
1951 {
1952         for (unsigned int i = 0; i < countXtags(); i++)
1953                 if (isTagExtraBitMarked (tag, i))
1954                         return true;
1955         return false;
1956 }
1957
1958 static void assignRoleFull(tagEntryInfo *const e, int roleIndex, bool assign)
1959 {
1960         if (roleIndex == ROLE_DEFINITION_INDEX)
1961         {
1962                 if (assign)
1963                 {
1964                         e->extensionFields.roleBits = 0;
1965                         markTagExtraBitFull (e, XTAG_REFERENCE_TAGS, false);
1966                 }
1967         }
1968         else if (roleIndex > ROLE_DEFINITION_INDEX)
1969         {
1970                 Assert (roleIndex < (int)countLanguageRoles(e->langType, e->kindIndex));
1971
1972                 if (assign)
1973                         e->extensionFields.roleBits |= (makeRoleBit(roleIndex));
1974                 else
1975                         e->extensionFields.roleBits &= ~(makeRoleBit(roleIndex));
1976                 markTagExtraBitFull (e, XTAG_REFERENCE_TAGS, e->extensionFields.roleBits);
1977         }
1978         else
1979                 AssertNotReached();
1980 }
1981
1982 extern void assignRole(tagEntryInfo *const e, int roleIndex)
1983 {
1984         assignRoleFull(e, roleIndex, true);
1985 }
1986
1987 extern bool isRoleAssigned(const tagEntryInfo *const e, int roleIndex)
1988 {
1989         if (roleIndex == ROLE_DEFINITION_INDEX)
1990                 return (!e->extensionFields.roleBits);
1991         else
1992                 return (e->extensionFields.roleBits & makeRoleBit(roleIndex));
1993 }
1994
1995 extern unsigned long numTagsAdded(void)
1996 {
1997         return TagFile.numTags.added;
1998 }
1999
2000 extern void setNumTagsAdded (unsigned long nadded)
2001 {
2002         TagFile.numTags.added = nadded;
2003 }
2004
2005 extern unsigned long numTagsTotal(void)
2006 {
2007         return TagFile.numTags.added + TagFile.numTags.prev;
2008 }
2009
2010 extern unsigned long maxTagsLine (void)
2011 {
2012         return (unsigned long)TagFile.max.line;
2013 }
2014
2015 extern void invalidatePatternCache(void)
2016 {
2017         TagFile.patternCacheValid = false;
2018 }
2019
2020 extern void tagFilePosition (MIOPos *p)
2021 {
2022         /* mini-geany doesn't set TagFile.mio. */
2023         if      (TagFile.mio == NULL)
2024                 return;
2025
2026         if (mio_getpos (TagFile.mio, p) == -1)
2027                 error (FATAL|PERROR,
2028                            "failed to get file position of the tag file\n");
2029 }
2030
2031 extern void setTagFilePosition (MIOPos *p, bool truncation)
2032 {
2033         /* mini-geany doesn't set TagFile.mio. */
2034         if      (TagFile.mio == NULL)
2035                 return;
2036
2037
2038         long t0 = 0;
2039         if (truncation)
2040                 t0 = mio_tell (TagFile.mio);
2041
2042         if (mio_setpos (TagFile.mio, p) == -1)
2043                 error (FATAL|PERROR,
2044                            "failed to set file position of the tag file\n");
2045
2046         if (truncation)
2047         {
2048                 long t1 = mio_tell (TagFile.mio);
2049                 if (!mio_try_resize (TagFile.mio, (size_t)t1))
2050                         error (FATAL|PERROR,
2051                                    "failed to truncate the tag file %ld -> %ld\n", t0, t1);
2052         }
2053 }
2054
2055 extern const char* getTagFileDirectory (void)
2056 {
2057         return TagFile.directory;
2058 }
2059
2060 static bool markAsPlaceholder  (int index, tagEntryInfo *e, void *data CTAGS_ATTR_UNUSED)
2061 {
2062         e->placeholder = 1;
2063         markAllEntriesInScopeAsPlaceholder (index);
2064         return true;
2065 }
2066
2067 extern void markAllEntriesInScopeAsPlaceholder (int index)
2068 {
2069         foreachEntriesInScope (index, NULL, markAsPlaceholder, NULL);
2070 }