parse.c

   1 /*
   2 *   $Id: parse.c 597 2007-07-31 05:35:30Z dhiebert $
   3 *
   4 *   Copyright (c) 1996-2003, Darren Hiebert
   5 *
   6 *   This source code is released for free distribution under the terms of the
   7 *   GNU General Public License.
   8 *
   9 *   This module contains functions for managing source languages and
  10 *   dispatching files to the appropriate language parser.
  11 */
  12
  13 /*
  14 *   INCLUDE FILES
  15 */
  16 #include "general.h"  /* must always come first */
  17
  18 #include <string.h>
  19
  20 #include "debug.h"
  21 #include "entry.h"
  22 #include "main.h"
  23 #define OPTION_WRITE
  24 #include "options.h"
  25 #include "parsers.h"
  26 #include "read.h"
  27 #include "routines.h"
  28 #include "vstring.h"
  29
  30 /*
  31 *   DATA DEFINITIONS
  32 */
  33 static parserDefinitionFunc* BuiltInParsers[] = { PARSER_LIST };
  34 static parserDefinition** LanguageTable = NULL;
  35 static unsigned int LanguageCount = 0;
  36
  37 /*
  38 *   FUNCTION DEFINITIONS
  39 */
  40
  41 extern void makeSimpleTag (
  42                 const vString* const name, kindOption* const kinds, const int kind)
  43 {
  44         if (kinds [kind].enabled  &&  name != NULL  &&  vStringLength (name) > 0)
  45         {
  46             tagEntryInfo e;
  47             initTagEntry (&e, vStringValue (name));
  48
  49             e.kindName = kinds [kind].name;
  50             e.kind     = kinds [kind].letter;
  51
  52             makeTagEntry (&e);
  53         }
  54 }
  55
  56 /*
  57 *   parserDescription mapping management
  58 */
  59
  60 extern parserDefinition* parserNew (const char* name)
  61 {
  62         parserDefinition* result = xCalloc (1, parserDefinition);
  63         result->name = eStrdup (name);
  64         return result;
  65 }
  66
  67 extern const char *getLanguageName (const langType language)
  68 {
  69         const char* result;
  70         if (language == LANG_IGNORE)
  71                 result = "unknown";
  72         else
  73         {
  74                 Assert (0 <= language  &&  language < (int) LanguageCount);
  75                 result = LanguageTable [language]->name;
  76         }
  77         return result;
  78 }
  79
  80 extern langType getNamedLanguage (const char *const name)
  81 {
  82         langType result = LANG_IGNORE;
  83         unsigned int i;
  84         Assert (name != NULL);
  85         for (i = 0  ;  i < LanguageCount  &&  result == LANG_IGNORE  ;  ++i)
  86         {
  87                 const parserDefinition* const lang = LanguageTable [i];
  88                 if (lang->name != NULL)
  89                         if (strcasecmp (name, lang->name) == 0)
  90                                 result = i;
  91         }
  92         return result;
  93 }
  94
  95 static langType getExtensionLanguage (const char *const extension)
  96 {
  97         langType result = LANG_IGNORE;
  98         unsigned int i;
  99         for (i = 0  ;  i < LanguageCount  &&  result == LANG_IGNORE  ;  ++i)
 100         {
 101                 stringList* const exts = LanguageTable [i]->currentExtensions;
 102                 if (exts != NULL  &&  stringListExtensionMatched (exts, extension))
 103                         result = i;
 104         }
 105         return result;
 106 }
 107
 108 static langType getPatternLanguage (const char *const fileName)
 109 {
 110         langType result = LANG_IGNORE;
 111         const char* base = baseFilename (fileName);
 112         unsigned int i;
 113         for (i = 0  ;  i < LanguageCount  &&  result == LANG_IGNORE  ;  ++i)
 114         {
 115                 stringList* const ptrns = LanguageTable [i]->currentPatterns;
 116                 if (ptrns != NULL  &&  stringListFileMatched (ptrns, base))
 117                         result = i;
 118         }
 119         return result;
 120 }
 121
 122 #ifdef SYS_INTERPRETER
 123
 124 /*  The name of the language interpreter, either directly or as the argument
 125  *  to "env".
 126  */
 127 static vString* determineInterpreter (const char* const cmd)
 128 {
 129         vString* const interpreter = vStringNew ();
 130         const char* p = cmd;
 131         do
 132         {
 133                 vStringClear (interpreter);
 134                 for ( ;  isspace ((int) *p)  ;  ++p)
 135                         ;  /* no-op */
 136                 for ( ;  *p != '\0'  &&  ! isspace ((int) *p)  ;  ++p)
 137                         vStringPut (interpreter, (int) *p);
 138                 vStringTerminate (interpreter);
 139         } while (strcmp (vStringValue (interpreter), "env") == 0);
 140         return interpreter;
 141 }
 142
 143 static langType getInterpreterLanguage (const char *const fileName)
 144 {
 145         langType result = LANG_IGNORE;
 146         FILE* const fp = fopen (fileName, "r");
 147         if (fp != NULL)
 148         {
 149                 vString* const vLine = vStringNew ();
 150                 const char* const line = readLine (vLine, fp);
 151                 if (line != NULL  &&  line [0] == '#'  &&  line [1] == '!')
 152                 {
 153                         const char* const lastSlash = strrchr (line, '/');
 154                         const char *const cmd = lastSlash != NULL ? lastSlash+1 : line+2;
 155                         vString* const interpreter = determineInterpreter (cmd);
 156                         result = getExtensionLanguage (vStringValue (interpreter));
 157                         if (result == LANG_IGNORE)
 158                                 result = getNamedLanguage (vStringValue (interpreter));
 159                         vStringDelete (interpreter);
 160                 }
 161                 vStringDelete (vLine);
 162                 fclose (fp);
 163         }
 164         return result;
 165 }
 166
 167 #endif
 168
 169 extern langType getFileLanguage (const char *const fileName)
 170 {
 171         langType language = Option.language;
 172         if (language == LANG_AUTO)
 173         {
 174                 language = getExtensionLanguage (fileExtension (fileName));
 175                 if (language == LANG_IGNORE)
 176                         language = getPatternLanguage (fileName);
 177 #ifdef SYS_INTERPRETER
 178                 if (language == LANG_IGNORE)
 179                 {
 180                         fileStatus *status = eStat (fileName);
 181                         if (status->isExecutable)
 182                                 language = getInterpreterLanguage (fileName);
 183                 }
 184 #endif
 185         }
 186         return language;
 187 }
 188
 189 extern void printLanguageMap (const langType language)
 190 {
 191         boolean first = TRUE;
 192         unsigned int i;
 193         stringList* map = LanguageTable [language]->currentPatterns;
 194         Assert (0 <= language  &&  language < (int) LanguageCount);
 195         for (i = 0  ;  map != NULL  &&  i < stringListCount (map)  ;  ++i)
 196         {
 197                 printf ("%s(%s)", (first ? "" : " "),
 198                                 vStringValue (stringListItem (map, i)));
 199                 first = FALSE;
 200         }
 201         map = LanguageTable [language]->currentExtensions;
 202         for (i = 0  ;  map != NULL  &&  i < stringListCount (map)  ;  ++i)
 203         {
 204                 printf ("%s.%s", (first ? "" : " "),
 205                                 vStringValue (stringListItem (map, i)));
 206                 first = FALSE;
 207         }
 208 }
 209
 210 extern void installLanguageMapDefault (const langType language)
 211 {
 212         parserDefinition* lang;
 213         Assert (0 <= language  &&  language < (int) LanguageCount);
 214         lang = LanguageTable [language];
 215         if (lang->currentPatterns != NULL)
 216                 stringListDelete (lang->currentPatterns);
 217         if (lang->currentExtensions != NULL)
 218                 stringListDelete (lang->currentExtensions);
 219
 220         if (lang->patterns == NULL)
 221                 lang->currentPatterns = stringListNew ();
 222         else
 223         {
 224                 lang->currentPatterns =
 225                         stringListNewFromArgv (lang->patterns);
 226         }
 227         if (lang->extensions == NULL)
 228                 lang->currentExtensions = stringListNew ();
 229         else
 230         {
 231                 lang->currentExtensions =
 232                         stringListNewFromArgv (lang->extensions);
 233         }
 234         if (Option.verbose)
 235                 printLanguageMap (language);
 236         verbose ("\n");
 237 }
 238
 239 extern void installLanguageMapDefaults (void)
 240 {
 241         unsigned int i;
 242         for (i = 0  ;  i < LanguageCount  ;  ++i)
 243         {
 244                 verbose ("    %s: ", getLanguageName (i));
 245                 installLanguageMapDefault (i);
 246         }
 247 }
 248
 249 extern void clearLanguageMap (const langType language)
 250 {
 251         Assert (0 <= language  &&  language < (int) LanguageCount);
 252         stringListClear (LanguageTable [language]->currentPatterns);
 253         stringListClear (LanguageTable [language]->currentExtensions);
 254 }
 255
 256 extern void addLanguagePatternMap (const langType language, const char* ptrn)
 257 {
 258         vString* const str = vStringNewInit (ptrn);
 259         parserDefinition* lang;
 260         Assert (0 <= language  &&  language < (int) LanguageCount);
 261         lang = LanguageTable [language];
 262         if (lang->currentPatterns == NULL)
 263                 lang->currentPatterns = stringListNew ();
 264         stringListAdd (lang->currentPatterns, str);
 265 }
 266
 267 extern boolean removeLanguageExtensionMap (const char *const extension)
 268 {
 269         boolean result = FALSE;
 270         unsigned int i;
 271         for (i = 0  ;  i < LanguageCount  &&  ! result ;  ++i)
 272         {
 273                 stringList* const exts = LanguageTable [i]->currentExtensions;
 274                 if (exts != NULL  &&  stringListRemoveExtension (exts, extension))
 275                 {
 276                         verbose (" (removed from %s)", getLanguageName (i));
 277                         result = TRUE;
 278                 }
 279         }
 280         return result;
 281 }
 282
 283 extern void addLanguageExtensionMap (
 284                 const langType language, const char* extension)
 285 {
 286         vString* const str = vStringNewInit (extension);
 287         Assert (0 <= language  &&  language < (int) LanguageCount);
 288         removeLanguageExtensionMap (extension);
 289         stringListAdd (LanguageTable [language]->currentExtensions, str);
 290 }
 291
 292 extern void enableLanguage (const langType language, const boolean state)
 293 {
 294         Assert (0 <= language  &&  language < (int) LanguageCount);
 295         LanguageTable [language]->enabled = state;
 296 }
 297
 298 extern void enableLanguages (const boolean state)
 299 {
 300         unsigned int i;
 301         for (i = 0  ;  i < LanguageCount  ;  ++i)
 302                 enableLanguage (i, state);
 303 }
 304
 305 static void initializeParsers (void)
 306 {
 307         unsigned int i;
 308         for (i = 0  ;  i < LanguageCount  ;  ++i)
 309                 if (LanguageTable [i]->initialize != NULL)
 310                         (LanguageTable [i]->initialize) ((langType) i);
 311 }
 312
 313 extern void initializeParsing (void)
 314 {
 315         unsigned int builtInCount;
 316         unsigned int i;
 317
 318         builtInCount = sizeof (BuiltInParsers) / sizeof (BuiltInParsers [0]);
 319         LanguageTable = xMalloc (builtInCount, parserDefinition*);
 320
 321         verbose ("Installing parsers: ");
 322         for (i = 0  ;  i < builtInCount  ;  ++i)
 323         {
 324                 parserDefinition* const def = (*BuiltInParsers [i]) ();
 325                 if (def != NULL)
 326                 {
 327                         boolean accepted = FALSE;
 328                         if (def->name == NULL  ||  def->name[0] == '\0')
 329                                 error (FATAL, "parser definition must contain name\n");
 330                         else if (def->regex)
 331                         {
 332 #ifdef HAVE_REGEX
 333                                 def->parser = findRegexTags;
 334                                 accepted = TRUE;
 335 #endif
 336                         }
 337                         else if ((def->parser == NULL)  ==  (def->parser2 == NULL))
 338                                 error (FATAL,
 339                 "%s parser definition must define one and only one parsing routine\n",
 340                                            def->name);
 341                         else
 342                                 accepted = TRUE;
 343                         if (accepted)
 344                         {
 345                                 verbose ("%s%s", i > 0 ? ", " : "", def->name);
 346                                 def->id = LanguageCount++;
 347                                 LanguageTable [def->id] = def;
 348                         }
 349                 }
 350         }
 351         verbose ("\n");
 352         enableLanguages (TRUE);
 353         initializeParsers ();
 354 }
 355
 356 extern void freeParserResources (void)
 357 {
 358         unsigned int i;
 359         for (i = 0  ;  i < LanguageCount  ;  ++i)
 360         {
 361                 parserDefinition* const lang = LanguageTable [i];
 362                 freeList (&lang->currentPatterns);
 363                 freeList (&lang->currentExtensions);
 364                 eFree (lang->name);
 365                 lang->name = NULL;
 366                 eFree (lang);
 367         }
 368         if (LanguageTable != NULL)
 369                 eFree (LanguageTable);
 370         LanguageTable = NULL;
 371         LanguageCount = 0;
 372 }
 373
 374 /*
 375 *   Option parsing
 376 */
 377
 378 extern void processLanguageDefineOption (
 379                 const char *const option, const char *const parameter __unused__)
 380 {
 381 #ifdef HAVE_REGEX
 382         if (parameter [0] == '\0')
 383                 error (WARNING, "No language specified for \"%s\" option", option);
 384         else if (getNamedLanguage (parameter) != LANG_IGNORE)
 385                 error (WARNING, "Language \"%s\" already defined", parameter);
 386         else
 387         {
 388                 unsigned int i = LanguageCount++;
 389                 parserDefinition* const def = parserNew (parameter);
 390                 def->parser            = findRegexTags;
 391                 def->currentPatterns   = stringListNew ();
 392                 def->currentExtensions = stringListNew ();
 393                 def->regex             = TRUE;
 394                 def->enabled           = TRUE;
 395                 def->id                = i;
 396                 LanguageTable = xRealloc (LanguageTable, i + 1, parserDefinition*);
 397                 LanguageTable [i] = def;
 398         }
 399 #else
 400         error (WARNING, "regex support not available; required for --%s option",
 401                    option);
 402 #endif
 403 }
 404
 405 static kindOption *langKindOption (const langType language, const int flag)
 406 {
 407         unsigned int i;
 408         kindOption* result = NULL;
 409         const parserDefinition* lang;
 410         Assert (0 <= language  &&  language < (int) LanguageCount);
 411         lang = LanguageTable [language];
 412         for (i=0  ;  i < lang->kindCount  &&  result == NULL  ;  ++i)
 413                 if (lang->kinds [i].letter == flag)
 414                         result = &lang->kinds [i];
 415         return result;
 416 }
 417
 418 static void disableLanguageKinds (const langType language)
 419 {
 420         const parserDefinition* lang;
 421         Assert (0 <= language  &&  language < (int) LanguageCount);
 422         lang = LanguageTable [language];
 423         if (lang->regex)
 424                 disableRegexKinds (language);
 425         else
 426         {
 427                 unsigned int i;
 428                 for (i = 0  ;  i < lang->kindCount  ;  ++i)
 429                         lang->kinds [i].enabled = FALSE;
 430         }
 431 }
 432
 433 static boolean enableLanguageKind (
 434                 const langType language, const int kind, const boolean mode)
 435 {
 436         boolean result = FALSE;
 437         if (LanguageTable [language]->regex)
 438                 result = enableRegexKind (language, kind, mode);
 439         else
 440         {
 441                 kindOption* const opt = langKindOption (language, kind);
 442                 if (opt != NULL)
 443                 {
 444                         opt->enabled = mode;
 445                         result = TRUE;
 446                 }
 447         }
 448         return result;
 449 }
 450
 451 static void processLangKindOption (
 452                 const langType language, const char *const option,
 453                 const char *const parameter)
 454 {
 455         const char *p = parameter;
 456         boolean mode = TRUE;
 457         int c;
 458
 459         Assert (0 <= language  &&  language < (int) LanguageCount);
 460         if (*p != '+'  &&  *p != '-')
 461                 disableLanguageKinds (language);
 462         while ((c = *p++) != '\0') switch (c)
 463         {
 464                 case '+': mode = TRUE;  break;
 465                 case '-': mode = FALSE; break;
 466                 default:
 467                         if (! enableLanguageKind (language, c, mode))
 468                                 error (WARNING, "Unsupported parameter '%c' for --%s option",
 469                                         c, option);
 470                         break;
 471         }
 472 }
 473
 474 extern boolean processKindOption (
 475                 const char *const option, const char *const parameter)
 476 {
 477         boolean handled = FALSE;
 478         const char* const dash = strchr (option, '-');
 479         if (dash != NULL  &&
 480                 (strcmp (dash + 1, "kinds") == 0  ||  strcmp (dash + 1, "types") == 0))
 481         {
 482                 langType language;
 483                 vString* langName = vStringNew ();
 484                 vStringNCopyS (langName, option, dash - option);
 485                 language = getNamedLanguage (vStringValue (langName));
 486                 if (language == LANG_IGNORE)
 487                         error (WARNING, "Unknown language \"%s\" in \"%s\" option", vStringValue (langName), option);
 488                 else
 489                         processLangKindOption (language, option, parameter);
 490                 vStringDelete (langName);
 491                 handled = TRUE;
 492         }
 493         return handled;
 494 }
 495
 496 static void printLanguageKind (const kindOption* const kind, boolean indent)
 497 {
 498         const char *const indentation = indent ? "    " : "";
 499         printf ("%s%c  %s%s\n", indentation, kind->letter,
 500                 kind->description != NULL ? kind->description :
 501                         (kind->name != NULL ? kind->name : ""),
 502                 kind->enabled ? "" : " [off]");
 503 }
 504
 505 static void printKinds (langType language, boolean indent)
 506 {
 507         const parserDefinition* lang;
 508         Assert (0 <= language  &&  language < (int) LanguageCount);
 509         lang = LanguageTable [language];
 510         if (lang->kinds != NULL  ||  lang->regex)
 511         {
 512                 unsigned int i;
 513                 for (i = 0  ;  i < lang->kindCount  ;  ++i)
 514                         printLanguageKind (lang->kinds + i, indent);
 515                 printRegexKinds (language, indent);
 516         }
 517 }
 518
 519 extern void printLanguageKinds (const langType language)
 520 {
 521         if (language == LANG_AUTO)
 522         {
 523                 unsigned int i;
 524                 for (i = 0  ;  i < LanguageCount  ;  ++i)
 525                 {
 526                         const parserDefinition* const lang = LanguageTable [i];
 527                         printf ("%s%s\n", lang->name, lang->enabled ? "" : " [disabled]");
 528                         printKinds (i, TRUE);
 529                 }
 530         }
 531         else
 532                 printKinds (language, FALSE);
 533 }
 534
 535 static void printMaps (const langType language)
 536 {
 537         const parserDefinition* lang;
 538         unsigned int i;
 539         Assert (0 <= language  &&  language < (int) LanguageCount);
 540         lang = LanguageTable [language];
 541         printf ("%-8s", lang->name);
 542         if (lang->currentExtensions != NULL)
 543                 for (i = 0  ;  i < stringListCount (lang->currentExtensions)  ;  ++i)
 544                         printf (" *.%s", vStringValue (
 545                                                 stringListItem (lang->currentExtensions, i)));
 546         if (lang->currentPatterns != NULL)
 547                 for (i = 0  ;  i < stringListCount (lang->currentPatterns)  ;  ++i)
 548                         printf (" %s", vStringValue (
 549                                                 stringListItem (lang->currentPatterns, i)));
 550         putchar ('\n');
 551 }
 552
 553 extern void printLanguageMaps (const langType language)
 554 {
 555         if (language == LANG_AUTO)
 556         {
 557                 unsigned int i;
 558                 for (i = 0  ;  i < LanguageCount  ;  ++i)
 559                         printMaps (i);
 560         }
 561         else
 562                 printMaps (language);
 563 }
 564
 565 static void printLanguage (const langType language)
 566 {
 567         const parserDefinition* lang;
 568         Assert (0 <= language  &&  language < (int) LanguageCount);
 569         lang = LanguageTable [language];
 570         if (lang->kinds != NULL  ||  lang->regex)
 571                 printf ("%s%s\n", lang->name, lang->enabled ? "" : " [disabled]");
 572 }
 573
 574 extern void printLanguageList (void)
 575 {
 576         unsigned int i;
 577         for (i = 0  ;  i < LanguageCount  ;  ++i)
 578                 printLanguage (i);
 579 }
 580
 581 /*
 582 *   File parsing
 583 */
 584
 585 static void makeFileTag (const char *const fileName)
 586 {
 587         if (Option.include.fileNames)
 588         {
 589                 tagEntryInfo tag;
 590                 initTagEntry (&tag, baseFilename (fileName));
 591
 592                 tag.isFileEntry     = TRUE;
 593                 tag.lineNumberEntry = TRUE;
 594                 tag.lineNumber      = 1;
 595                 tag.kindName        = "file";
 596                 tag.kind            = 'F';
 597
 598                 makeTagEntry (&tag);
 599         }
 600 }
 601
 602 static boolean createTagsForFile (
 603                 const char *const fileName, const langType language,
 604                 const unsigned int passCount)
 605 {
 606         boolean retried = FALSE;
 607         Assert (0 <= language  &&  language < (int) LanguageCount);
 608         if (fileOpen (fileName, language))
 609         {
 610                 const parserDefinition* const lang = LanguageTable [language];
 611                 if (Option.etags)
 612                         beginEtagsFile ();
 613
 614                 makeFileTag (fileName);
 615
 616                 if (lang->parser != NULL)
 617                         lang->parser ();
 618                 else if (lang->parser2 != NULL)
 619                         retried = lang->parser2 (passCount);
 620
 621                 if (Option.etags)
 622                         endEtagsFile (getSourceFileTagPath ());
 623
 624                 fileClose ();
 625         }
 626
 627         return retried;
 628 }
 629
 630 static boolean createTagsWithFallback (
 631                 const char *const fileName, const langType language)
 632 {
 633         const unsigned long numTags     = TagFile.numTags.added;
 634         fpos_t tagFilePosition;
 635         unsigned int passCount = 0;
 636         boolean tagFileResized = FALSE;
 637
 638         fgetpos (TagFile.fp, &tagFilePosition);
 639         while (createTagsForFile (fileName, language, ++passCount))
 640         {
 641                 /*  Restore prior state of tag file.
 642                  */
 643                 fsetpos (TagFile.fp, &tagFilePosition);
 644                 TagFile.numTags.added = numTags;
 645                 tagFileResized = TRUE;
 646         }
 647         return tagFileResized;
 648 }
 649
 650 extern boolean parseFile (const char *const fileName)
 651 {
 652         boolean tagFileResized = FALSE;
 653         langType language = Option.language;
 654         if (Option.language == LANG_AUTO)
 655                 language = getFileLanguage (fileName);
 656         Assert (language != LANG_AUTO);
 657         if (language == LANG_IGNORE)
 658                 verbose ("ignoring %s (unknown language)\n", fileName);
 659         else if (! LanguageTable [language]->enabled)
 660                 verbose ("ignoring %s (language disabled)\n", fileName);
 661         else
 662         {
 663                 if (Option.filter)
 664                         openTagFile ();
 665
 666                 tagFileResized = createTagsWithFallback (fileName, language);
 667
 668                 if (Option.filter)
 669                         closeTagFile (tagFileResized);
 670                 addTotals (1, 0L, 0L);
 671
 672                 return tagFileResized;
 673         }
 674         return tagFileResized;
 675 }
 676
 677 /* vi:set tabstop=4 shiftwidth=4 nowrap: */