tagmanager/parse.c

   1 /*
   2 *   $Id$
   3 *
   4 *   Copyright (c) 1996-2003, Darren Hiebert
   5 *
   6 *   This source code is released for free distribution under the terms of the
   7 *   GNU General Public License.
   8 *
   9 *   This module contains functions for managing source languages and
  10 *   dispatching files to the appropriate language parser.
  11 */
  12
  13 /*
  14 *   INCLUDE FILES
  15 */
  16 #include "general.h"  /* must always come first */
  17
  18 #include <string.h>
  19 #ifdef HAVE_FNMATCH_H
  20 #include <fnmatch.h>
  21 #endif
  22
  23 #include "debug.h"
  24 #include "entry.h"
  25 #include "main.h"
  26 #define OPTION_WRITE
  27 #include "options.h"
  28 #include "parsers.h"
  29 #include "read.h"
  30 #include "routines.h"
  31 #include "vstring.h"
  32
  33 /*
  34 *   DATA DEFINITIONS
  35 */
  36 static parserDefinitionFunc* BuiltInParsers[] = { PARSER_LIST };
  37 parserDefinition** LanguageTable = NULL;
  38 static unsigned int LanguageCount = 0;
  39 tagEntryFunction TagEntryFunction = NULL;
  40
  41 /*
  42 *   FUNCTION DEFINITIONS
  43 */
  44
  45 extern void makeSimpleTag (
  46                 const vString* const name, kindOption* const kinds, const int kind)
  47 {
  48         if (kinds [kind].enabled  &&  name != NULL  &&  vStringLength (name) > 0)
  49         {
  50             tagEntryInfo e;
  51             initTagEntry (&e, vStringValue (name));
  52
  53             e.kindName = kinds [kind].name;
  54             e.kind     = kinds [kind].letter;
  55
  56             makeTagEntry (&e);
  57         }
  58 }
  59
  60 /*
  61 *   parserDescription mapping management
  62 */
  63
  64 extern parserDefinition* parserNew (const char* name)
  65 {
  66         parserDefinition* result = xCalloc (1, parserDefinition);
  67         result->name = eStrdup (name);
  68         return result;
  69 }
  70
  71 extern const char *getLanguageName (const langType language)
  72 {
  73         const char* result;
  74         if (language == LANG_IGNORE)
  75                 result = "unknown";
  76         else
  77         {
  78                 Assert (0 <= language  &&  language < (int) LanguageCount);
  79                 result = LanguageTable [language]->name;
  80         }
  81         return result;
  82 }
  83
  84 extern langType getNamedLanguage (const char *const name)
  85 {
  86         langType result = LANG_IGNORE;
  87         unsigned int i;
  88         Assert (name != NULL);
  89         for (i = 0  ;  i < LanguageCount  &&  result == LANG_IGNORE  ;  ++i)
  90         {
  91                 const parserDefinition* const lang = LanguageTable [i];
  92                 if (lang->name != NULL)
  93                         if (strcasecmp (name, lang->name) == 0)
  94                                 result = i;
  95         }
  96         return result;
  97 }
  98
  99 static langType getExtensionLanguage (const char *const extension)
 100 {
 101         langType result = LANG_IGNORE;
 102         unsigned int i;
 103         for (i = 0  ;  i < LanguageCount  &&  result == LANG_IGNORE  ;  ++i)
 104         {
 105                 stringList* const exts = LanguageTable [i]->currentExtensions;
 106                 if (exts != NULL  &&  stringListExtensionMatched (exts, extension))
 107                         result = i;
 108         }
 109         return result;
 110 }
 111
 112 static langType getPatternLanguage (const char *const fileName)
 113 {
 114         langType result = LANG_IGNORE;
 115         const char* base = baseFilename (fileName);
 116         unsigned int i;
 117         for (i = 0  ;  i < LanguageCount  &&  result == LANG_IGNORE  ;  ++i)
 118         {
 119                 stringList* const ptrns = LanguageTable [i]->currentPatterns;
 120                 if (ptrns != NULL  &&  stringListFileMatched (ptrns, base))
 121                         result = i;
 122         }
 123         return result;
 124 }
 125
 126 #ifdef SYS_INTERPRETER
 127
 128 /*  The name of the language interpreter, either directly or as the argument
 129  *  to "env".
 130  */
 131 static vString* determineInterpreter (const char* const cmd)
 132 {
 133         vString* const interpreter = vStringNew ();
 134         const char* p = cmd;
 135         do
 136         {
 137                 vStringClear (interpreter);
 138                 for ( ;  isspace ((int) *p)  ;  ++p)
 139                         ;  /* no-op */
 140                 for ( ;  *p != '\0'  &&  ! isspace ((int) *p)  ;  ++p)
 141                         vStringPut (interpreter, (int) *p);
 142                 vStringTerminate (interpreter);
 143         } while (strcmp (vStringValue (interpreter), "env") == 0);
 144         return interpreter;
 145 }
 146
 147 static langType getInterpreterLanguage (const char *const fileName)
 148 {
 149         langType result = LANG_IGNORE;
 150         FILE* const fp = fopen (fileName, "r");
 151         if (fp != NULL)
 152         {
 153                 vString* const vLine = vStringNew ();
 154                 const char* const line = readLine (vLine, fp);
 155                 if (line != NULL  &&  line [0] == '#'  &&  line [1] == '!')
 156                 {
 157                         const char* const lastSlash = strrchr (line, '/');
 158                         const char *const cmd = lastSlash != NULL ? lastSlash+1 : line+2;
 159                         vString* const interpreter = determineInterpreter (cmd);
 160                         result = getExtensionLanguage (vStringValue (interpreter));
 161                         vStringDelete (interpreter);
 162                 }
 163                 vStringDelete (vLine);
 164                 fclose (fp);
 165         }
 166         return result;
 167 }
 168
 169 #endif
 170
 171 extern langType getFileLanguage (const char *const fileName)
 172 {
 173         langType language = Option.language;
 174         if (language == LANG_AUTO)
 175         {
 176                 language = getExtensionLanguage (fileExtension (fileName));
 177                 if (language == LANG_IGNORE)
 178                         language = getPatternLanguage (fileName);
 179 #ifdef SYS_INTERPRETER
 180                 if (language == LANG_IGNORE)
 181                 {
 182                         fileStatus *status = eStat (fileName);
 183                         if (status->isExecutable)
 184                                 language = getInterpreterLanguage (fileName);
 185                 }
 186 #endif
 187         }
 188         return language;
 189 }
 190
 191 extern void printLanguageMap (const langType language)
 192 {
 193         boolean first = TRUE;
 194         unsigned int i;
 195         stringList* map = LanguageTable [language]->currentPatterns;
 196         Assert (0 <= language  &&  language < (int) LanguageCount);
 197         for (i = 0  ;  map != NULL  &&  i < stringListCount (map)  ;  ++i)
 198         {
 199                 printf ("%s(%s)", (first ? "" : " "),
 200                                 vStringValue (stringListItem (map, i)));
 201                 first = FALSE;
 202         }
 203         map = LanguageTable [language]->currentExtensions;
 204         for (i = 0  ;  map != NULL  &&  i < stringListCount (map)  ;  ++i)
 205         {
 206                 printf ("%s.%s", (first ? "" : " "),
 207                                 vStringValue (stringListItem (map, i)));
 208                 first = FALSE;
 209         }
 210 }
 211
 212 extern void installLanguageMapDefault (const langType language)
 213 {
 214         parserDefinition* lang;
 215         Assert (0 <= language  &&  language < (int) LanguageCount);
 216         lang = LanguageTable [language];
 217         if (lang->currentPatterns != NULL)
 218                 stringListDelete (lang->currentPatterns);
 219         if (lang->currentExtensions != NULL)
 220                 stringListDelete (lang->currentExtensions);
 221
 222         if (lang->patterns == NULL)
 223                 lang->currentPatterns = stringListNew ();
 224         else
 225         {
 226                 lang->currentPatterns =
 227                         stringListNewFromArgv (lang->patterns);
 228         }
 229         if (lang->extensions == NULL)
 230                 lang->currentExtensions = stringListNew ();
 231         else
 232         {
 233                 lang->currentExtensions =
 234                         stringListNewFromArgv (lang->extensions);
 235         }
 236         if (Option.verbose)
 237                 printLanguageMap (language);
 238         verbose ("\n");
 239 }
 240
 241 extern void installLanguageMapDefaults (void)
 242 {
 243         unsigned int i;
 244         for (i = 0  ;  i < LanguageCount  ;  ++i)
 245         {
 246                 verbose ("    %s: ", getLanguageName (i));
 247                 installLanguageMapDefault (i);
 248         }
 249 }
 250
 251 extern void clearLanguageMap (const langType language)
 252 {
 253         Assert (0 <= language  &&  language < (int) LanguageCount);
 254         stringListClear (LanguageTable [language]->currentPatterns);
 255         stringListClear (LanguageTable [language]->currentExtensions);
 256 }
 257
 258 extern void addLanguagePatternMap (const langType language, const char* ptrn)
 259 {
 260         vString* const str = vStringNewInit (ptrn);
 261         parserDefinition* lang;
 262         Assert (0 <= language  &&  language < (int) LanguageCount);
 263         lang = LanguageTable [language];
 264         if (lang->currentPatterns == NULL)
 265                 lang->currentPatterns = stringListNew ();
 266         stringListAdd (lang->currentPatterns, str);
 267 }
 268
 269 extern boolean removeLanguageExtensionMap (const char *const extension)
 270 {
 271         boolean result = FALSE;
 272         unsigned int i;
 273         for (i = 0  ;  i < LanguageCount  &&  ! result ;  ++i)
 274         {
 275                 stringList* const exts = LanguageTable [i]->currentExtensions;
 276                 if (exts != NULL  &&  stringListRemoveExtension (exts, extension))
 277                 {
 278                         verbose (" (removed from %s)", getLanguageName (i));
 279                         result = TRUE;
 280                 }
 281         }
 282         return result;
 283 }
 284
 285 extern void addLanguageExtensionMap (
 286                 const langType language, const char* extension)
 287 {
 288         vString* const str = vStringNewInit (extension);
 289         Assert (0 <= language  &&  language < (int) LanguageCount);
 290         removeLanguageExtensionMap (extension);
 291         stringListAdd (LanguageTable [language]->currentExtensions, str);
 292 }
 293
 294 extern void enableLanguage (const langType language, const boolean state)
 295 {
 296         Assert (0 <= language  &&  language < (int) LanguageCount);
 297         LanguageTable [language]->enabled = state;
 298 }
 299
 300 extern void enableLanguages (const boolean state)
 301 {
 302         unsigned int i;
 303         for (i = 0  ;  i < LanguageCount  ;  ++i)
 304                 enableLanguage (i, state);
 305 }
 306
 307 static void initializeParsers (void)
 308 {
 309         unsigned int i;
 310         for (i = 0  ;  i < LanguageCount  ;  ++i)
 311                 if (LanguageTable [i]->initialize != NULL)
 312                         (LanguageTable [i]->initialize) ((langType) i);
 313 }
 314
 315 extern void initializeParsing (void)
 316 {
 317         unsigned int builtInCount;
 318         unsigned int i;
 319
 320         builtInCount = sizeof (BuiltInParsers) / sizeof (BuiltInParsers [0]);
 321         LanguageTable = xMalloc (builtInCount, parserDefinition*);
 322
 323         verbose ("Installing parsers: ");
 324         for (i = 0  ;  i < builtInCount  ;  ++i)
 325         {
 326                 parserDefinition* const def = (*BuiltInParsers [i]) ();
 327                 if (def != NULL)
 328                 {
 329                         boolean accepted = FALSE;
 330                         if (def->name == NULL  ||  def->name[0] == '\0')
 331                                 error (FATAL, "parser definition must contain name\n");
 332                         else if (def->regex)
 333                         {
 334 #ifdef HAVE_REGEX
 335                                 def->parser = findRegexTags;
 336                                 accepted = TRUE;
 337 #endif
 338                         }
 339                         else if ((def->parser == NULL)  ==  (def->parser2 == NULL))
 340                                 error (FATAL,
 341                 "%s parser definition must define one and only one parsing routine\n",
 342                                            def->name);
 343                         else
 344                                 accepted = TRUE;
 345                         if (accepted)
 346                         {
 347                                 verbose ("%s%s", i > 0 ? ", " : "", def->name);
 348                                 def->id = LanguageCount++;
 349                                 LanguageTable [def->id] = def;
 350                         }
 351                 }
 352         }
 353         verbose ("\n");
 354         enableLanguages (TRUE);
 355         initializeParsers ();
 356 }
 357
 358 extern void freeParserResources (void)
 359 {
 360         unsigned int i;
 361         for (i = 0  ;  i < LanguageCount  ;  ++i)
 362         {
 363                 parserDefinition* const lang = LanguageTable [i];
 364                 freeList (&lang->currentPatterns);
 365                 freeList (&lang->currentExtensions);
 366                 eFree (lang->name);
 367                 lang->name = NULL;
 368                 eFree (lang);
 369         }
 370         if (LanguageTable != NULL)
 371                 eFree (LanguageTable);
 372         LanguageTable = NULL;
 373         LanguageCount = 0;
 374 }
 375
 376 /*
 377 *   Option parsing
 378 */
 379
 380 extern void processLanguageDefineOption (
 381                 const char *const option, const char *const parameter __unused__)
 382 {
 383 #ifdef HAVE_REGEX
 384         if (parameter [0] == '\0')
 385                 error (WARNING, "No language specified for \"%s\" option", option);
 386         else if (getNamedLanguage (parameter) != LANG_IGNORE)
 387                 error (WARNING, "Language \"%s\" already defined", parameter);
 388         else
 389         {
 390                 unsigned int i = LanguageCount++;
 391                 parserDefinition* const def = parserNew (parameter);
 392                 def->parser            = findRegexTags;
 393                 def->currentPatterns   = stringListNew ();
 394                 def->currentExtensions = stringListNew ();
 395                 def->regex             = TRUE;
 396                 def->enabled           = TRUE;
 397                 def->id                = i;
 398                 LanguageTable = xRealloc (LanguageTable, i + 1, parserDefinition*);
 399                 LanguageTable [i] = def;
 400         }
 401 #else
 402         error (WARNING, "regex support not available; required for --%s option",
 403                    option);
 404 #endif
 405 }
 406
 407 static kindOption *langKindOption (const langType language, const int flag)
 408 {
 409         unsigned int i;
 410         kindOption* result = NULL;
 411         const parserDefinition* lang;
 412         Assert (0 <= language  &&  language < (int) LanguageCount);
 413         lang = LanguageTable [language];
 414         for (i=0  ;  i < lang->kindCount  &&  result == NULL  ;  ++i)
 415                 if (lang->kinds [i].letter == flag)
 416                         result = &lang->kinds [i];
 417         return result;
 418 }
 419
 420 static void disableLanguageKinds (const langType language)
 421 {
 422         const parserDefinition* lang;
 423         Assert (0 <= language  &&  language < (int) LanguageCount);
 424         lang = LanguageTable [language];
 425         if (lang->regex)
 426                 disableRegexKinds (language);
 427         else
 428         {
 429                 unsigned int i;
 430                 for (i = 0  ;  i < lang->kindCount  ;  ++i)
 431                         lang->kinds [i].enabled = FALSE;
 432         }
 433 }
 434
 435 static boolean enableLanguageKind (
 436                 const langType language, const int kind, const boolean mode)
 437 {
 438         boolean result = FALSE;
 439         if (LanguageTable [language]->regex)
 440                 result = enableRegexKind (language, kind, mode);
 441         else
 442         {
 443                 kindOption* const opt = langKindOption (language, kind);
 444                 if (opt != NULL)
 445                 {
 446                         opt->enabled = mode;
 447                         result = TRUE;
 448                 }
 449         }
 450         return result;
 451 }
 452
 453 static void processLangKindOption (
 454                 const langType language, const char *const option,
 455                 const char *const parameter)
 456 {
 457         const char *p = parameter;
 458         boolean mode = TRUE;
 459         int c;
 460
 461         Assert (0 <= language  &&  language < (int) LanguageCount);
 462         if (*p != '+'  &&  *p != '-')
 463                 disableLanguageKinds (language);
 464         while ((c = *p++) != '\0') switch (c)
 465         {
 466                 case '+': mode = TRUE;  break;
 467                 case '-': mode = FALSE; break;
 468                 default:
 469                         if (! enableLanguageKind (language, c, mode))
 470                                 error (WARNING, "Unsupported parameter '%c' for --%s option",
 471                                         c, option);
 472                         break;
 473         }
 474 }
 475
 476 extern boolean processKindOption (
 477                 const char *const option, const char *const parameter)
 478 {
 479         boolean handled = FALSE;
 480         const char* const dash = strchr (option, '-');
 481         if (dash != NULL  &&
 482                 (strcmp (dash + 1, "kinds") == 0  ||  strcmp (dash + 1, "types") == 0))
 483         {
 484                 langType language;
 485                 vString* langName = vStringNew ();
 486                 vStringNCopyS (langName, option, dash - option);
 487                 language = getNamedLanguage (vStringValue (langName));
 488                 if (language == LANG_IGNORE)
 489                         error (WARNING, "Unknown language specified in \"%s\" option", option);
 490                 else
 491                         processLangKindOption (language, option, parameter);
 492                 vStringDelete (langName);
 493                 handled = TRUE;
 494         }
 495         return handled;
 496 }
 497
 498 static void printLanguageKind (const kindOption* const kind, boolean indent)
 499 {
 500         const char *const indentation = indent ? "    " : "";
 501         printf ("%s%c  %s%s\n", indentation, kind->letter,
 502                 kind->description != NULL ? kind->description :
 503                         (kind->name != NULL ? kind->name : ""),
 504                 kind->enabled ? "" : " [off]");
 505 }
 506
 507 static void printKinds (langType language, boolean indent)
 508 {
 509         const parserDefinition* lang;
 510         Assert (0 <= language  &&  language < (int) LanguageCount);
 511         lang = LanguageTable [language];
 512         if (lang->kinds != NULL  ||  lang->regex)
 513         {
 514                 unsigned int i;
 515                 for (i = 0  ;  i < lang->kindCount  ;  ++i)
 516                         printLanguageKind (lang->kinds + i, indent);
 517                 printRegexKinds (language, indent);
 518         }
 519 }
 520
 521 extern void printLanguageKinds (const langType language)
 522 {
 523         if (language == LANG_AUTO)
 524         {
 525                 unsigned int i;
 526                 for (i = 0  ;  i < LanguageCount  ;  ++i)
 527                 {
 528                         const parserDefinition* const lang = LanguageTable [i];
 529                         printf ("%s%s\n", lang->name, lang->enabled ? "" : " [disabled]");
 530                         printKinds (i, TRUE);
 531                 }
 532         }
 533         else
 534                 printKinds (language, FALSE);
 535 }
 536
 537 static void printMaps (const langType language)
 538 {
 539         const parserDefinition* lang;
 540         unsigned int i;
 541         Assert (0 <= language  &&  language < (int) LanguageCount);
 542         lang = LanguageTable [language];
 543         printf ("%-8s", lang->name);
 544         if (lang->currentExtensions != NULL)
 545                 for (i = 0  ;  i < stringListCount (lang->currentExtensions)  ;  ++i)
 546                         printf (" *.%s", vStringValue (
 547                                                 stringListItem (lang->currentExtensions, i)));
 548         if (lang->currentPatterns != NULL)
 549                 for (i = 0  ;  i < stringListCount (lang->currentPatterns)  ;  ++i)
 550                         printf (" %s", vStringValue (
 551                                                 stringListItem (lang->currentPatterns, i)));
 552         putchar ('\n');
 553 }
 554
 555 extern void printLanguageMaps (const langType language)
 556 {
 557         if (language == LANG_AUTO)
 558         {
 559                 unsigned int i;
 560                 for (i = 0  ;  i < LanguageCount  ;  ++i)
 561                         printMaps (i);
 562         }
 563         else
 564                 printMaps (language);
 565 }
 566
 567 static void printLanguage (const langType language)
 568 {
 569         const parserDefinition* lang;
 570         Assert (0 <= language  &&  language < (int) LanguageCount);
 571         lang = LanguageTable [language];
 572         if (lang->kinds != NULL  ||  lang->regex)
 573                 printf ("%s%s\n", lang->name, lang->enabled ? "" : " [disabled]");
 574 }
 575
 576 extern void printLanguageList (void)
 577 {
 578         unsigned int i;
 579         for (i = 0  ;  i < LanguageCount  ;  ++i)
 580                 printLanguage (i);
 581 }
 582
 583 /*
 584 *   File parsing
 585 */
 586
 587 static void makeFileTag (const char *const fileName)
 588 {
 589         if (Option.include.fileNames)
 590         {
 591                 tagEntryInfo tag;
 592                 initTagEntry (&tag, baseFilename (fileName));
 593
 594                 tag.isFileEntry     = TRUE;
 595                 tag.lineNumberEntry = TRUE;
 596                 tag.lineNumber      = 1;
 597                 tag.kindName        = "file";
 598                 tag.kind            = 'F';
 599
 600                 makeTagEntry (&tag);
 601         }
 602 }
 603
 604 static boolean createTagsForFile (
 605                 const char *const fileName, const langType language,
 606                 const unsigned int passCount)
 607 {
 608         boolean retried = FALSE;
 609         Assert (0 <= language  &&  language < (int) LanguageCount);
 610         if (fileOpen (fileName, language))
 611         {
 612                 const parserDefinition* const lang = LanguageTable [language];
 613                 if (Option.etags)
 614                         beginEtagsFile ();
 615
 616                 makeFileTag (fileName);
 617
 618                 if (lang->parser != NULL)
 619                         lang->parser ();
 620                 else if (lang->parser2 != NULL)
 621                         retried = lang->parser2 (passCount);
 622
 623                 if (Option.etags)
 624                         endEtagsFile (getSourceFileTagPath ());
 625
 626                 fileClose ();
 627         }
 628
 629         return retried;
 630 }
 631
 632 static boolean createTagsWithFallback (
 633                 const char *const fileName, const langType language)
 634 {
 635         const unsigned long numTags     = TagFile.numTags.added;
 636         fpos_t tagFilePosition;
 637         unsigned int passCount = 0;
 638         boolean tagFileResized = FALSE;
 639
 640         fgetpos (TagFile.fp, &tagFilePosition);
 641         while (createTagsForFile (fileName, language, ++passCount))
 642         {
 643                 /*  Restore prior state of tag file.
 644                  */
 645                 fsetpos (TagFile.fp, &tagFilePosition);
 646                 TagFile.numTags.added = numTags;
 647                 tagFileResized = TRUE;
 648         }
 649         return tagFileResized;
 650 }
 651
 652 extern boolean parseFile (const char *const fileName)
 653 {
 654         boolean tagFileResized = FALSE;
 655         langType language = Option.language;
 656         if (Option.language == LANG_AUTO)
 657                 language = getFileLanguage (fileName);
 658         Assert (language != LANG_AUTO);
 659         if (language == LANG_IGNORE)
 660                 verbose ("ignoring %s (unknown language)\n", fileName);
 661         else if (! LanguageTable [language]->enabled)
 662                 verbose ("ignoring %s (language disabled)\n", fileName);
 663         else
 664         {
 665                 if (Option.filter)
 666                         openTagFile ();
 667
 668                 tagFileResized = createTagsWithFallback (fileName, language);
 669
 670                 if (Option.filter)
 671                         closeTagFile (tagFileResized);
 672                 addTotals (1, 0L, 0L);
 673
 674                 return tagFileResized;
 675         }
 676         return tagFileResized;
 677 }
 678
 679 /* vi:set tabstop=4 shiftwidth=4 nowrap: */