tools/migratetmpl.pl

   1 #!/usr/bin/perl -w
   2
   3 #############################################################################
   4 # Script      : migratetmpl.pl
   5 # Description : Read template files and reformate them as source code
   6 #               comments.
   7 #               Run from doc dir.
   8 # Example     : cd glib/docs/reference/glib
   9 #               migratetmpl.pl --module=glib --source-dir=../../../glib
  10 #               cd glib/docs/reference/gmodule
  11 #               migratetmpl.pl --module=glib --source-dir=../../../glib
  12 #               ...
  13 # Todo        : - remove docs from tmpl files if there are comments in the src
  14 #               - allow running for only 1 tmpl file (= one section)
  15 #############################################################################
  16
  17 use strict;
  18 use Getopt::Long;
  19 use Parse::ExuberantCTags;
  20
  21 # Options
  22
  23 # name of documentation module
  24 my $MODULE;
  25 my $TMPL_DIR;
  26 my $EXTRA_SRC_DIR;
  27 my $SRC_DIR;
  28 my $PRINT_HELP;
  29
  30 my %optctl = ('module' => \$MODULE,
  31               'tmpl-dir' => \$TMPL_DIR,
  32               'extra-source-dir' => \$EXTRA_SRC_DIR,
  33               'source-dir' => \$SRC_DIR,
  34               'help' => \$PRINT_HELP);
  35 GetOptions(\%optctl, "module=s", "tmpl-dir:s", "source-dir:s", "help");
  36
  37 if (!$MODULE) {
  38     $PRINT_HELP = 1;
  39 }
  40
  41 if ($PRINT_HELP) {
  42     print <<EOF;
  43 migratetmpl.pl
  44
  45 --module=MODULE_NAME       Name of the doc module being parsed
  46 --tmpl-dir=DIRNAME         Directory in which template files may be found
  47 --extra-source-dir=DIRNAME Directory where to put the generated sources
  48 --source-dir=DIRNAME       Directory of existing sources
  49 --help                     Print this help
  50 EOF
  51     exit 0;
  52 }
  53
  54
  55 my $ROOT_DIR = ".";
  56 if (! -e "$ROOT_DIR/$MODULE-sections.txt") {
  57     die "No $ROOT_DIR/$MODULE-sections.txt file found, please run this from doc-dir";
  58 }
  59
  60 # All the files are read from subdirectories beneath here.
  61 $TMPL_DIR = $TMPL_DIR ? $TMPL_DIR : "$ROOT_DIR/tmpl";
  62 $EXTRA_SRC_DIR = $EXTRA_SRC_DIR ? $EXTRA_SRC_DIR : "$ROOT_DIR/src";
  63
  64 # These global hashes store the existing documentation.
  65 my %SymbolDocs;
  66 my %SymbolTypes;
  67 my %SymbolParams;
  68 # These global hashes store declaration info keyed on a symbol name.
  69 my %Deprecated;
  70 my %Since;
  71 my %StabilityLevel;
  72
  73 # build and read tags
  74 my $tags;
  75 if (-e $SRC_DIR) {
  76     `cd $SRC_DIR; make ctags`;
  77     $tags = Parse::ExuberantCTags->new("$SRC_DIR/tags");
  78 }
  79
  80 # Create the src output directory if it doens't exist.
  81 if (! -e $EXTRA_SRC_DIR) {
  82     mkdir ("$EXTRA_SRC_DIR", 0777)
  83         || die "Can't create directory: $EXTRA_SRC_DIR";
  84 }
  85
  86 # now process the files
  87 &Convert ("$ROOT_DIR/$MODULE-sections.txt");
  88
  89
  90 #############################################################################
  91 # Function    : Convert
  92 # Description : Parse the section file and convert each template back to source
  93 #               comments.
  94 # Arguments   : $file - the section file that lists all templates
  95 #############################################################################
  96 sub Convert {
  97     my ($file) = @_;
  98
  99     #print "Reading: $file\n";
 100     open (INPUT, $file)
 101         || die "Can't open $file: $!";
 102
 103     my $filename = "";
 104
 105     while (<INPUT>) {
 106         if (m/^#/) {
 107             next;
 108         } elsif (m/^<FILE>(.*)<\/FILE>/) {
 109             $filename = $1;
 110             if (&ReadTemplateFile ("$TMPL_DIR/$filename")) {
 111                &OutputSourceFile ("$EXTRA_SRC_DIR/$filename",$filename);
 112             }
 113         }
 114     }
 115     close (INPUT);
 116 }
 117
 118
 119 #############################################################################
 120 # Function    : ReadTemplateFile
 121 # Description : This reads in the manually-edited documentation file
 122 #               corresponding to the file currently being created, so we can
 123 #               insert the documentation at the appropriate places.
 124 #               It outputs %SymbolTypes, %SymbolDocs and %SymbolParams, which
 125 #               is a hash of arrays.
 126 #               NOTE: This function is duplicated in gtkdoc-mktmpl (but
 127 #               slightly different).
 128 # Arguments   : $docsfile - the template file to read in.
 129 #############################################################################
 130 sub ReadTemplateFile {
 131     my ($docsfile) = @_;
 132
 133     my $template = "$docsfile.sgml";
 134     if (! -f $template) {
 135         #print "File doesn't exist: $template\n";
 136         return 0;
 137     }
 138     #print "Reading $template\n";
 139
 140     # start with empty hashes, we merge the source comment for each file
 141     # afterwards
 142     %SymbolDocs = ();
 143     %SymbolTypes = ();
 144     %SymbolParams = ();
 145
 146     my $current_type = "";      # Type of symbol being read.
 147     my $current_symbol = "";    # Name of symbol being read.
 148     my $symbol_doc = "";                # Description of symbol being read.
 149     my @params;                 # Parameter names and descriptions of current
 150                                 #   function/macro/function typedef.
 151     my $current_param = -1;     # Index of parameter currently being read.
 152                                 #   Note that the param array contains pairs
 153                                 #   of param name & description.
 154     my $in_unused_params = 0;   # True if we are reading in the unused params.
 155     my $in_deprecated = 0;
 156     my $in_since = 0;
 157     my $in_stability = 0;
 158
 159     open (DOCS, "$template")
 160         || die "Can't open $template: $!";
 161     while (<DOCS>) {
 162         if (m/^<!-- ##### ([A-Z_]+) (\S+) ##### -->/) {
 163             my $type = $1;
 164             my $symbol = $2;
 165             if ($symbol eq "Title"
 166                 || $symbol eq "Short_Description"
 167                 || $symbol eq "Long_Description"
 168                 || $symbol eq "See_Also"
 169                 || $symbol eq "Stability_Level"
 170                 || $symbol eq "Include") {
 171
 172                 $symbol = $docsfile . ":" . $symbol;
 173             }
 174
 175             #print "Found symbol: $symbol\n";
 176
 177             # Store previous symbol, but remove any trailing blank lines.
 178             if ($current_symbol ne "") {
 179                 $symbol_doc =~ s/\s+$//;
 180                 $SymbolTypes{$current_symbol} = $current_type;
 181                 $SymbolDocs{$current_symbol} = $symbol_doc;
 182
 183                 # Check that the stability level is valid.
 184                 if ($StabilityLevel{$current_symbol}) {
 185                     $StabilityLevel{$current_symbol} = &ParseStabilityLevel($StabilityLevel{$current_symbol}, $template, $., "Stability level for $current_symbol");
 186                 }
 187
 188                 if ($current_param >= 0) {
 189                     $SymbolParams{$current_symbol} = [ @params ];
 190                 } else {
 191                     # Delete any existing params in case we are overriding a
 192                     # previously read template.
 193                     delete $SymbolParams{$current_symbol};
 194                 }
 195             }
 196             $current_type = $type;
 197             $current_symbol = $symbol;
 198             $current_param = -1;
 199             $in_unused_params = 0;
 200             $in_deprecated = 0;
 201             $in_since = 0;
 202             $in_stability = 0;
 203             $symbol_doc = "";
 204             @params = ();
 205
 206         } elsif (m/^<!-- # Unused Parameters # -->/) {
 207             #print "DEBUG: Found unused parameters\n";
 208             $in_unused_params = 1;
 209             next;
 210
 211         } elsif ($in_unused_params) {
 212             #print "DEBUG: Skipping unused param: $_";
 213             next;
 214
 215         } else {
 216             # Check if param found. Need to handle "..." and "format...".
 217             if (s/^\@([\w\.]+):\040?//) {
 218                 my $param_name = $1;
 219                 # Allow variations of 'Returns'
 220                 if ($param_name =~ m/^[Rr]eturns?$/) {
 221                     $param_name = "Returns";
 222                 }
 223                 #print "Found param for symbol $current_symbol : '$param_name'= '$_'\n";
 224
 225                 if ($param_name eq "Deprecated") {
 226                     $in_deprecated = 1;
 227                     $Deprecated{$current_symbol} = $_;
 228                 } elsif ($param_name eq "Since") {
 229                     $in_since = 1;
 230                     $Since{$current_symbol} = $_;
 231                 } elsif ($param_name eq "Stability") {
 232                     $in_stability = 1;
 233                     $StabilityLevel{$current_symbol} = $_;
 234                 } else {
 235                     push (@params, $param_name);
 236                     push (@params, $_);
 237                     $current_param += 2;
 238                 }
 239             } else {
 240                 if ($in_deprecated) {
 241                     $Deprecated{$current_symbol} .= $_;
 242                 } elsif ($in_since) {
 243                     $Since{$current_symbol} .= $_;
 244                 } elsif ($in_stability) {
 245                     $StabilityLevel{$current_symbol} .= $_;
 246                 } elsif ($current_param >= 0) {
 247                     $params[$current_param] .= $_;
 248                 } else {
 249                     $symbol_doc .= $_;
 250                 }
 251             }
 252         }
 253     }
 254
 255     # Remember to finish the current symbol doccs.
 256     if ($current_symbol ne "") {
 257         $symbol_doc =~ s/\s+$//;
 258         $SymbolTypes{$current_symbol} = $current_type;
 259         $SymbolDocs{$current_symbol} = $symbol_doc;
 260
 261         # Check that the stability level is valid.
 262         if ($StabilityLevel{$current_symbol}) {
 263             $StabilityLevel{$current_symbol} = &ParseStabilityLevel($StabilityLevel{$current_symbol}, $template, $., "Stability level for $current_symbol");
 264         }
 265
 266         if ($current_param >= 0) {
 267             $SymbolParams{$current_symbol} = [ @params ];
 268         } else {
 269             # Delete any existing params in case we are overriding a
 270             # previously read template.
 271             delete $SymbolParams{$current_symbol};
 272         }
 273     }
 274
 275     close (DOCS);
 276     return 1;
 277 }
 278
 279 #############################################################################
 280 # Function    : OutputSourceFile
 281 # Description : Format the source comments for one file.
 282 # Arguments   : $docsfile -  the basename for the output file
 283 #               $file - base filename
 284 #############################################################################
 285 sub OutputSourceFile {
 286     my ($docsfile,$file) = @_;
 287
 288     my $source = "$docsfile.c";
 289
 290     open (OUTPUT, ">$source")
 291         || die "Can't create $source";
 292
 293     # output section docs
 294     my ($title, $short_desc, $long_desc, $see_also, $stability);
 295
 296     if (defined ($SymbolDocs{"$TMPL_DIR/$file:Title"})) {
 297         $title = $SymbolDocs{"$TMPL_DIR/$file:Title"};
 298         delete $SymbolDocs{"$TMPL_DIR/$file:Title"};
 299     }
 300     if (defined ($SymbolDocs{"$TMPL_DIR/$file:Short_Description"})) {
 301         $short_desc = $SymbolDocs{"$TMPL_DIR/$file:Short_Description"};
 302         delete $SymbolDocs{"$TMPL_DIR/$file:Short_Description"};
 303     }
 304     if (defined ($SymbolDocs{"$TMPL_DIR/$file:Long_Description"})) {
 305         $long_desc = $SymbolDocs{"$TMPL_DIR/$file:Long_Description"};
 306         $long_desc = &ConvertMarkup ($long_desc);
 307         $long_desc = &ConvertComments ($long_desc);
 308         delete $SymbolDocs{"$TMPL_DIR/$file:Long_Description"};
 309     }
 310     if (defined ($SymbolDocs{"$TMPL_DIR/$file:See_Also"})) {
 311         $see_also = $SymbolDocs{"$TMPL_DIR/$file:See_Also"};
 312         $see_also = &ConvertMarkup ($see_also);
 313         delete $SymbolDocs{"$TMPL_DIR/$file:See_Also"};
 314     }
 315     if (defined ($SymbolDocs{"$TMPL_DIR/$file:Stability_Level"})) {
 316         $stability = $SymbolDocs{"$TMPL_DIR/$file:Stability_Level"};
 317         delete $SymbolDocs{"$TMPL_DIR/$file:Stability_Level"};
 318     }
 319
 320     print (OUTPUT <<EOF);
 321 /**
 322  * SECTION:$file
 323 EOF
 324     if (defined($short_desc) && ($short_desc ne "")) {
 325         print (OUTPUT " * \@Short_description: $short_desc\n");
 326     }
 327     if (defined($title) && ($title ne "")) {
 328         print (OUTPUT " * \@Title: $title\n");
 329     }
 330     if (defined($see_also) && ($see_also ne "")) {
 331         my $line;
 332         my $first="\@See_also:";
 333         for $line (split (/\n/, $see_also)) {
 334             print (OUTPUT " * $first$line\n");
 335             $first="";
 336         }
 337     }
 338     if (defined($stability) && ($stability ne "")) {
 339         print (OUTPUT " * \@Stability: $stability\n");
 340     }
 341     if (defined($long_desc) && ($long_desc ne "")) {
 342         my $line;
 343         print (OUTPUT " * \n");
 344         for $line (split (/\n/, $long_desc)) {
 345             print (OUTPUT " * $line\n");
 346         }
 347     }
 348     print (OUTPUT <<EOF);
 349  */
 350
 351
 352 EOF
 353
 354     # output symbol docs
 355     my $symbol;
 356     my $docblob;
 357     my $merged;
 358     foreach $symbol (keys (%SymbolDocs)) {
 359         $docblob=&GetSymbolDoc ($symbol);
 360         next if (!defined $docblob);
 361
 362         $merged=0;
 363         # if we have tags, check if we find the symbol location
 364         if (defined $tags) {
 365             my $tag = $tags->findTag($symbol, ignore_case => 0, partial => 0);
 366             if (defined $tag) {
 367                 my $srcline;
 368                 my $srcfile=$tag->{file};
 369
 370                 if (defined $tag->{addressLineNumber}) {
 371                     $srcline=$tag->{addressLineNumber}
 372                 }
 373                 if (-e "$SRC_DIR/$srcfile") {
 374                     my @lines;
 375                     my $line;
 376
 377                     my $source = "$SRC_DIR/$srcfile";
 378
 379                     open (SRC, "$source");
 380                     @lines = <SRC>;
 381                     close (SRC);
 382
 383                     if (!defined $srcline) {
 384                         my $re  = $tag->{addressPattern};
 385                         $re =~ m#^/(.*)/$#;
 386                         $re = $1;
 387                         $re =~ s/([*()])/\\$1/g;
 388                         #$re = qr/$re/xo;
 389
 390                         for (0..$#lines) {
 391                             if ($lines[$_] =~ $re) {
 392                                 $srcline=$_+1;
 393                                 last;
 394                             }
 395                         }
 396                         if (!defined $srcline) {
 397                             print "no line found for : $symbol in $srcfile using regexp: ", $re, "\n";
 398                         }
 399                     }
 400
 401                     if (defined $srcline) {
 402                         my $offset = $srcline-1;
 403
 404                         if ($SymbolTypes{$symbol} eq "FUNCTION") {
 405                             # go one up to skip return type
 406                             # FIXME: check if the $symbol starts at the begin of the line
 407                             # if ($lines[$srcline] =~ m/^$symbol/)
 408                             $offset -= 1;
 409                         }
 410
 411                         splice @lines,$offset,$#lines-$offset,($docblob,@lines[$offset..($#lines+-1)]);
 412
 413                         # patch file and overwrite
 414                         open (SRC, ">$source");
 415                         print SRC @lines;
 416                         close (SRC);
 417                         $merged=1;
 418                         # rebuild and reread ctags
 419                         `cd $SRC_DIR; make ctags`;
 420                         $tags = Parse::ExuberantCTags->new("$SRC_DIR/tags");
 421                     }
 422
 423                 }
 424                 else {
 425                     print "no source found for : $symbol\n";
 426                 }
 427             }
 428             else {
 429                 print "no tag found for : $symbol\n";
 430             }
 431         }
 432         if ($merged == 0) {
 433           print (OUTPUT $docblob."\n\n");
 434         }
 435         undef $docblob;
 436     }
 437
 438     close (OUTPUT);
 439 }
 440
 441 #############################################################################
 442 # Function    : GetSymbolDoc
 443 # Description : Format the docs for one symbol
 444 # Arguments   : $symbol -  the symbol
 445 #############################################################################
 446 sub GetSymbolDoc {
 447     my ($symbol) = @_;
 448
 449     my $str;
 450     my ($params, $long_desc, $stability, $deprecated, $since, $returns);
 451     my $is_empty =1;
 452
 453     if (defined ($SymbolParams{$symbol})) {
 454         $params = $SymbolParams{$symbol}
 455     }
 456     if (defined ($StabilityLevel{$symbol})) {
 457         $stability = $StabilityLevel{$symbol};
 458     }
 459     if (defined ($Deprecated{$symbol})) {
 460         $deprecated = $Deprecated{$symbol};
 461         $deprecated = &FormatMultiline ($deprecated);
 462     }
 463     if (defined ($Since{$symbol})) {
 464         ($since, undef) = split (/\n/,$Since{$symbol});
 465     }
 466     $long_desc = $SymbolDocs{$symbol};
 467     $long_desc = &ConvertMarkup ($long_desc);
 468     $long_desc = &ConvertComments ($long_desc);
 469
 470     $str=<<EOF;
 471 /**
 472  * $symbol:
 473 EOF
 474     if (defined($params)) {
 475         my $j;
 476         for ($j = 0; $j <= $#$params; $j += 2) {
 477             my $param_name = $$params[$j];
 478             my $param_desc = $$params[$j+1];
 479             my $line;
 480             $param_desc = &FormatMultiline ($param_desc);
 481
 482             if ($param_desc ne "\n") {
 483                 $is_empty = 0;
 484             }
 485
 486             if ($param_name eq "Varargs") {
 487                 $param_name="...";
 488             }
 489
 490             if ($param_name eq "Returns") {
 491                 $returns = $param_desc;
 492                 chomp($returns);
 493             } else {
 494                 $str = $str." * \@$param_name: $param_desc";
 495             }
 496         }
 497     }
 498     if (defined($long_desc) && ($long_desc ne "")) {
 499         my $line;
 500         $str = $str." * \n";
 501         for $line (split (/\n/, $long_desc)) {
 502             $str = $str." * $line\n";
 503         }
 504         $is_empty = 0;
 505     }
 506     my $spacer=" * \n";
 507     if (defined($stability) && ($stability ne "")) {
 508         $str = $str.$spacer." * Stability: $stability\n";
 509         $spacer="";
 510         $is_empty = 0;
 511     }
 512     if (defined($deprecated) && ($deprecated ne "")) {
 513         $str = $str.$spacer." * Deprecated: $deprecated";
 514         $spacer="";
 515         $is_empty = 0;
 516     }
 517     if (defined($since) && ($since ne "")) {
 518         $str = $str.$spacer." * Since: $since\n";
 519         $spacer="";
 520         $is_empty = 0;
 521     }
 522     if (defined($returns) && ($returns ne "")) {
 523         $str = $str.$spacer." * Returns: $returns\n";
 524         $is_empty = 0;
 525     }
 526     $str = $str. <<EOF;
 527  */
 528 EOF
 529     if($is_empty == 1) {
 530         #print "empty docs for $symbol\n";
 531         return;
 532     }
 533     return $str;
 534 }
 535
 536 #############################################################################
 537 # Function    : ConvertMarkup
 538 # Description : Convert para tags to newlines and character entities back
 539 # Arguments   : $istr -  string to convert
 540 #############################################################################
 541 sub ConvertMarkup {
 542     my ($istr) = @_;
 543     my ($line,$ostr);
 544
 545     $ostr="";
 546     for $line (split (/\n/, $istr)) {
 547         if ($line =~ m/^\s*<para>\s*$/) {
 548             # new paragraph
 549             next;
 550         } elsif ($line =~ m/^\s*<\/para>\s*$/) {
 551             # end of paragraph
 552             $ostr.="\n";
 553         } else {
 554             # convert character entities back.
 555             $line =~ s/&amp;/&/g;
 556             $line =~ s/&num;/#/g;
 557             $line =~ s/&lt;/</g;
 558             $line =~ s/&gt;/>/g;
 559             $ostr.="$line\n";
 560         }
 561     }
 562
 563     return $ostr;
 564 }
 565
 566 #############################################################################
 567 # Function    : ConvertComments
 568 # Description : Convert single line c comments to c++ comments
 569 # Arguments   : $istr -  string to convert
 570 #############################################################################
 571 sub ConvertComments {
 572     my ($istr) = @_;
 573     my ($line,$ostr);
 574
 575     $ostr="";
 576     for $line (split (/\n/, $istr)) {
 577         if ($line =~ m#/\*.*\*/#) {
 578             $line =~ s#/\*#//#;
 579             $line =~ s#\s*\*/##;
 580         }
 581         $line =~ s#/\*#/<!---->\*#;
 582         $line =~ s#\*/#\*<!---->/#;
 583         $ostr.="$line\n";
 584     }
 585
 586     return $ostr;
 587 }
 588
 589 #############################################################################
 590 # Function    : FormatMultiline
 591 # Description : Format multiline text and remove blank lines
 592 # Arguments   : $istr -  string to convert
 593 #############################################################################
 594 sub FormatMultiline {
 595     my ($istr) = @_;
 596     my ($line,$ostr);
 597
 598     $ostr="";
 599     for $line (split (/\n/, $istr)) {
 600         if ($line ne "") {
 601             $line =~ m/\s*(.*)\s*/g;
 602             if ($ostr eq "") {
 603                 $ostr.="$1\n";
 604             } else {
 605                 $ostr.=" *  $1\n";
 606             }
 607         }
 608     }
 609     if ($ostr eq "") {
 610         $ostr="\n";
 611     }
 612
 613     return $ostr;
 614 }
 615
 616