misc/translator/tmpl_process3.pl

   1 #!/usr/bin/perl
   2 # This file is part of Koha
   3 # Parts copyright 2003-2004 Paul Poulain
   4 # Parts copyright 2003-2004 Jerome Vizcaino
   5 # Parts copyright 2004 Ambrose Li
   6
   7 =head1 NAME
   8
   9 tmpl_process3.pl - Alternative version of tmpl_process.pl
  10 using gettext-compatible translation files
  11
  12 =cut
  13
  14 use strict;
  15 use Getopt::Long;
  16 use Locale::PO;
  17 use File::Temp qw( :POSIX );
  18 use TmplTokenizer;
  19 use VerboseWarnings qw( :warn :die );
  20
  21 ###############################################################################
  22
  23 use vars qw( @in_files $in_dir $str_file $out_dir $quiet );
  24 use vars qw( @excludes $exclude_regex );
  25 use vars qw( $recursive_p );
  26 use vars qw( $pedantic_p );
  27 use vars qw( $href );
  28 use vars qw( $type );   # file extension (DOS form without the dot) to match
  29 use vars qw( $charset_in $charset_out );
  30
  31 ###############################################################################
  32
  33 sub find_translation ($) {
  34     my($s) = @_;
  35     my $key = $s;
  36     if ($s =~ /\S/s) {
  37         $key = TmplTokenizer::string_canon($key);
  38         $key = TmplTokenizer::charset_convert($key, $charset_in, $charset_out);
  39         $key = TmplTokenizer::quote_po($key);
  40     }
  41     return defined $href->{$key}
  42                 && !$href->{$key}->fuzzy
  43                 && length Locale::PO->dequote($href->{$key}->msgstr)?
  44            Locale::PO->dequote($href->{$key}->msgstr): $s;
  45 }
  46
  47 sub text_replace_tag ($$) {
  48     my($t, $attr) = @_;
  49     my $it;
  50     # value [tag=input], meta
  51     my $tag = lc($1) if $t =~ /^<(\S+)/s;
  52     my $translated_p = 0;
  53     for my $a ('alt', 'content', 'title', 'value') {
  54         if ($attr->{$a}) {
  55             next if $a eq 'content' && $tag ne 'meta';
  56             next if $a eq 'value' && ($tag ne 'input'
  57                 || (ref $attr->{'type'} && $attr->{'type'}->[1] =~ /^(?:hidden|radio|text)$/)); # FIXME
  58             my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
  59             if ($val =~ /\S/s) {
  60                 my $s = find_translation($val);
  61                 if ($attr->{$a}->[1] ne $s) { #FIXME
  62                     $attr->{$a}->[1] = $s; # FIXME
  63                     $attr->{$a}->[2] = ($s =~ /"/s)? "'$s'": "\"$s\""; #FIXME
  64                     $translated_p = 1;
  65                 }
  66             }
  67         }
  68     }
  69     if ($translated_p) {
  70         $it = "<$tag"
  71             . join('', map {
  72                     sprintf(' %s=%s', $_, $attr->{$_}->[2]) #FIXME
  73                 } sort {
  74                     $attr->{$a}->[3] <=> $attr->{$b}->[3] #FIXME
  75                 } keys %$attr)
  76             . '>';
  77     } else {
  78         $it = $t;
  79     }
  80     return $it;
  81 }
  82
  83 sub text_replace (**) {
  84     my($h, $output) = @_;
  85     for (;;) {
  86         my $s = TmplTokenizer::next_token $h;
  87     last unless defined $s;
  88         my($kind, $t, $attr) = ($s->type, $s->string, $s->attributes);
  89         if ($kind eq TmplTokenType::TEXT) {
  90             print $output find_translation($t);
  91         } elsif ($kind eq TmplTokenType::TEXT_PARAMETRIZED) {
  92             my $fmt = find_translation($s->form);
  93             print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
  94                 $_ = $_[0];
  95                 my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
  96                 $kind == TmplTokenType::TAG && %$attr?
  97                     text_replace_tag($t, $attr): $t });
  98         } elsif ($kind eq TmplTokenType::TAG && %$attr) {
  99             print $output text_replace_tag($t, $attr);
 100         } elsif ($s->has_js_data) {
 101             for my $t (@{$s->js_data}) {
 102                 # FIXME for this whole block
 103                 if ($t->[0]) {
 104                     printf $output "%s%s%s", $t->[2], find_translation $t->[3],
 105                             $t->[2];
 106                 } else {
 107                     print $output $t->[1];
 108                 }
 109             }
 110         } elsif (defined $t) {
 111             print $output $t;
 112         }
 113     }
 114 }
 115
 116 sub listfiles ($$$) {
 117     my($dir, $type, $action) = @_;
 118     my @it = ();
 119     if (opendir(DIR, $dir)) {
 120         my @dirent = readdir DIR;       # because DIR is shared when recursing
 121         closedir DIR;
 122         for my $dirent (@dirent) {
 123             my $path = "$dir/$dirent";
 124             if ($dirent =~ /^\./ || $dirent eq 'CVS' || $dirent eq 'RCS'
 125             || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) {
 126                 ;
 127             } elsif (-f $path) {
 128                 push @it, $path if (!defined $type || $dirent =~ /\.(?:$type)$/) || $action eq 'install';
 129             } elsif (-d $path && $recursive_p) {
 130                 push @it, listfiles($path, $type, $action);
 131             }
 132         }
 133     } else {
 134         warn_normal "$dir: $!", undef;
 135     }
 136     return @it;
 137 }
 138
 139 ###############################################################################
 140
 141 sub mkdir_recursive ($) {
 142     my($dir) = @_;
 143     local($`, $&, $', $1);
 144     $dir = $` if $dir ne /^\/+$/ && $dir =~ /\/+$/;
 145     my ($prefix, $basename) = ($dir =~ /\/([^\/]+)$/s)? ($`, $1): ('.', $dir);
 146     mkdir_recursive($prefix) if $prefix ne '.' && !-d $prefix;
 147     if (!-d $dir) {
 148         print STDERR "Making directory $dir..." unless $quiet;
 149         # creates with rwxrwxr-x permissions
 150         mkdir($dir, 0775) || warn_normal "$dir: $!", undef;
 151     }
 152 }
 153
 154 ###############################################################################
 155
 156 sub usage ($) {
 157     my($exitcode) = @_;
 158     my $h = $exitcode? *STDERR: *STDOUT;
 159     print $h <<EOF;
 160 Usage: $0 create [OPTION]
 161   or:  $0 update [OPTION]
 162   or:  $0 install [OPTION]
 163   or:  $0 --help
 164 Create or update PO files from templates, or install translated templates.
 165
 166   -i, --input=SOURCE          Get or update strings from SOURCE file.
 167                               SOURCE is a directory if -r is also specified.
 168   -o, --outputdir=DIRECTORY   Install translation(s) to specified DIRECTORY
 169       --pedantic-warnings     Issue warnings even for detected problems
 170                               which are likely to be harmless
 171   -r, --recursive             SOURCE in the -i option is a directory
 172   -s, --str-file=FILE         Specify FILE as the translation (po) file
 173                               for input (install) or output (create, update)
 174   -x, --exclude=REGEXP        Exclude files matching the given REGEXP
 175       --help                  Display this help and exit
 176   -q, --quiet                 no output to screen (except for errors)
 177
 178 The -o option is ignored for the "create" and "update" actions.
 179 Try `perldoc $0 for perhaps more information.
 180 EOF
 181     exit($exitcode);
 182 }#`
 183
 184 ###############################################################################
 185
 186 sub usage_error (;$) {
 187     for my $msg (split(/\n/, $_[0])) {
 188         print STDERR "$msg\n";
 189     }
 190     print STDERR "Try `$0 --help for more information.\n";
 191     exit(-1);
 192 }
 193
 194 ###############################################################################
 195
 196 GetOptions(
 197     'input|i=s'                         => \@in_files,
 198     'outputdir|o=s'                     => \$out_dir,
 199     'recursive|r'                       => \$recursive_p,
 200     'str-file|s=s'                      => \$str_file,
 201     'exclude|x=s'                       => \@excludes,
 202         'quiet|q'                               => \$quiet,
 203     'pedantic-warnings|pedantic'        => sub { $pedantic_p = 1 },
 204     'help'                              => \&usage,
 205 ) || usage_error;
 206
 207 VerboseWarnings::set_application_name $0;
 208 VerboseWarnings::set_pedantic_mode $pedantic_p;
 209
 210 # keep the buggy Locale::PO quiet if it says stupid things
 211 $SIG{__WARN__} = sub {
 212         my($s) = @_;
 213         print STDERR $s unless $s =~ /^Strange line in [^:]+: #~/s
 214     };
 215
 216 my $action = shift or usage_error('You must specify an ACTION.');
 217 usage_error('You must at least specify input and string list filenames.')
 218     if !@in_files || !defined $str_file;
 219
 220 # Type match defaults to *.tmpl plus *.inc if not specified
 221 $type = "tmpl|inc" if !defined($type);
 222
 223 # Check the inputs for being files or directories
 224 for my $input (@in_files) {
 225     usage_error("$input: Input must be a file or directory.\n"
 226             . "(Symbolic links are not supported at the moment)")
 227         unless -d $input || -f $input;;
 228 }
 229
 230 # Generates the global exclude regular expression
 231 $exclude_regex =  '(?:'.join('|', @excludes).')' if @excludes;
 232
 233 # Generate the list of input files if a directory is specified
 234 if (-d $in_files[0]) {
 235     die "If you specify a directory as input, you must specify only it.\n"
 236             if @in_files > 1;
 237
 238     # input is a directory, generates list of files to process
 239     $in_dir = $in_files[0];
 240     $in_dir =~ s/\/$//; # strips the trailing / if any
 241     @in_files = listfiles($in_dir, $type, $action);
 242 } else {
 243     for my $input (@in_files) {
 244         die "You cannot specify input files and directories at the same time.\n"
 245                 unless -f $input;
 246     }
 247 }
 248
 249 # restores the string list from file
 250 $href = Locale::PO->load_file_ashash($str_file);
 251
 252 # guess the charsets. HTML::Templates defaults to iso-8859-1
 253 if (defined $href) {
 254     die "$str_file: PO file is corrupted, or not a PO file\n"
 255             unless defined $href->{'""'};
 256     $charset_out = TmplTokenizer::charset_canon $2
 257             if $href->{'""'}->msgstr =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/;
 258             $charset_in = 'utf-8';
 259 #     for my $msgid (keys %$href) {
 260 #       if ($msgid =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/) {
 261 #           my $candidate = TmplTokenizer::charset_canon $2;
 262 #           die "Conflicting charsets in msgid: $charset_in vs $candidate => $msgid\n"
 263 #                   if defined $charset_in && $charset_in ne $candidate;
 264 #           $charset_in = $candidate;
 265 #       }
 266 #     }
 267 }
 268 if (!defined $charset_in) {
 269     $charset_in = TmplTokenizer::charset_canon 'utf-8';
 270     warn "Warning: Can't determine original templates' charset, defaulting to $charset_in\n";
 271 }
 272
 273 my $xgettext = './xgettext.pl'; # actual text extractor script
 274 my $st;
 275
 276 if ($action eq 'create')  {
 277     # updates the list. As the list is empty, every entry will be added
 278     if (!-s $str_file) {
 279         warn "Removing empty file $str_file\n";
 280         unlink $str_file || die "$str_file: $!\n";
 281     }
 282     die "$str_file: Output file already exists\n" if -f $str_file;
 283     my($tmph1, $tmpfile1) = tmpnam();
 284     my($tmph2, $tmpfile2) = tmpnam();
 285     close $tmph2; # We just want a name
 286     # Generate the temporary file that acts as <MODULE>/POTFILES.in
 287     for my $input (@in_files) {
 288         print $tmph1 "$input\n";
 289     }
 290     close $tmph1;
 291     # Generate the specified po file ($str_file)
 292     $st = system ($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2);
 293     # Run msgmerge so that the pot file looks like a real pot file
 294     # We need to help msgmerge a bit by pre-creating a dummy po file that has
 295     # the headers and the "" msgid & msgstr. It will fill in the rest.
 296     if ($st == 0) {
 297         # Merge the temporary "pot file" with the specified po file ($str_file)
 298         # FIXME: msgmerge(1) is a Unix dependency
 299         # FIXME: need to check the return value
 300         unless (-f $str_file) {
 301             local(*INPUT, *OUTPUT);
 302             open(INPUT, "<$tmpfile2");
 303             open(OUTPUT, ">$str_file");
 304             while (<INPUT>) {
 305                 print OUTPUT;
 306             last if /^\n/s;
 307             }
 308             close INPUT;
 309             close OUTPUT;
 310         }
 311         $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
 312     } else {
 313         error_normal "Text extraction failed: $xgettext: $!\n", undef;
 314         error_additional "Will not run msgmerge\n", undef;
 315     }
 316 #   unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
 317 #   unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
 318
 319 } elsif ($action eq 'update') {
 320     my($tmph1, $tmpfile1) = tmpnam();
 321     my($tmph2, $tmpfile2) = tmpnam();
 322     close $tmph2; # We just want a name
 323     # Generate the temporary file that acts as <MODULE>/POTFILES.in
 324     for my $input (@in_files) {
 325         print $tmph1 "$input\n";
 326     }
 327     close $tmph1;
 328     # Generate the temporary file that acts as <MODULE>/<LANG>.pot
 329     $st = system($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
 330             '--po-mode',
 331             (defined $charset_in? ('-I', $charset_in): ()),
 332             (defined $charset_out? ('-O', $charset_out): ()));
 333     if ($st == 0) {
 334         # Merge the temporary "pot file" with the specified po file ($str_file)
 335         # FIXME: msgmerge(1) is a Unix dependency
 336         # FIXME: need to check the return value
 337         $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
 338     } else {
 339         error_normal "Text extraction failed: $xgettext: $!\n", undef;
 340         error_additional "Will not run msgmerge\n", undef;
 341     }
 342 #   unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
 343 #   unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
 344
 345 } elsif ($action eq 'install') {
 346     if(!defined($out_dir)) {
 347         usage_error("You must specify an output directory when using the install method.");
 348     }
 349
 350     if ($in_dir eq $out_dir) {
 351         warn "You must specify a different input and output directory.\n";
 352         exit -1;
 353     }
 354
 355     # Make sure the output directory exists
 356     # (It will auto-create it, but for compatibility we should not)
 357     -d $out_dir || die "$out_dir: The directory does not exist\n";
 358
 359     # Try to open the file, because Locale::PO doesn't check :-/
 360     open(INPUT, "<$str_file") || die "$str_file: $!\n";
 361     close INPUT;
 362
 363     # creates the new tmpl file using the new translation
 364     for my $input (@in_files) {
 365                 die "Assertion failed"
 366                         unless substr($input, 0, length($in_dir) + 1) eq "$in_dir/";
 367 #               print "$input / $type\n";
 368                 if (!defined $type || $input =~ /\.(?:$type)$/) {
 369                         my $h = TmplTokenizer->new( $input );
 370                         $h->set_allow_cformat( 1 );
 371                         VerboseWarnings::set_input_file_name $input;
 372
 373                         my $target = $out_dir . substr($input, length($in_dir));
 374                         my $targetdir = $` if $target =~ /[^\/]+$/s;
 375                         mkdir_recursive($targetdir) unless -d $targetdir;
 376                         print STDERR "Creating $target...\n" unless $quiet;
 377                         open( OUTPUT, ">$target" ) || die "$target: $!\n";
 378                         text_replace( $h, *OUTPUT );
 379                         close OUTPUT;
 380                 } else {
 381                 # just copying the file
 382                         my $target = $out_dir . substr($input, length($in_dir));
 383                         my $targetdir = $` if $target =~ /[^\/]+$/s;
 384                         mkdir_recursive($targetdir) unless -d $targetdir;
 385                         system("cp -f $input $target");
 386                         print STDERR "Copying $input...\n" unless $quiet;
 387                 }
 388         }
 389
 390 } else {
 391     usage_error('Unknown action specified.');
 392 }
 393
 394 if ($st == 0) {
 395     printf "The %s seems to be successful.\n", $action unless $quiet;
 396 } else {
 397     printf "%s FAILED.\n", "\u$action" unless $quiet;
 398 }
 399 exit 0;
 400
 401 ###############################################################################
 402
 403 =head1 SYNOPSIS
 404
 405 ./tmpl_process3.pl [ I<tmpl_process.pl options> ]
 406
 407 =head1 DESCRIPTION
 408
 409 This is an alternative version of the tmpl_process.pl script,
 410 using standard gettext-style PO files.  While there still might
 411 be changes made to the way it extracts strings, at this moment
 412 it should be stable enough for general use; it is already being
 413 used for the Chinese and Polish translations.
 414
 415 Currently, the create, update, and install actions have all been
 416 reimplemented and seem to work.
 417
 418 =head2 Features
 419
 420 =over
 421
 422 =item -
 423
 424 Translation files in standard Uniforum PO format.
 425 All standard tools including all gettext tools,
 426 plus PO file editors like kbabel(1) etc.
 427 can be used.
 428
 429 =item -
 430
 431 Minor changes in whitespace in source templates
 432 do not generally require strings to be re-translated.
 433
 434 =item -
 435
 436 Able to handle <TMPL_VAR> variables in the templates;
 437 <TMPL_VAR> variables are usually extracted in proper context,
 438 represented by a short %s placeholder.
 439
 440 =item -
 441
 442 Able to handle text input and radio button INPUT elements
 443 in the templates; these INPUT elements are also usually
 444 extracted in proper context,
 445 represented by a short %S or %p placeholder.
 446
 447 =item -
 448
 449 Automatic comments in the generated PO files to provide
 450 even more context (line numbers, and the names and types
 451 of the variables).
 452
 453 =item -
 454
 455 The %I<n>$s (or %I<n>$p, etc.) notation can be used
 456 for change the ordering of the variables,
 457 if such a reordering is required for correct translation.
 458
 459 =item -
 460
 461 If a particular <TMPL_VAR> should not appear in the
 462 translation, it can be suppressed with the %0.0s notation.
 463
 464 =item -
 465
 466 Using the PO format also means translators can add their
 467 own comments in the translation files, if necessary.
 468
 469 =item -
 470
 471 Create, update, and install actions are all based on the
 472 same scanner module. This ensures that update and install
 473 have the same idea of what is a translatable string;
 474 attribute names in tags, for example, will not be
 475 accidentally translated.
 476
 477 =back
 478
 479 =head1 NOTES
 480
 481 Anchors are represented by an <AI<n>> notation.
 482 The meaning of this non-standard notation might not be obvious.
 483
 484 The create action calls xgettext.pl to do the actual work;
 485 the update action calls xgettext.pl and msgmerge(1) to do the
 486 actual work.
 487
 488 =head1 BUGS
 489
 490 xgettext.pl must be present in the current directory; the
 491 msgmerge(1) command must also be present in the search path.
 492 The script currently does not check carefully whether these
 493 dependent commands are present.
 494
 495 Locale::PO(3) has a lot of bugs. It can neither parse nor
 496 generate GNU PO files properly; a couple of workarounds have
 497 been written in TmplTokenizer and more is likely to be needed
 498 (e.g., to get rid of the "Strange line" warning for #~).
 499
 500 This script may not work in Windows.
 501
 502 There are probably some other bugs too, since this has not been
 503 tested very much.
 504
 505 =head1 SEE ALSO
 506
 507 xgettext.pl,
 508 TmplTokenizer.pm,
 509 msgmerge(1),
 510 Locale::PO(3),
 511 translator_doc.txt
 512
 513 http://www.saas.nsw.edu.au/koha_wiki/index.php?page=DifficultTerms
 514
 515 =cut