2 # This file is part of Koha
3 # Parts copyright 2003-2004 Paul Poulain
4 # Parts copyright 2003-2004 Jerome Vizcaino
5 # Parts copyright 2004 Ambrose Li
9 tmpl_process3.pl - Alternative version of tmpl_process.pl
10 using gettext-compatible translation files
17 use File
::Temp
qw( :POSIX );
19 use VerboseWarnings
qw( :warn :die );
21 ###############################################################################
23 use vars
qw( @in_files $in_dir $str_file $out_dir $quiet );
24 use vars qw( @excludes $exclude_regex );
25 use vars qw( $recursive_p );
26 use vars qw( $pedantic_p );
28 use vars qw( $type ); # file extension (DOS form without the dot) to match
29 use vars qw( $charset_in $charset_out );
31 ###############################################################################
33 sub find_translation ($) {
37 $key = TmplTokenizer::string_canon($key);
38 $key = TmplTokenizer::charset_convert($key, $charset_in, $charset_out);
39 $key = TmplTokenizer::quote_po($key);
41 return defined $href->{$key}
42 && !$href->{$key}->fuzzy
43 && length Locale::PO->dequote($href->{$key}->msgstr)?
44 Locale::PO->dequote($href->{$key}->msgstr): $s;
47 sub text_replace_tag ($$) {
50 # value [tag=input], meta
51 my $tag = lc($1) if $t =~ /^<(\S+)/s;
53 for my $a ('alt', 'content', 'title', 'value','label') {
55 next if $a eq 'label' && $tag ne 'optgroup';
56 next if $a eq 'content' && $tag ne 'meta';
57 next if $a eq 'value' && ($tag ne 'input'
58 || (ref $attr->{'type'} && $attr->{'type'}->[1] =~ /^(?:checkbox|hidden|radio|text)$/)); # FIXME
59 my($key, $val, $val_orig, $order) = @{$attr->{$a}}; #FIXME
61 my $s = find_translation($val);
62 if ($attr->{$a}->[1] ne $s) { #FIXME
63 $attr->{$a}->[1] = $s; # FIXME
64 $attr->{$a}->[2] = ($s =~ /"/s)? "'$s'": "\"$s\""; #FIXME
73 sprintf(' %s=%s', $_, $attr->{$_}->[2]) #FIXME
75 $attr->{$a}->[3] <=> $attr->{$b}->[3] #FIXME
84 sub text_replace (**) {
87 my $s = TmplTokenizer::next_token $h;
88 last unless defined $s;
89 my($kind, $t, $attr) = ($s->type, $s->string, $s->attributes);
90 if ($kind eq TmplTokenType::TEXT) {
91 print $output find_translation($t);
92 } elsif ($kind eq TmplTokenType::TEXT_PARAMETRIZED) {
93 my $fmt = find_translation($s->form);
94 print $output TmplTokenizer::parametrize($fmt, 1, $s, sub {
96 my($kind, $t, $attr) = ($_->type, $_->string, $_->attributes);
97 $kind == TmplTokenType::TAG && %$attr?
98 text_replace_tag($t, $attr): $t });
99 } elsif ($kind eq TmplTokenType::TAG && %$attr) {
100 print $output text_replace_tag($t, $attr);
101 } elsif ($s->has_js_data) {
102 for my $t (@{$s->js_data}) {
103 # FIXME for this whole block
105 printf $output "%s%s%s", $t->[2], find_translation $t->[3],
108 print $output $t->[1];
111 } elsif (defined $t) {
117 sub listfiles ($$$) {
118 my($dir, $type, $action) = @_;
120 if (opendir(DIR, $dir)) {
121 my @dirent = readdir DIR; # because DIR is shared when recursing
123 for my $dirent (@dirent) {
124 my $path = "$dir/$dirent";
125 if ($dirent =~ /^\./ || $dirent eq 'CVS' || $dirent eq 'RCS'
126 || (defined $exclude_regex && $dirent =~ /^(?:$exclude_regex)$/)) {
129 push @it, $path if (!defined $type || $dirent =~ /\.(?:$type)$/) || $action eq 'install';
130 } elsif (-d $path && $recursive_p) {
131 push @it, listfiles($path, $type, $action);
135 warn_normal "$dir: $!", undef;
140 ###############################################################################
142 sub mkdir_recursive ($) {
144 local($`, $&, $', $1);
145 $dir = $` if $dir ne /^\/+$/ && $dir =~ /\/+$/;
146 my ($prefix, $basename) = ($dir =~ /\/([^\/]+)$/s)? ($`, $1): ('.', $dir);
147 mkdir_recursive($prefix) if $prefix ne '.' && !-d $prefix;
149 print STDERR "Making directory $dir..." unless $quiet;
150 # creates with rwxrwxr-x permissions
151 mkdir($dir, 0775) || warn_normal "$dir: $!", undef;
155 ###############################################################################
159 my $h = $exitcode? *STDERR: *STDOUT;
161 Usage: $0 create [OPTION]
162 or: $0 update [OPTION]
163 or: $0 install [OPTION]
165 Create or update PO files from templates, or install translated templates.
167 -i, --input=SOURCE Get or update strings from SOURCE file.
168 SOURCE is a directory if -r is also specified.
169 -o, --outputdir=DIRECTORY Install translation(s) to specified DIRECTORY
170 --pedantic-warnings Issue warnings even for detected problems
171 which are likely to be harmless
172 -r, --recursive SOURCE in the -i option is a directory
173 -s, --str-file=FILE Specify FILE as the translation (po) file
174 for input (install) or output (create, update)
175 -x, --exclude=REGEXP Exclude files matching the given REGEXP
176 --help Display this help and exit
177 -q, --quiet no output to screen (except for errors)
179 The -o option is ignored for the "create" and "update" actions.
180 Try `perldoc $0 for perhaps more information.
185 ###############################################################################
187 sub usage_error
(;$) {
188 for my $msg (split(/\n/, $_[0])) {
189 print STDERR
"$msg\n";
191 print STDERR
"Try `$0 --help for more information.\n";
195 ###############################################################################
198 'input|i=s' => \
@in_files,
199 'outputdir|o=s' => \
$out_dir,
200 'recursive|r' => \
$recursive_p,
201 'str-file|s=s' => \
$str_file,
202 'exclude|x=s' => \
@excludes,
203 'quiet|q' => \
$quiet,
204 'pedantic-warnings|pedantic' => sub { $pedantic_p = 1 },
208 VerboseWarnings
::set_application_name
$0;
209 VerboseWarnings
::set_pedantic_mode
$pedantic_p;
211 # keep the buggy Locale::PO quiet if it says stupid things
212 $SIG{__WARN__
} = sub {
214 print STDERR
$s unless $s =~ /^Strange line in [^:]+: #~/s
217 my $action = shift or usage_error
('You must specify an ACTION.');
218 usage_error
('You must at least specify input and string list filenames.')
219 if !@in_files || !defined $str_file;
221 # Type match defaults to *.tmpl plus *.inc if not specified
222 $type = "tmpl|inc|xsl" if !defined($type);
224 # Check the inputs for being files or directories
225 for my $input (@in_files) {
226 usage_error
("$input: Input must be a file or directory.\n"
227 . "(Symbolic links are not supported at the moment)")
228 unless -d
$input || -f
$input;;
231 # Generates the global exclude regular expression
232 $exclude_regex = '(?:'.join('|', @excludes).')' if @excludes;
234 # Generate the list of input files if a directory is specified
235 if (-d
$in_files[0]) {
236 die "If you specify a directory as input, you must specify only it.\n"
239 # input is a directory, generates list of files to process
240 $in_dir = $in_files[0];
241 $in_dir =~ s/\/$//; # strips the trailing / if any
242 @in_files = listfiles
($in_dir, $type, $action);
244 for my $input (@in_files) {
245 die "You cannot specify input files and directories at the same time.\n"
250 # restores the string list from file
251 $href = Locale
::PO
->load_file_ashash($str_file);
253 # guess the charsets. HTML::Templates defaults to iso-8859-1
255 die "$str_file: PO file is corrupted, or not a PO file\n" unless defined $href->{'""'};
256 $charset_out = TmplTokenizer
::charset_canon
$2 if $href->{'""'}->msgstr =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/;
257 $charset_in = $charset_out;
258 warn "Charset in/out: ".$charset_out;
259 # for my $msgid (keys %$href) {
260 # if ($msgid =~ /\bcharset=(["']?)([^;\s"'\\]+)\1/) {
261 # my $candidate = TmplTokenizer::charset_canon $2;
262 # die "Conflicting charsets in msgid: $charset_in vs $candidate => $msgid\n"
263 # if defined $charset_in && $charset_in ne $candidate;
264 # $charset_in = $candidate;
269 # set our charset in to UTF-8
270 if (!defined $charset_in) {
271 $charset_in = TmplTokenizer
::charset_canon
'UTF-8';
272 warn "Warning: Can't determine original templates' charset, defaulting to $charset_in\n";
274 # set our charset out to UTF-8
275 if (!defined $charset_out) {
276 $charset_out = TmplTokenizer
::charset_canon
'UTF-8';
277 warn "Warning: Charset Out defaulting to $charset_out\n";
279 my $xgettext = './xgettext.pl'; # actual text extractor script
282 if ($action eq 'create') {
283 # updates the list. As the list is empty, every entry will be added
285 warn "Removing empty file $str_file\n";
286 unlink $str_file || die "$str_file: $!\n";
288 die "$str_file: Output file already exists\n" if -f
$str_file;
289 my($tmph1, $tmpfile1) = tmpnam
();
290 my($tmph2, $tmpfile2) = tmpnam
();
291 close $tmph2; # We just want a name
292 # Generate the temporary file that acts as <MODULE>/POTFILES.in
293 for my $input (@in_files) {
294 print $tmph1 "$input\n";
297 warn "I $charset_in O $charset_out";
298 # Generate the specified po file ($str_file)
299 $st = system ($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
300 (defined $charset_in?
('-I', $charset_in): ()),
301 (defined $charset_out?
('-O', $charset_out): ())
303 # Run msgmerge so that the pot file looks like a real pot file
304 # We need to help msgmerge a bit by pre-creating a dummy po file that has
305 # the headers and the "" msgid & msgstr. It will fill in the rest.
307 # Merge the temporary "pot file" with the specified po file ($str_file)
308 # FIXME: msgmerge(1) is a Unix dependency
309 # FIXME: need to check the return value
310 unless (-f
$str_file) {
311 local(*INPUT
, *OUTPUT
);
312 open(INPUT
, "<$tmpfile2");
313 open(OUTPUT
, ">$str_file");
321 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
323 error_normal
"Text extraction failed: $xgettext: $!\n", undef;
324 error_additional
"Will not run msgmerge\n", undef;
326 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
327 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
329 } elsif ($action eq 'update') {
330 my($tmph1, $tmpfile1) = tmpnam
();
331 my($tmph2, $tmpfile2) = tmpnam
();
332 close $tmph2; # We just want a name
333 # Generate the temporary file that acts as <MODULE>/POTFILES.in
334 for my $input (@in_files) {
335 print $tmph1 "$input\n";
338 # Generate the temporary file that acts as <MODULE>/<LANG>.pot
339 $st = system($xgettext, '-s', '-f', $tmpfile1, '-o', $tmpfile2,
341 (defined $charset_in?
('-I', $charset_in): ()),
342 (defined $charset_out?
('-O', $charset_out): ()));
344 # Merge the temporary "pot file" with the specified po file ($str_file)
345 # FIXME: msgmerge(1) is a Unix dependency
346 # FIXME: need to check the return value
347 $st = system('msgmerge', '-U', '-s', $str_file, $tmpfile2);
349 error_normal
"Text extraction failed: $xgettext: $!\n", undef;
350 error_additional
"Will not run msgmerge\n", undef;
352 # unlink $tmpfile1 || warn_normal "$tmpfile1: unlink failed: $!\n", undef;
353 # unlink $tmpfile2 || warn_normal "$tmpfile2: unlink failed: $!\n", undef;
355 } elsif ($action eq 'install') {
356 if(!defined($out_dir)) {
357 usage_error
("You must specify an output directory when using the install method.");
360 if ($in_dir eq $out_dir) {
361 warn "You must specify a different input and output directory.\n";
365 # Make sure the output directory exists
366 # (It will auto-create it, but for compatibility we should not)
367 -d
$out_dir || die "$out_dir: The directory does not exist\n";
369 # Try to open the file, because Locale::PO doesn't check :-/
370 open(INPUT
, "<$str_file") || die "$str_file: $!\n";
373 # creates the new tmpl file using the new translation
374 for my $input (@in_files) {
375 die "Assertion failed"
376 unless substr($input, 0, length($in_dir) + 1) eq "$in_dir/";
377 # print "$input / $type\n";
378 if (!defined $type || $input =~ /\.(?:$type)$/) {
379 my $h = TmplTokenizer
->new( $input );
380 $h->set_allow_cformat( 1 );
381 VerboseWarnings
::set_input_file_name
$input;
383 my $target = $out_dir . substr($input, length($in_dir));
384 my $targetdir = $` if $target =~ /[^\/]+$/s;
385 mkdir_recursive($targetdir) unless -d $targetdir;
386 print STDERR "Creating $target...\n" unless $quiet;
387 open( OUTPUT, ">$target" ) || die "$target: $!\n";
388 text_replace( $h, *OUTPUT );
391 # just copying the file
392 my $target = $out_dir . substr($input, length($in_dir));
393 my $targetdir = $` if $target =~ /[^\/]+$/s
;
394 mkdir_recursive
($targetdir) unless -d
$targetdir;
395 system("cp -f $input $target");
396 print STDERR
"Copying $input...\n" unless $quiet;
401 usage_error
('Unknown action specified.');
405 printf "The %s seems to be successful.\n", $action unless $quiet;
407 printf "%s FAILED.\n", "\u$action" unless $quiet;
411 ###############################################################################
415 ./tmpl_process3.pl [ I<tmpl_process.pl options> ]
419 This is an alternative version of the tmpl_process.pl script,
420 using standard gettext-style PO files. While there still might
421 be changes made to the way it extracts strings, at this moment
422 it should be stable enough for general use; it is already being
423 used for the Chinese and Polish translations.
425 Currently, the create, update, and install actions have all been
426 reimplemented and seem to work.
434 Translation files in standard Uniforum PO format.
435 All standard tools including all gettext tools,
436 plus PO file editors like kbabel(1) etc.
441 Minor changes in whitespace in source templates
442 do not generally require strings to be re-translated.
446 Able to handle <TMPL_VAR> variables in the templates;
447 <TMPL_VAR> variables are usually extracted in proper context,
448 represented by a short %s placeholder.
452 Able to handle text input and radio button INPUT elements
453 in the templates; these INPUT elements are also usually
454 extracted in proper context,
455 represented by a short %S or %p placeholder.
459 Automatic comments in the generated PO files to provide
460 even more context (line numbers, and the names and types
465 The %I<n>$s (or %I<n>$p, etc.) notation can be used
466 for change the ordering of the variables,
467 if such a reordering is required for correct translation.
471 If a particular <TMPL_VAR> should not appear in the
472 translation, it can be suppressed with the %0.0s notation.
476 Using the PO format also means translators can add their
477 own comments in the translation files, if necessary.
481 Create, update, and install actions are all based on the
482 same scanner module. This ensures that update and install
483 have the same idea of what is a translatable string;
484 attribute names in tags, for example, will not be
485 accidentally translated.
491 Anchors are represented by an <AI<n>> notation.
492 The meaning of this non-standard notation might not be obvious.
494 The create action calls xgettext.pl to do the actual work;
495 the update action calls xgettext.pl and msgmerge(1) to do the
500 xgettext.pl must be present in the current directory; the
501 msgmerge(1) command must also be present in the search path.
502 The script currently does not check carefully whether these
503 dependent commands are present.
505 Locale::PO(3) has a lot of bugs. It can neither parse nor
506 generate GNU PO files properly; a couple of workarounds have
507 been written in TmplTokenizer and more is likely to be needed
508 (e.g., to get rid of the "Strange line" warning for #~).
510 This script may not work in Windows.
512 There are probably some other bugs too, since this has not been
523 http://www.saas.nsw.edu.au/koha_wiki/index.php?page=DifficultTerms