7 use VerboseWarnings
qw( pedantic_p error_normal warn_normal warn_pedantic );
10 use vars
qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
12 ###############################################################################
16 TmplTokenizer.pm - Simple-minded wrapper class for TTParser
20 A wrapper for the functionality found in TTParser to allow an easier transition to Template Toolkit
24 ###############################################################################
30 use vars
qw( $pedantic_attribute_error_in_nonpedantic_mode_p );
31 use vars qw( $pedantic_tmpl_var_use_in_nonpedantic_mode_p );
32 use vars qw( $pedantic_error_markup_in_pcdata_p );
34 ###############################################################################
37 use vars qw( $re_xsl $re_end_entity $re_tmpl_var);
39 $re_tmpl_var = q{\[%\s*[get|set|default]?\s*[\w\.]+\s*[|.*?]?\s*%\]};
40 $re_xsl = q{<\/?(?:xsl:)(?:[\s\-a-zA-Z0-9"'\/\.\[\]\@\(\):=,$]+)\/?>};
41 $re_end_entity = '(?:;|$|(?=\s))'; # semicolon or before-whitespace
43 # End of the hideous stuff
45 use vars
qw( $serial );
47 ###############################################################################
49 sub FATAL_P () {'fatal-p'}
50 sub SYNTAXERROR_P () {'syntaxerror-p'}
52 sub FILENAME () {'input'}
53 #sub HANDLE () {'handle'}
55 #sub READAHEAD () {'readahead'}
56 sub LINENUM_START () {'lc_0'}
58 sub CDATA_MODE_P () {'cdata-mode-p'}
59 sub CDATA_CLOSE () {'cdata-close'}
60 #sub PCDATA_MODE_P () {'pcdata-mode-p'} # additional submode for CDATA
61 sub JS_MODE_P () {'js-mode-p'} # cdata-mode-p must also be true
63 sub ALLOW_CFORMAT_P () {'allow-cformat-p'}
68 #open my $handle,$filename or die "can't open $filename";
69 my $parser = C4::TTParser->new;
70 $parser->build_tokens( $filename );
72 filename => $filename,
79 ###############################################################################
85 return $this->{filename};
90 return $this->{+FATAL_P};
93 # work around, currently not implemented
96 # return $this->{+SYNTAXERROR_P};
102 return $this->{+JS_MODE_P};
105 sub allow_cformat_p {
107 return $this->{+ALLOW_CFORMAT_P};
114 $this->{+FATAL_P} = $_[0];
120 $this->{+JS_MODE_P} = $_[0];
124 #used in xgettext, tmpl_process3
125 sub set_allow_cformat {
127 $this->{+ALLOW_CFORMAT_P} = $_[0];
131 ###############################################################################
133 use vars qw( $js_EscapeSequence );
135 # Perl quoting is really screwed up, but this common subexp is way too long
136 $js_EscapeSequence = q{\\\\(?:['"\\\\bfnrt]|[^0-7xu]|[0-3]?[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4})};
138 sub parenleft () { '(' }
139 sub parenright () { ')' }
145 if ($s0 =~ /^\s+/s) { # whitespace
148 } elsif ($s0 =~ /^\/\/[^\r\n]*(?:[\r\n]|$)/s) { # C++-style comment
151 } elsif ($s0 =~ /^\/\*(?:(?!\*\/).)*\*\//s) { # C-style comment
154 # Keyword or identifier, ECMA-262 p.13 (section 7.5)
155 } elsif ($s0 =~ /^[A-Z_\$][A-Z\d_\$]*/is) { # IdentifierName
158 # Punctuator, ECMA-262 p.13 (section 7.6)
159 } elsif ($s0 =~ /^(?:[\(\){}\[\];]|>>>=|<<=|>>=|[-\+\*\/\&\|\^\%]=|>>>|<<|>>|--|\+\+|\|\||\&\&|==|<=|>=|!=|[=><,!~\?:\.\-\+\*\/\&\|\^\%])/s) {
162 # DecimalLiteral, ECMA-262 p.14 (section 7.7.3); note: bug in the spec
163 } elsif ($s0 =~ /^(?:0|[1-9]\d+(?:\.\d*(?:[eE][-\+]?\d+)?)?)/s) {
166 # HexIntegerLiteral, ECMA-262 p.15 (section 7.7.3)
167 } elsif ($s0 =~ /^0[xX][\da-fA-F]+/s) {
170 # OctalIntegerLiteral, ECMA-262 p.15 (section 7.7.3)
171 } elsif ($s0 =~ /^0[\da-fA-F]+/s) {
174 # StringLiteral, ECMA-262 p.17 (section 7.7.4)
175 # XXX SourceCharacter doesn't seem to be defined (?)
176 } elsif ($s0 =~ /^(?:"(?:(?!["\\\r\n]).|$js_EscapeSequence)*"|'(?:(?!['\\\r\n]).|$js_EscapeSequence)*')/os) {
179 } elsif ($s0 =~ /^./) { # UNKNOWN TOKEN !!!
187 sub STATE_UNDERSCORE () { 1 }
188 sub STATE_PARENLEFT () { 2 }
189 sub STATE_STRING_LITERAL () { 3 }
191 # XXX This is a crazy hack. I don't want to write an ECMAScript parser.
192 # XXX A scanner is one thing; a parser another thing.
193 sub _identify_js_translatables (@) {
196 # We mark a JavaScript translatable string as in C, i.e., _("literal")
197 # For simplicity, we ONLY look for "_" "(" StringLiteral ")"
198 for (my $i = 0, my $state = 0, my($j, $q, $s); $i <= $#input; $i += 1) {
200 my $reset_state_p = 0;
201 push @output, [0, $input[$i]];
202 if ($input[$i] !~ /\S/s) {
204 } elsif ($state == 0) {
205 $state = STATE_UNDERSCORE if $input[$i] eq '_';
206 } elsif ($state == STATE_UNDERSCORE) {
207 $state = $input[$i] eq parenleft ? STATE_PARENLEFT : 0;
208 } elsif ($state == STATE_PARENLEFT) {
209 if ($input[$i] =~ /^(['"])(.*)\1$/s) {
210 ($state, $j, $q, $s) = (STATE_STRING_LITERAL, $#output, $1, $2);
214 } elsif ($state == STATE_STRING_LITERAL) {
215 if ($input[$i] eq parenright) {
216 $output[$j] = [1, $output[$j]->[1], $q, $s];
220 die "identify_js_translatables internal error: Unknown state $state"
224 # warn Dumper \@output;
228 ###############################################################################
230 sub string_canon ($) {
232 # Fold all whitespace into single blanks
238 # safer version used internally, preserves new lines
239 sub string_canon_safe ($) {
241 # fold tabs and spaces into single spaces
242 $s =~ s/[\ \t]+/ /gs;
253 sub _formalize_string_cformat{
255 return _quote_cformat( string_canon_safe $s );
260 if( $t->type == C4::TmplTokenType::DIRECTIVE ){
262 } elsif( $t->type == C4::TmplTokenType::TEXT ){
263 return _formalize_string_cformat( $t->string );
264 } elsif( $t->type == C4::TmplTokenType::TAG ){
265 if( $t->string =~ m/^a\b/is ){
267 } elsif( $t->string =~ m/^input\b/is ){
268 if( lc $t->attributes->{'type'}->[1] eq 'text' ){
274 return _quote_cformat $t->string;
277 return _quote_cformat $t->string;
281 # internal parametization, used within next_token
282 # method that takes in an array of TEXT and DIRECTIVE tokens (DIRECTIVEs must be GET) and return a C4::TmplTokenType::TEXT_PARAMETRIZED
283 sub _parametrize_internal{
287 # for my $item (@parts){
288 # if( $item->type == C4::TmplTokenType::TEXT ){
289 # $s .= $item->string;
291 # #must be a variable directive
295 my $s = join( "", map { _formalize $_ } @parts );
296 # should both the string and form be $s? maybe only the later? posibly the former....
297 # used line number from first token, should suffice
298 my $t = C4::TmplToken->new( $s, C4::TmplTokenType::TEXT_PARAMETRIZED, $parts[0]->line_number, $this->filename );
299 $t->set_children(@parts);
307 # warn "in next_token";
308 # parts that make up a text_parametrized (future children of the token)
311 $next = $self->{_parser}->next_token;
314 return $self->_parametrize_internal(@parts);
320 # if cformat mode is off, dont bother parametrizing, just return them as they come
321 return $next unless $self->allow_cformat_p;
322 if( $next->type == C4::TmplTokenType::TEXT ){
325 # elsif( $next->type == C4::TmplTokenType::DIRECTIVE && $next->string =~ m/\[%\s*\w+\s*%\]/ ){
326 elsif( $next->type == C4::TmplTokenType::DIRECTIVE ){
329 elsif ( $next->type == C4::TmplTokenType::CDATA){
330 $self->_set_js_mode(1);
331 my $s0 = $next->string;
335 if ($s0 =~ /^(\s*\[%\s*)(.*)(\s%=]\s*)$/s) {
340 push @head, _split_js $s0;
341 $next->set_js_data(_identify_js_translatables(@head, @tail) );
342 return $next unless @parts;
343 $self->{_parser}->unshift_token($next);
344 return $self->_parametrize_internal(@parts);
347 # if there is nothing in parts, return this token
348 return $next unless @parts;
350 # OTHERWISE, put this token back and return the parametrized string of @parts
351 $self->{_parser}->unshift_token($next);
352 return $self->_parametrize_internal(@parts);
358 ###############################################################################
360 # function taken from old version
361 # used by tmpl_process3
362 sub parametrize ($$$$) {
363 my($fmt_0, $cformat_p, $t, $f) = @_;
366 my @params = $t->parameters_and_fields;
367 for (my $n = 0, my $fmt = $fmt_0; length $fmt;) {
368 if ($fmt =~ /^[^%]+/) {
371 } elsif ($fmt =~ /^%%/) {
374 } elsif ($fmt =~ /^%(?:(\d+)\$)?(?:(\d+)(?:\.(\d+))?)?s/s) {
376 my($i, $width, $prec) = ((defined $1? $1: $n), $2, $3);
378 if (defined $width && defined $prec && !$width && !$prec) {
380 } elsif (defined $params[$i - 1]) {
381 my $param = $params[$i - 1];
382 warn_normal "$fmt_0: $&: Expected a TMPL_VAR, but found a "
383 . $param->type->to_string . "\n", undef
384 if $param->type != C4::TmplTokenType::DIRECTIVE;
385 warn_normal "$fmt_0: $&: Unsupported "
386 . "field width or precision\n", undef
387 if defined $width || defined $prec;
388 warn_normal "$fmt_0: $&: Parameter $i not known", undef
389 unless defined $param;
390 $it .= defined $f? &$f( $param ): $param->string;
392 } elsif ($fmt =~ /^%(?:(\d+)\$)?(?:(\d+)(?:\.(\d+))?)?([pS])/s) {
394 my($i, $width, $prec, $conv) = ((defined $1? $1: $n), $2, $3, $4);
397 my $param = $params[$i - 1];
398 if (!defined $param) {
399 warn_normal "$fmt_0: $&: Parameter $i not known", undef;
401 if ($param->type == C4::TmplTokenType::TAG
402 && $param->string =~ /^<input\b/is) {
403 my $type = defined $param->attributes?
404 lc($param->attributes->{'type'}->[1]): undef;
406 warn_normal "$fmt_0: $&: Expected type=text, "
407 . "but found type=$type", undef
408 unless $type eq 'text';
409 } elsif ($conv eq 'p') {
410 warn_normal "$fmt_0: $&: Expected type=radio, "
411 . "but found type=$type", undef
412 unless $type eq 'radio';
415 warn_normal "$&: Expected an INPUT, but found a "
416 . $param->type->to_string . "\n", undef
418 warn_normal "$fmt_0: $&: Unsupported "
419 . "field width or precision\n", undef
420 if defined $width || defined $prec;
421 $it .= defined $f? &$f( $param ): $param->string;
423 } elsif ($fmt =~ /^%[^%a-zA-Z]*[a-zA-Z]/) {
426 die "$&: Unknown or unsupported format specification\n"; #XXX
428 die "$&: Completely confused parametrizing -- msgid: $fmt_0\n";#XXX
432 my @anchors = $t->anchors;
433 for (my $n = 0, my $fmt = $it, $it = ''; length $fmt;) {
434 if ($fmt =~ /^(?:(?!<a\d+>).)+/is) {
437 } elsif ($fmt =~ /^<a(\d+)>/is) {
441 my $anchor = $anchors[$i - 1];
442 warn_normal "$&: Anchor $1 not found for msgid \"$fmt_0\"", undef #FIXME
443 unless defined $anchor;
444 $it .= $anchor->string;
446 die "Completely confused decoding anchors: $fmt\n";#XXX
453 # Other simple functions (These are not methods)
457 return $s =~ /^(?:\s|\ $re_end_entity|$re_tmpl_var|$re_xsl)*$/osi;
464 $s =~ s/^(\s|\ $re_end_entity)+//os; my $l1 = $l0 - length $s;
465 $s =~ s/(\s|\ $re_end_entity)+$//os; my $l2 = $l0 - $l1 - length $s;
466 return wantarray? (substr($s0, 0, $l1), $s, substr($s0, $l0 - $l2)): $s;
471 # Locale::PO->quote is buggy, it doesn't quote newlines :-/
472 $s =~ s/([\\"])/\\$1/gs;
474 #$s =~ s/[\177-\377]/ sprintf("\\%03o", ord($&)) /egs;
478 sub charset_canon ($) {
480 $charset = uc($charset);
481 $charset = "$1-$2" if $charset =~ /^(ISO|UTF)(\d.*)/i;
482 $charset = 'Big5' if $charset eq 'BIG5'; # "Big5" must be in mixed case
486 use vars qw( @latin1_utf8 );
488 "\302\200", "\302\201", "\302\202", "\302\203", "\302\204", "\302\205",
489 "\302\206", "\302\207", "\302\210", "\302\211", "\302\212", "\302\213",
490 "\302\214", "\302\215", undef, undef, "\302\220", "\302\221",
491 "\302\222", "\302\223", "\302\224", "\302\225", "\302\226", "\302\227",
492 "\302\230", "\302\231", "\302\232", "\302\233", "\302\234", "\302\235",
493 "\302\236", "\302\237", "\302\240", "\302\241", "\302\242", "\302\243",
494 "\302\244", "\302\245", "\302\246", "\302\247", "\302\250", "\302\251",
495 "\302\252", "\302\253", "\302\254", "\302\255", "\302\256", "\302\257",
496 "\302\260", "\302\261", "\302\262", "\302\263", "\302\264", "\302\265",
497 "\302\266", "\302\267", "\302\270", "\302\271", "\302\272", "\302\273",
498 "\302\274", "\302\275", "\302\276", "\302\277", "\303\200", "\303\201",
499 "\303\202", "\303\203", "\303\204", "\303\205", "\303\206", "\303\207",
500 "\303\210", "\303\211", "\303\212", "\303\213", "\303\214", "\303\215",
501 "\303\216", "\303\217", "\303\220", "\303\221", "\303\222", "\303\223",
502 "\303\224", "\303\225", "\303\226", "\303\227", "\303\230", "\303\231",
503 "\303\232", "\303\233", "\303\234", "\303\235", "\303\236", "\303\237",
504 "\303\240", "\303\241", "\303\242", "\303\243", "\303\244", "\303\245",
505 "\303\246", "\303\247", "\303\250", "\303\251", "\303\252", "\303\253",
506 "\303\254", "\303\255", "\303\256", "\303\257", "\303\260", "\303\261",
507 "\303\262", "\303\263", "\303\264", "\303\265", "\303\266", "\303\267",
508 "\303\270", "\303\271", "\303\272", "\303\273", "\303\274", "\303\275",
509 "\303\276", "\303\277" );
511 sub charset_convert ($$$) {
512 my($s, $charset_in, $charset_out) = @_;
513 if ($s !~ /[\200-\377]/s) { # FIXME: don't worry about iso2022 for now
515 } elsif ($charset_in eq 'ISO-8859-1' && $charset_out eq 'UTF-8') {
516 $s =~ s/[\200-\377]/ $latin1_utf8[ord($&) - 128] /egs;
517 } elsif ($charset_in ne $charset_out) {
518 VerboseWarnings::warn_normal "conversion from $charset_in to $charset_out is not supported\n", undef;
523 ###############################################################################
527 In addition to the basic scanning, this class will also perform
534 Emulation of c-format strings (see below)
538 Display of warnings for certain things that affects either the
539 ability of this class to yield correct output, or things that
540 are known to cause the original template to cause trouble.
544 Automatic correction of some of the things warned about
545 (e.g., SGML "closed start tag" notation).
549 =head2 c-format strings emulation
551 Because English word order is not universal, a simple extraction
552 of translatable strings may yield some strings like "Accounts for"
553 or ambiguous strings like "in". This makes the resulting strings
554 difficult to translate, but does not affect all languages alike.
555 For example, Chinese (with a somewhat different word order) would
556 be hit harder, but French would be relatively unaffected.
558 To overcome this problem, the scanner can be configured to detect
559 patterns with <TMPL_VAR> directives (as well as certain HTML tags),
560 and try to construct a larger pattern that will appear in the PO
561 file as c-format strings with %s placeholders. This additional
562 step allows the translator to deal with cases where word order
563 is different (replacing %s with %1$s, %2$s, etc.), or when certain
564 words will require certain inflectional suffixes in sentences.
566 Because this is an incompatible change, this mode must be explicitly
567 turned on using the set_allow_cformat(1) method call.
569 =head2 The flag characters
571 The character % is followed by zero or more of the following flags:
577 The value comes from HTML <INPUT> elements.
578 This abuse of the flag character is somewhat reasonable,
579 since TMPL_VAR and INPUT are both variables, but of different kinds.
583 =head2 The field width and precision
585 An optional 0.0 can be specified for %s to specify
586 that the <TMPL_VAR> should be suppressed.
588 =head2 The conversion specifier
594 Specifies any input field that is neither text nor hidden
595 (which currently mean radio buttons).
596 The p conversion specifier is chosen because this does not
597 evoke any certain sensible data type.
601 Specifies a text input field (<INPUT TYPE=TEXT>).
602 This use of the S conversion specifier is somewhat reasonable,
603 since text input fields contain values of undeterminable type,
604 which can be treated as strings.
608 Specifies a <TMPL_VAR>.
609 This use of the o conversion specifier is somewhat reasonable,
610 since <TMPL_VAR> denotes values of undeterminable type, which
611 can be treated as strings.
617 There is no code to save the tag name anywhere in the scanned token.
619 The use of <AI<i>> to stand for the I<i>th anchor
620 is not very well thought out.
621 Some abuse of c-format specifies might have been more appropriate.
625 This tokenizer is mostly based
626 on Ambrose's hideous Perl script known as subst.pl.