3 # Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 # Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 # Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 # Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 # Copyright (C) 2006 - 2008 by Daniel Stenberg
13 # See apps/language.c (TODO: Use common include for both)
14 # Cookie and binary version for the binary lang file
15 my $LANGUAGE_COOKIE = 0x1a;
16 my $LANGUAGE_VERSION = 0x05;
17 my $LANGUAGE_FLAG_RTL = 0x01;
19 # A note for future users and readers: The original v1 language system allowed
20 # the build to create and use a different language than english built-in. We
21 # removed that feature from our build-system, but the build scripts still had
22 # the ability. But, starting now, this ability is no longer provided since I
23 # figured it was boring and unnecessary to write support for now since we
24 # don't use it anymore.
32 Usage: genlang [options] <langv2 file>
35 Make the tool create a [prefix].c and [prefix].h file.
38 Make the tool create a binary language (.lng) file named [outfile].
39 The use of this option requires that you also use -e, -t and -i.
42 Update language file. Given the translated file and the most recent english
43 file, you\'ll get an updated version sent to stdout. Suitable action to do
44 when you intend to update a translation.
46 -e=<english lang file>
47 Point out the english (original source) file, to use that as master
48 language template. Used in combination with -b, -u or -s.
51 Sort the Update language file in the same order as the strings in the
55 Specify which target you want the translations/phrases for. Required when
58 The target can in fact be specified as numerous different strings,
59 separated with colons. This will make genlang to use all the specified
60 strings when searching for a matching phrase.
63 The target id number, needed for -b.
66 Voice mode output. Outputs all id: and voice: lines for the given target!
69 Enables verbose (debug) output.
77 # 1) scan the english file, keep the whole <phrase> for each phrase.
78 # 2) read the translated file, for each end of phrase, compare:
79 # A) all source strings, if there's any change there should be a comment about
83 # 3) output the phrase with the comments from above
84 # 4) check which phrases that the translated version didn't have, and spit out
85 # the english version of those
96 my $check = ($binary?
1:0) + ($prefix?
1:0) + ($update?
1:0) + ($voiceout?
1:0) + ($sortfile?
1:0);
99 print STDERR
"Please use only one of -p, -u, -o, -b and -s\n";
103 print STDERR
"Please use at least one of -p, -u, -o, -b and -s\n";
108 if(($binary || $update || $voiceout || $sortfile) && !$english) {
109 print STDERR
"Please use -e too when you use -b, -o, -u or -s\n";
114 if($binary && !$target_id) {
115 print STDERR
"Please specify a target id number (with -i)!\n";
120 if(!$target && !$update && !$sortfile) {
121 print STDERR
"Please specify a target (with -t)!\n";
126 my %id; # string to num hash
127 my @idnum; # num to string array
129 my %allphrases; # For sorting - an array of the <phrase> elements
130 my %source; # id string to source phrase hash
131 my %dest; # id string to dest phrase hash
132 my %voice; # id string to voice phrase hash
134 my $input = $ARGV[0];
147 my ($string, $pattern)=@_;
149 $pattern =~ s/\*/.*/g;
150 $pattern =~ s/\?/./g;
152 return ($string =~ /^$pattern\z/);
161 my ($full, $n, $v)=@_;
167 my ($full, $n, $v)=@_;
173 my ($full, $n, $v)=@_;
178 my ($debug, $strref, $full, $n, $v)=@_;
180 my @all= split(" *, *", $n);
183 # print "TEST ($debug) $target for $test\n";
184 for my $part (split(":", $target)) {
185 if(match
($part, $test)) {
187 # print "MATCH: $test => $v\n";
197 parsetarget
("src", \
$src, @_);
202 parsetarget
("dest", \
$dest, @_);
207 parsetarget
("voice", \
$voice, @_);
213 # For the cases where the english file needs to be scanned/read, we do
214 # it before we read the translated file. For -b it isn't necessary, but for
215 # -u it is convenient.
217 my $idnum=0; # start with a true number
218 my $vidnum=0x8000; # first voice id
219 open(ENG
, "<$english") || die "Error: can't open $english";
229 # get rid of DOS newlines
232 if($_ =~ /^ *\<phrase\>/) {
233 # this is the start of a phrase
235 elsif($_ =~ /^ *\<\/phrase\
>/) {
237 # if id is something, when we count and store this phrase
239 # voice-only entries get a difference range
240 if($id =~ /^VOICE_/) {
241 # Assign an ID number to this entry
246 # Assign an ID number to this entry
249 # print STDERR "DEST: bumped idnum to $idnum\n";
252 # this is the end of a phrase, add it to the english hash
253 $english{$id}=join("", @phrase);
259 # gather everything related to this phrase
261 if($_ =~ /^ *\<dest\>/i) {
265 elsif($withindest && ($_ =~ /^ *\<\/dest\
>/i
)) {
268 if($update || ($deststr && ($deststr !~ /^none\z/i))) {
269 # we unconditionally always use all IDs when the "update"
272 $user = $user2num{$maybeuser};
273 # print "DEST: use this id $id\n";
276 # print "skip $maybeid for $name\n";
279 elsif($withindest && ($_ =~ / *([^:]+): *(.*)/)) {
280 my ($name, $val)=($1, $2);
281 $dest=""; # in case it is left untouched for when the
282 # model name isn't "our"
283 dest
($_, $name, $val);
286 # Store the current dest string. If this target matches
287 # multiple strings, it will get updated several times.
293 if($_ =~ /^ *id: ([^ \t\n]+)/i) {
295 $sortorder{$maybeid}=$numphrases++;
297 if($_ =~ /^ *user: ([^ \t\n]+)/i) {
304 # a function that compares the english phrase with the translated one.
305 # compare source strings and desc
307 # Then output the updated version!
309 my ($idstr, $engref, $locref)=@_;
311 my ($esource, $lsource);
314 for my $l (@
$engref) {
319 if($l =~ /^ *desc: (.*)/) {
322 elsif($l =~ / *\<source\>/i) {
326 if($l =~ / *\<\/source\
>/i
) {
337 for my $l (@
$locref) {
338 if($l =~ /^ *desc: (.*)/) {
340 if(trim
($edesc) ne trim
($ldesc)) {
341 $l = "### The 'desc' field differs from the english!\n### the previously used desc is commented below:\n### desc: $ldesc\n desc: $edesc\n";
345 elsif($l =~ / *\<source\>/i) {
350 if($l =~ / *\<\/source\
>/i
) {
353 if(trim
($esource) ne trim
($lsource)) {
354 print "### The <source> section differs from the english!\n",
355 "### the previously used one is commented below:\n";
356 for(split("\n", $lsource)) {
364 undef @show; # start over
381 my $idcount; # counter for lang ID numbers
382 my $voiceid=0x8000; # counter for voice-only ID numbers
385 # Now start the scanning of the selected language string
388 open(LANG
, "<$input") || die "Error: couldn't read language file named $input\n";
397 # get rid of DOS newlines
400 if($_ =~ /^( *\#|[ \t\n\r]*\z)/) {
401 # comment or empty line - output it if it's part of the header
402 if ($header and ($update || $sortfile)) {
415 # this is an XML-lookalike tag
416 if (/^(<|[^\"<]+<)([^>]*)>/) {
418 # print "P: $part\n";
421 # this was a closing tag
423 if($part eq "/phrase") {
426 my $idstr = $phrase{'id'};
429 if($binary && !$english{$idstr}) {
430 # $idstr doesn't exist for english, skip it\n";
432 elsif($dest =~ /^none\z/i) {
433 # "none" as dest (without quotes) means that this entire
434 # phrase is to be ignored
437 $allphrases{$idstr}=join('',@phrase);
440 # we don't do the fully detailed analysis when we "update"
441 # since we don't do it for a particular target etc
443 # allow the keyword 'deprecated' to be used on dest and
444 # voice strings to mark that as deprecated. It will then
445 # be replaced with "".
447 $dest =~ s/^deprecate(|d)\z/\"\"/i;
448 $voice =~ s/^deprecate(|d)\z/\"\"/i;
450 # basic syntax error alerts, if there are no quotes we
451 # will assume an empty string was intended
453 print STDERR
"$input:$line:1: warning: dest before line lacks quotes ($dest)!\n";
457 print STDERR
"$input:$line:1: warning: source before line lacks quotes ($src)!\n";
460 if($voice !~ /^\"/ and $voice !~ /^none\z/i) {
461 print STDERR
"$input:$line:1: warning: voice before line lacks quotes ($voice)!\n";
464 if($dest eq '""' && $phrase{'desc'} !~ /deprecated/i && $idstr !~ /^VOICE/) {
465 print STDERR
"$input:$line:1: warning: empty dest before line in non-deprecated phrase!\n";
468 my $user = $user2num{trim
($phrase{'user'})};
470 print STDERR
"$input:$line:1: warning: unknown user!\n";
474 # Use the ID name to figure out which id number range we
475 # should use for this phrase. Voice-only strings are
478 if($idstr =~ /^VOICE/) {
485 $id{$idstr} = $idnum;
486 $idnum[$idnum]=$idstr;
488 $source{$idstr}=$src;
490 $voice{$idstr}=$voice;
493 print "id: $phrase{id} ($idnum)\n";
494 print "source: $src\n";
495 print "dest: $dest\n";
496 print "voice: $voice\n";
497 print "user: $user\n";
508 my $e = $english{$idstr};
511 # compare original english with this!
512 my @eng = split("\n", $english{$idstr});
514 compare
($idstr, \
@eng, \
@phrase);
516 $english{$idstr}=""; # clear it
519 print "### $idstr: The phrase is not used. Skipped\n";
523 elsif($part eq "/options") {
524 # closing the options
525 if ($options{'rtl'}) {
526 $langoptions |= $LANGUAGE_FLAG_RTL;
528 } # end of </options>
532 # starts with a slash, this _ends_ this section
533 $m = pop @m; # get back old value, the previous level's tag
537 # This is an opening (sub) tag
539 push @m, $m; # store old value
544 if(/^ *([^:]+): *(.*)/) {
545 my ($name, $val)=($1, $2);
546 &$m($_, $name, $val);
556 "### This phrase below was not present in the translated file\n",
565 for(sort { $sortorder{$a} <=> $sortorder{$b} } keys %allphrases) {
566 print $allphrases{$_};
571 # We create a .c and .h file
573 open(HFILE
, ">$prefix/lang.h") ||
574 die "Error: couldn't create file $prefix/lang.h\n";
575 open(CFILE
, ">$prefix/lang_core.c") ||
576 die "Error: couldn't create file $prefix/lang_core.c\n";
578 # get header file name
579 $headername = "$prefix/lang.h";
580 $headername =~ s/(.*\/)*//;
583 /* This file was automatically generated using genlang */
585 * The str() macro/functions is how to access strings that might be
586 * translated. Use it like str(MACRO) and expect a string to be
589 #define str(x) language_strings[x]
591 /* this is the array for holding the string pointers.
592 It will be initialized at runtime. */
593 extern unsigned char *language_strings[];
594 /* this contains the concatenation of all strings, separated by \\0 chars */
595 extern const unsigned char language_builtin[];
597 /* The enum below contains all available strings */
603 /* This file was automaticly generated using genlang, the strings come
606 #include "$headername"
608 unsigned char *language_strings[LANG_LAST_INDEX_IN_ARRAY];
609 const unsigned char language_builtin[] =
613 # Output the ID names for the enum in the header file
615 for $i (1 .. $idcount) {
616 my $name=$idnum[$i - 1]; # get the ID name
618 $name =~ s/\"//g; # cut off the quotes
620 printf HFILE
(" %s, /* %d */\n", $name, $i-1);
623 # Output separation marker for last string ID and the upcoming voice IDs
626 LANG_LAST_INDEX_IN_ARRAY, /* this is not a string, this is a marker */
627 /* --- below this follows voice-only strings --- */
628 VOICEONLY_DELIMITER = 0x8000,
632 # Output the ID names for the enum in the header file
633 for $i (0x8000 .. ($voiceid-1)) {
634 my $name=$idnum[$i]; # get the ID name
636 $name =~ s/\"//g; # cut off the quotes
638 printf HFILE
(" %s,\n", $name);
642 print HFILE
"\n};\n/* end of generated enum list */\n";
644 # Output the target phrases for the source file
645 for $i (1 .. $idcount) {
646 my $name=$idnum[$i - 1]; # get the ID
647 my $dest = $dest{$name}; # get the destination phrase
649 $dest =~ s
:\"$:\\0\":; # insert a \0 before the second quote
652 # this is just to be on the safe side
656 printf CFILE
(" %s\n", $dest);
659 # Output end of string chunk
662 /* end of generated string list */
668 } # end of the c/h file generation
670 # Creation of a binary lang file was requested
672 # We must first scan the english file to get the correct order of the id
673 # numbers used there, as that is what sets the id order for all language
674 # files. The english file is scanned before the translated file was
677 open(OUTF
, ">$binary") or die "Error: Can't create $binary";
679 printf OUTF
("%c%c%c%c", $LANGUAGE_COOKIE, $LANGUAGE_VERSION, $target_id,
680 $langoptions); # magic lang file header
682 # loop over the target phrases
683 for $i (1 .. $idcount) {
684 my $name=$idnum[$i - 1]; # get the ID
685 my $dest = $dest{$name}; # get the destination phrase
688 $dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
690 # Now, make sure we get the number from the english sort order:
691 $idnum = $idmap{$name};
693 printf OUTF
("%c%c%s\x00", ($idnum>>8), ($idnum&0xff), $dest);
698 # voice output requested, display id: and voice: strings in a v1-like
703 # This loops over the strings in the translated language file order
704 my @ids = ((0 .. ($idcount-1)));
705 push @ids, (0x8000 .. ($voiceid-1));
712 my $name=$idnum[$i]; # get the ID
713 my $dest = $voice{$name}; # get the destination voice string
716 $dest =~ s/^\"(.*)\"\s*$/$1/g; # cut off quotes
718 # Now, make sure we get the number from the english sort order:
719 $idnum = $idmap{$name};
724 #print "Input index $i output index $idnum\n";
727 # not used, mark it so
737 if(($o < 0) || !length($o)) {
738 print "#$i\nid: NOT_USED_$i\nvoice: \"\"\n";
742 my $name=$idnum[$o]; # get the ID
743 my $dest = $voice{$name}; # get the destination voice string
745 print "#$i ($o)\nid: $name\nvoice: $dest\n";
752 printf("%d ID strings scanned\n", $idcount);
756 printf "$_: %s\n", $head{$_};