i18n.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Script to generate the template file and update the translation files.
   5 # Copy the script into the mod or modpack root folder and run it there.
   6 #
   7 # Copyright (C) 2019 Joachim Stolberg, 2020 FaceDeer, 2020 Louis Royer
   8 # LGPLv2.1+
   9 #
  10 # See https://github.com/minetest-tools/update_translations for
  11 # potential future updates to this script.
  12
  13 from __future__ import print_function
  14 import os, fnmatch, re, shutil, errno
  15 from sys import argv as _argv
  16
  17 # Running params
  18 params = {"recursive": False,
  19     "help": False,
  20     "mods": False,
  21     "verbose": False,
  22     "folders": [],
  23     "no-old-file": False,
  24     "sort": False
  25 }
  26 # Available CLI options
  27 options = {"recursive": ['--recursive', '-r'],
  28     "help": ['--help', '-h'],
  29     "mods": ['--installed-mods', '-m'],
  30     "verbose": ['--verbose', '-v'],
  31     "no-old-file": ['--no-old-file', '-O'],
  32     "sort": ['--sort', '-s']
  33 }
  34
  35 # Strings longer than this will have extra space added between
  36 # them in the translation files to make it easier to distinguish their
  37 # beginnings and endings at a glance
  38 doublespace_threshold = 80
  39
  40 def set_params_folders(tab: list):
  41     '''Initialize params["folders"] from CLI arguments.'''
  42     # Discarding argument 0 (tool name)
  43     for param in tab[1:]:
  44         stop_param = False
  45         for option in options:
  46             if param in options[option]:
  47                 stop_param = True
  48                 break
  49         if not stop_param:
  50             params["folders"].append(os.path.abspath(param))
  51
  52 def set_params(tab: list):
  53     '''Initialize params from CLI arguments.'''
  54     for option in options:
  55         for option_name in options[option]:
  56             if option_name in tab:
  57                 params[option] = True
  58                 break
  59
  60 def print_help(name):
  61     '''Prints some help message.'''
  62     print(f'''SYNOPSIS
  63     {name} [OPTIONS] [PATHS...]
  64 DESCRIPTION
  65     {', '.join(options["help"])}
  66         prints this help message
  67     {', '.join(options["recursive"])}
  68         run on all subfolders of paths given
  69     {', '.join(options["mods"])}
  70         run on locally installed modules
  71     {', '.join(options["no-old-file"])}
  72         do not create *.old files
  73     {', '.join(options["sort"])}
  74         sort strings alphabetically
  75     {', '.join(options["verbose"])}
  76         add output information
  77 ''')
  78
  79
  80 def main():
  81     '''Main function'''
  82     set_params(_argv)
  83     set_params_folders(_argv)
  84     if params["help"]:
  85         print_help(_argv[0])
  86     elif params["recursive"] and params["mods"]:
  87         print("Option --installed-mods is incompatible with --recursive")
  88     else:
  89         # Add recursivity message
  90         print("Running ", end='')
  91         if params["recursive"]:
  92             print("recursively ", end='')
  93         # Running
  94         if params["mods"]:
  95             print(f"on all locally installed modules in {os.path.abspath('~/.minetest/mods/')}")
  96             run_all_subfolders("~/.minetest/mods")
  97         elif len(params["folders"]) >= 2:
  98             print("on folder list:", params["folders"])
  99             for f in params["folders"]:
 100                 if params["recursive"]:
 101                     run_all_subfolders(f)
 102                 else:
 103                     update_folder(f)
 104         elif len(params["folders"]) == 1:
 105             print("on folder", params["folders"][0])
 106             if params["recursive"]:
 107                 run_all_subfolders(params["folders"][0])
 108             else:
 109                 update_folder(params["folders"][0])
 110         else:
 111             print("on folder", os.path.abspath("./"))
 112             if params["recursive"]:
 113                 run_all_subfolders(os.path.abspath("./"))
 114             else:
 115                 update_folder(os.path.abspath("./"))
 116
 117 #group 2 will be the string, groups 1 and 3 will be the delimiters (" or ')
 118 #See https://stackoverflow.com/questions/46967465/regex-match-text-in-either-single-or-double-quote
 119 pattern_lua = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*(["\'])((?:\\\1|(?:(?!\1)).)*)(\1)[\s,\)]', re.DOTALL)
 120 pattern_lua_bracketed = re.compile(r'[\.=^\t,{\(\s]N?S\(\s*\[\[(.*?)\]\][\s,\)]', re.DOTALL)
 121
 122 # Handles "concatenation" .. " of strings"
 123 pattern_concat = re.compile(r'["\'][\s]*\.\.[\s]*["\']', re.DOTALL)
 124
 125 pattern_tr = re.compile(r'(.+?[^@])=(.*)')
 126 pattern_name = re.compile(r'^name[ ]*=[ ]*([^ \n]*)')
 127 pattern_tr_filename = re.compile(r'\.tr$')
 128 pattern_po_language_code = re.compile(r'(.*)\.po$')
 129
 130 #attempt to read the mod's name from the mod.conf file. Returns None on failure
 131 def get_modname(folder):
 132     try:
 133         with open(os.path.join(folder, "mod.conf"), "r", encoding='utf-8') as mod_conf:
 134             for line in mod_conf:
 135                 match = pattern_name.match(line)
 136                 if match:
 137                     return match.group(1)
 138     except FileNotFoundError:
 139         pass
 140     return None
 141
 142 #If there are already .tr files in /locale, returns a list of their names
 143 def get_existing_tr_files(folder):
 144     out = []
 145     for root, dirs, files in os.walk(os.path.join(folder, 'locale/')):
 146         for name in files:
 147             if pattern_tr_filename.search(name):
 148                 out.append(name)
 149     return out
 150
 151 # A series of search and replaces that massage a .po file's contents into
 152 # a .tr file's equivalent
 153 def process_po_file(text):
 154     # The first three items are for unused matches
 155     text = re.sub(r'#~ msgid "', "", text)
 156     text = re.sub(r'"\n#~ msgstr ""\n"', "=", text)
 157     text = re.sub(r'"\n#~ msgstr "', "=", text)
 158     # comment lines
 159     text = re.sub(r'#.*\n', "", text)
 160     # converting msg pairs into "=" pairs
 161     text = re.sub(r'msgid "', "", text)
 162     text = re.sub(r'"\nmsgstr ""\n"', "=", text)
 163     text = re.sub(r'"\nmsgstr "', "=", text)
 164     # various line breaks and escape codes
 165     text = re.sub(r'"\n"', "", text)
 166     text = re.sub(r'"\n', "\n", text)
 167     text = re.sub(r'\\"', '"', text)
 168     text = re.sub(r'\\n', '@n', text)
 169     # remove header text
 170     text = re.sub(r'=Project-Id-Version:.*\n', "", text)
 171     # remove double-spaced lines
 172     text = re.sub(r'\n\n', '\n', text)
 173     return text
 174
 175 # Go through existing .po files and, if a .tr file for that language
 176 # *doesn't* exist, convert it and create it.
 177 # The .tr file that results will subsequently be reprocessed so
 178 # any "no longer used" strings will be preserved.
 179 # Note that "fuzzy" tags will be lost in this process.
 180 def process_po_files(folder, modname):
 181     for root, dirs, files in os.walk(os.path.join(folder, 'locale/')):
 182         for name in files:
 183             code_match = pattern_po_language_code.match(name)
 184             if code_match == None:
 185                 continue
 186             language_code = code_match.group(1)
 187             tr_name = modname + "." + language_code + ".tr"
 188             tr_file = os.path.join(root, tr_name)
 189             if os.path.exists(tr_file):
 190                 if params["verbose"]:
 191                     print(f"{tr_name} already exists, ignoring {name}")
 192                 continue
 193             fname = os.path.join(root, name)
 194             with open(fname, "r", encoding='utf-8') as po_file:
 195                 if params["verbose"]:
 196                     print(f"Importing translations from {name}")
 197                 text = process_po_file(po_file.read())
 198                 with open(tr_file, "wt", encoding='utf-8') as tr_out:
 199                     tr_out.write(text)
 200
 201 # from https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python/600612#600612
 202 # Creates a directory if it doesn't exist, silently does
 203 # nothing if it already exists
 204 def mkdir_p(path):
 205     try:
 206         os.makedirs(path)
 207     except OSError as exc: # Python >2.5
 208         if exc.errno == errno.EEXIST and os.path.isdir(path):
 209             pass
 210         else: raise
 211
 212 # Converts the template dictionary to a text to be written as a file
 213 # dKeyStrings is a dictionary of localized string to source file sets
 214 # dOld is a dictionary of existing translations and comments from
 215 # the previous version of this text
 216 def strings_to_text(dkeyStrings, dOld, mod_name):
 217     lOut = [f"# textdomain: {mod_name}"]
 218
 219     dGroupedBySource = {}
 220
 221     for key in dkeyStrings:
 222         sourceList = list(dkeyStrings[key])
 223         if params["sort"]:
 224             sourceList.sort()
 225         sourceString = "\n".join(sourceList)
 226         listForSource = dGroupedBySource.get(sourceString, [])
 227         listForSource.append(key)
 228         dGroupedBySource[sourceString] = listForSource
 229
 230     lSourceKeys = list(dGroupedBySource.keys())
 231     lSourceKeys.sort()
 232     for source in lSourceKeys:
 233         localizedStrings = dGroupedBySource[source]
 234         if params["sort"]:
 235             localizedStrings.sort()
 236         for localizedString in localizedStrings:
 237             val = dOld.get(localizedString, {})
 238             translation = val.get("translation", "")
 239             comment = val.get("comment")
 240             if len(localizedString) > doublespace_threshold and not lOut[-1] == "":
 241                 lOut.append("")
 242             if comment != None and comment != "" and not comment.startswith("# textdomain:"):
 243                 lOut.append(comment)
 244             lOut.append(f"{localizedString}={translation}")
 245             if len(localizedString) > doublespace_threshold:
 246                 lOut.append("")
 247
 248
 249     unusedExist = False
 250     for key in dOld:
 251         if key not in dkeyStrings:
 252             val = dOld[key]
 253             translation = val.get("translation")
 254             comment = val.get("comment")
 255             # only keep an unused translation if there was translated
 256             # text or a comment associated with it
 257             if translation != None and (translation != "" or comment):
 258                 if not unusedExist:
 259                     unusedExist = True
 260                     lOut.append("\n\n##### not used anymore #####\n")
 261                 if len(key) > doublespace_threshold and not lOut[-1] == "":
 262                     lOut.append("")
 263                 if comment != None:
 264                     lOut.append(comment)
 265                 lOut.append(f"{key}={translation}")
 266                 if len(key) > doublespace_threshold:
 267                     lOut.append("")
 268     return "\n".join(lOut) + '\n'
 269
 270 # Writes a template.txt file
 271 # dkeyStrings is the dictionary returned by generate_template
 272 def write_template(templ_file, dkeyStrings, mod_name):
 273     # read existing template file to preserve comments
 274     existing_template = import_tr_file(templ_file)
 275
 276     text = strings_to_text(dkeyStrings, existing_template[0], mod_name)
 277     mkdir_p(os.path.dirname(templ_file))
 278     with open(templ_file, "wt", encoding='utf-8') as template_file:
 279         template_file.write(text)
 280
 281
 282 # Gets all translatable strings from a lua file
 283 def read_lua_file_strings(lua_file):
 284     lOut = []
 285     with open(lua_file, encoding='utf-8') as text_file:
 286         text = text_file.read()
 287         #TODO remove comments here
 288
 289         text = re.sub(pattern_concat, "", text)
 290
 291         strings = []
 292         for s in pattern_lua.findall(text):
 293             strings.append(s[1])
 294         for s in pattern_lua_bracketed.findall(text):
 295             strings.append(s)
 296
 297         for s in strings:
 298             s = re.sub(r'"\.\.\s+"', "", s)
 299             s = re.sub("@[^@=0-9]", "@@", s)
 300             s = s.replace('\\"', '"')
 301             s = s.replace("\\'", "'")
 302             s = s.replace("\n", "@n")
 303             s = s.replace("\\n", "@n")
 304             s = s.replace("=", "@=")
 305             lOut.append(s)
 306     return lOut
 307
 308 # Gets strings from an existing translation file
 309 # returns both a dictionary of translations
 310 # and the full original source text so that the new text
 311 # can be compared to it for changes.
 312 def import_tr_file(tr_file):
 313     dOut = {}
 314     text = None
 315     if os.path.exists(tr_file):
 316         with open(tr_file, "r", encoding='utf-8') as existing_file :
 317             # save the full text to allow for comparison
 318             # of the old version with the new output
 319             text = existing_file.read()
 320             existing_file.seek(0)
 321             # a running record of the current comment block
 322             # we're inside, to allow preceeding multi-line comments
 323             # to be retained for a translation line
 324             latest_comment_block = None
 325             for line in existing_file.readlines():
 326                 line = line.rstrip('\n')
 327                 if line[:3] == "###":
 328                     # Reset comment block if we hit a header
 329                     latest_comment_block = None
 330                     continue
 331                 if line[:1] == "#":
 332                     # Save the comment we're inside
 333                     if not latest_comment_block:
 334                         latest_comment_block = line
 335                     else:
 336                         latest_comment_block = latest_comment_block + "\n" + line
 337                     continue
 338                 match = pattern_tr.match(line)
 339                 if match:
 340                     # this line is a translated line
 341                     outval = {}
 342                     outval["translation"] = match.group(2)
 343                     if latest_comment_block:
 344                         # if there was a comment, record that.
 345                         outval["comment"] = latest_comment_block
 346                     latest_comment_block = None
 347                     dOut[match.group(1)] = outval
 348     return (dOut, text)
 349
 350 # Walks all lua files in the mod folder, collects translatable strings,
 351 # and writes it to a template.txt file
 352 # Returns a dictionary of localized strings to source file sets
 353 # that can be used with the strings_to_text function.
 354 def generate_template(folder, mod_name):
 355     dOut = {}
 356     for root, dirs, files in os.walk(folder):
 357         for name in files:
 358             if fnmatch.fnmatch(name, "*.lua"):
 359                 fname = os.path.join(root, name)
 360                 found = read_lua_file_strings(fname)
 361                 if params["verbose"]:
 362                     print(f"{fname}: {str(len(found))} translatable strings")
 363
 364                 for s in found:
 365                     sources = dOut.get(s, set())
 366                     sources.add(f"### {os.path.basename(fname)} ###")
 367                     dOut[s] = sources
 368
 369     if len(dOut) == 0:
 370         return None
 371     templ_file = os.path.join(folder, "locale/template.txt")
 372     write_template(templ_file, dOut, mod_name)
 373     return dOut
 374
 375 # Updates an existing .tr file, copying the old one to a ".old" file
 376 # if any changes have happened
 377 # dNew is the data used to generate the template, it has all the
 378 # currently-existing localized strings
 379 def update_tr_file(dNew, mod_name, tr_file):
 380     if params["verbose"]:
 381         print(f"updating {tr_file}")
 382
 383     tr_import = import_tr_file(tr_file)
 384     dOld = tr_import[0]
 385     textOld = tr_import[1]
 386
 387     textNew = strings_to_text(dNew, dOld, mod_name)
 388
 389     if textOld and textOld != textNew:
 390         print(f"{tr_file} has changed.")
 391         if not params["no-old-file"]:
 392             shutil.copyfile(tr_file, f"{tr_file}.old")
 393
 394     with open(tr_file, "w", encoding='utf-8') as new_tr_file:
 395         new_tr_file.write(textNew)
 396
 397 # Updates translation files for the mod in the given folder
 398 def update_mod(folder):
 399     modname = get_modname(folder)
 400     if modname is not None:
 401         process_po_files(folder, modname)
 402         print(f"Updating translations for {modname}")
 403         data = generate_template(folder, modname)
 404         if data == None:
 405             print(f"No translatable strings found in {modname}")
 406         else:
 407             for tr_file in get_existing_tr_files(folder):
 408                 update_tr_file(data, modname, os.path.join(folder, "locale/", tr_file))
 409     else:
 410         print("Unable to find modname in folder " + folder)
 411
 412 # Determines if the folder being pointed to is a mod or a mod pack
 413 # and then runs update_mod accordingly
 414 def update_folder(folder):
 415     is_modpack = os.path.exists(os.path.join(folder, "modpack.txt")) or os.path.exists(os.path.join(folder, "modpack.conf"))
 416     if is_modpack:
 417         subfolders = [f.path for f in os.scandir(folder) if f.is_dir()]
 418         for subfolder in subfolders:
 419             update_mod(subfolder + "/")
 420     else:
 421         update_mod(folder)
 422     print("Done.")
 423
 424 def run_all_subfolders(folder):
 425     for modfolder in [f.path for f in os.scandir(folder) if f.is_dir()]:
 426         update_folder(modfolder + "/")
 427
 428
 429 main()