325 lines
14 KiB
Python
325 lines
14 KiB
Python
|
# Written for Python 3.6+
|
||
|
|
# Older versions don't retain insertion order of regular dicts
|
||
|
|
import argparse
|
||
|
|
import cmd
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
from pprint import pprint
|
||
|
|
|
||
|
|
INDENT = 2
|
||
|
|
PRIMARY_LANGUAGE = 'en-US.json'
|
||
|
|
PRIMARY_FALLBACK_PREFIX = '🇺🇸' # This is invisible in-game, terminal emulators might render it
|
||
|
|
LANGUAGE_FOLDER = 'src/main/resources/languages/'
|
||
|
|
LANGUAGE_FILENAMES = sorted(os.listdir(LANGUAGE_FOLDER), key=lambda x: 'AAA' if x == PRIMARY_LANGUAGE else x)
|
||
|
|
SOURCE_FOLDER = 'src/'
|
||
|
|
SOURCE_EXTENSIONS = ('java')
|
||
|
|
|
||
|
|
|
||
|
|
def ppprint(data):
|
||
|
|
pprint(data, width=130, sort_dicts=False, compact=True)
|
||
|
|
|
||
|
|
|
||
|
|
class JsonHelpers:
|
||
|
|
@staticmethod
|
||
|
|
def load(filename: str) -> dict:
|
||
|
|
with open(filename, 'r', encoding='utf-8') as file:
|
||
|
|
return json.load(file)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def save(filename: str, data: dict) -> None:
|
||
|
|
with open(filename, 'w', encoding='utf-8', newline='\n') as file:
|
||
|
|
json.dump(data, file, ensure_ascii=False, indent=INDENT)
|
||
|
|
file.write('\n') # json.dump doesn't terminate last line
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def flatten(data: dict, prefix='') -> dict:
|
||
|
|
output = {}
|
||
|
|
for key, value in data.items():
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for k,v in JsonHelpers.flatten(value, f'{prefix}{key}.').items():
|
||
|
|
output[k] = v
|
||
|
|
else:
|
||
|
|
output[f'{prefix}{key}'] = value
|
||
|
|
return output
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def unflatten(data: dict) -> dict:
|
||
|
|
output = {}
|
||
|
|
def add_key(k: list, value, d: dict):
|
||
|
|
if len(k) == 1:
|
||
|
|
d[k[0]] = value
|
||
|
|
else:
|
||
|
|
d[k[0]] = d.get(k[0], {})
|
||
|
|
add_key(k[1:], value, d[k[0]])
|
||
|
|
for key, value in data.items():
|
||
|
|
add_key(key.split('.'), value, output)
|
||
|
|
return output
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def pprint_keys(keys, indent=4) -> str:
|
||
|
|
# Only strip down to one level
|
||
|
|
padding = ' ' * indent
|
||
|
|
roots = {}
|
||
|
|
for key in keys:
|
||
|
|
root, _, k = key.rpartition('.')
|
||
|
|
roots[root] = roots.get(root, [])
|
||
|
|
roots[root].append(k)
|
||
|
|
lines = []
|
||
|
|
for root, ks in roots.items():
|
||
|
|
if len(ks) > 1:
|
||
|
|
lines.append(f'{padding}{root}.[{", ".join(ks)}]')
|
||
|
|
else:
|
||
|
|
lines.append(f'{padding}{root}.{ks[0]}')
|
||
|
|
return ',\n'.join(lines)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def deep_clone_and_fill(d1: dict, d2: dict, fallback_prefix=PRIMARY_FALLBACK_PREFIX) -> dict:
|
||
|
|
out = {}
|
||
|
|
for key, value in d1.items():
|
||
|
|
if isinstance(value, dict):
|
||
|
|
out[key] = JsonHelpers.deep_clone_and_fill(value, d2.get(key, {}), fallback_prefix)
|
||
|
|
else:
|
||
|
|
v2 = d2.get(key, value)
|
||
|
|
if type(value) == str and v2 == value:
|
||
|
|
out[key] = fallback_prefix + value
|
||
|
|
else:
|
||
|
|
out[key] = v2
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
class LanguageManager:
|
||
|
|
TRANSLATION_KEY = re.compile(r'[Tt]ranslate.*"(\w+\.[\w\.]+)"')
|
||
|
|
POTENTIAL_KEY = re.compile(r'"(\w+\.[\w\.]+)"')
|
||
|
|
COMMAND_LABEL = re.compile(r'@Command\s*\([\W\w]*?label\s*=\s*"(\w+)"', re.MULTILINE) # [\W\w] is a cheeky way to match everything including \n
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.load_jsons()
|
||
|
|
|
||
|
|
def load_jsons(self):
|
||
|
|
self.language_jsons = [JsonHelpers.load(LANGUAGE_FOLDER + filename) for filename in LANGUAGE_FILENAMES]
|
||
|
|
self.flattened_jsons = [JsonHelpers.flatten(j) for j in self.language_jsons]
|
||
|
|
self.update_keys()
|
||
|
|
|
||
|
|
def update_keys(self):
|
||
|
|
self.key_sets = [set(j.keys()) for j in self.flattened_jsons]
|
||
|
|
self.common_keys = set.intersection(*self.key_sets)
|
||
|
|
self.all_keys = set.union(*self.key_sets)
|
||
|
|
self.used_keys = self.find_all_used_keys(self.all_keys)
|
||
|
|
self.missing_keys = self.used_keys - self.common_keys
|
||
|
|
self.unused_keys = self.all_keys - self.used_keys
|
||
|
|
|
||
|
|
def find_all_used_keys(self, expected_keys=[]) -> set:
|
||
|
|
# Note that this will only find string literals passed to the translate() or sendTranslatedMessage() methods!
|
||
|
|
# String variables passed to them can be checked against expected_keys
|
||
|
|
used = set()
|
||
|
|
potential = set()
|
||
|
|
for root, dirs, files in os.walk(SOURCE_FOLDER):
|
||
|
|
for file in files:
|
||
|
|
if file.rpartition('.')[-1] in SOURCE_EXTENSIONS:
|
||
|
|
filename = os.path.join(root, file)
|
||
|
|
with open(filename, 'r', encoding='utf-8') as f:
|
||
|
|
data = f.read() # Loads in entire file at once
|
||
|
|
for k in self.TRANSLATION_KEY.findall(data):
|
||
|
|
used.add(k)
|
||
|
|
for k in self.POTENTIAL_KEY.findall(data):
|
||
|
|
potential.add(k)
|
||
|
|
for label in self.COMMAND_LABEL.findall(data):
|
||
|
|
used.add(f'commands.{label}.description')
|
||
|
|
return used | (potential & expected_keys)
|
||
|
|
|
||
|
|
def _lint_report_language(self, lang: str, keys: set, flattened: dict, primary_language_flattened: dict) -> None:
|
||
|
|
missing = self.used_keys - keys
|
||
|
|
unused = keys - self.used_keys
|
||
|
|
identical_keys = set() if (lang == PRIMARY_LANGUAGE) else {key for key in keys if primary_language_flattened.get(key, None) == flattened.get(key)}
|
||
|
|
placeholder_keys = {key for key in keys if flattened.get(key).startswith(PRIMARY_FALLBACK_PREFIX)}
|
||
|
|
p1 = f'Language {lang} has {len(missing)} missing keys and {len(unused)} unused keys.'
|
||
|
|
p2 = 'This is the primary language.' if (lang == PRIMARY_LANGUAGE) else f'{len(identical_keys)} match {PRIMARY_LANGUAGE}, {len(placeholder_keys)} have the placeholder mark.'
|
||
|
|
print(f'{p1} {p2}')
|
||
|
|
|
||
|
|
lint_categories = {
|
||
|
|
'Missing': missing,
|
||
|
|
'Unused': unused,
|
||
|
|
f'Matches {PRIMARY_LANGUAGE}': identical_keys,
|
||
|
|
'Placeholder': placeholder_keys,
|
||
|
|
}
|
||
|
|
for name, category in lint_categories.items():
|
||
|
|
if len(category) > 0:
|
||
|
|
print(name + ':')
|
||
|
|
print(JsonHelpers.pprint_keys(sorted(category)))
|
||
|
|
|
||
|
|
def lint_report(self) -> None:
|
||
|
|
print(f'There are {len(self.missing_keys)} translation keys in use that are missing from one or more language files.')
|
||
|
|
print(f'There are {len(self.unused_keys)} translation keys in language files that are not used.')
|
||
|
|
primary_language_flattened = self.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
|
||
|
|
for lang, keys, flattened in zip(LANGUAGE_FILENAMES, self.key_sets, self.flattened_jsons):
|
||
|
|
print('')
|
||
|
|
self._lint_report_language(lang, keys, flattened, primary_language_flattened)
|
||
|
|
|
||
|
|
def rename_keys(self, key_remappings: dict) -> None:
|
||
|
|
# Unfortunately we can't rename keys in-place preserving insertion order, so we have to make new dicts
|
||
|
|
for i in range(len(self.flattened_jsons)):
|
||
|
|
self.flattened_jsons[i] = {key_remappings.get(k,k):v for k,v in self.flattened_jsons[i].items()}
|
||
|
|
|
||
|
|
def update_secondary_languages(self):
|
||
|
|
# Push en_US fallback
|
||
|
|
primary_language_json = self.language_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
|
||
|
|
for filename, lang in zip(LANGUAGE_FILENAMES, self.language_jsons):
|
||
|
|
if filename != PRIMARY_LANGUAGE:
|
||
|
|
js = JsonHelpers.deep_clone_and_fill(primary_language_json, lang)
|
||
|
|
JsonHelpers.save(LANGUAGE_FOLDER + filename, js)
|
||
|
|
|
||
|
|
def update_all_languages_from_flattened(self):
|
||
|
|
for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
|
||
|
|
JsonHelpers.save(LANGUAGE_FOLDER + filename, JsonHelpers.unflatten(flat))
|
||
|
|
|
||
|
|
def save_flattened_languages(self, prefix='flat_'):
|
||
|
|
for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons):
|
||
|
|
JsonHelpers.save(prefix + filename, flat)
|
||
|
|
|
||
|
|
|
||
|
|
class InteractiveRename(cmd.Cmd):
|
||
|
|
intro = 'Welcome to the interactive rename shell. Type help or ? to list commands.\n'
|
||
|
|
prompt = '(rename) '
|
||
|
|
file = None
|
||
|
|
|
||
|
|
def __init__(self, language_manager: LanguageManager) -> None:
|
||
|
|
super().__init__()
|
||
|
|
self.language_manager = language_manager
|
||
|
|
self.flat_keys = [key for key in language_manager.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)].keys()]
|
||
|
|
self.mappings = {}
|
||
|
|
|
||
|
|
def do_add(self, arg):
|
||
|
|
'''
|
||
|
|
Prepare to rename an existing translation key. Will not actually rename anything until you confirm all your pending changes with 'rename'.
|
||
|
|
e.g. a single string: add commands.execution.argument_error commands.generic.invalid.argument
|
||
|
|
e.g. a group: add commands.enter_dungeon commands.new_enter_dungeon
|
||
|
|
'''
|
||
|
|
args = arg.split()
|
||
|
|
if len(args) < 2:
|
||
|
|
self.do_help('add')
|
||
|
|
return
|
||
|
|
old, new = args[:2]
|
||
|
|
if old in self.flat_keys:
|
||
|
|
self.mappings[old] = new
|
||
|
|
else:
|
||
|
|
# Check if we are renaming a higher level
|
||
|
|
if not old.endswith('.'):
|
||
|
|
old = old + '.'
|
||
|
|
results = [key for key in self.flat_keys if key.startswith(old)]
|
||
|
|
if len(results) > 0:
|
||
|
|
if not new.endswith('.'):
|
||
|
|
new = new + '.'
|
||
|
|
new_mappings = {key: key.replace(old, new) for key in results}
|
||
|
|
# Ask for confirmation
|
||
|
|
print('Will add the following mappings:')
|
||
|
|
ppprint(new_mappings)
|
||
|
|
print('Add these mappings? [y/N]')
|
||
|
|
if self.prompt_yn():
|
||
|
|
for k,v in new_mappings.items():
|
||
|
|
self.mappings[k] = v
|
||
|
|
else:
|
||
|
|
print('No translation keys matched!')
|
||
|
|
|
||
|
|
def complete_add(self, text: str, line: str, begidx: int, endidx: int) -> list:
|
||
|
|
if text == '':
|
||
|
|
return [k for k in {key.partition('.')[0] for key in self.flat_keys}]
|
||
|
|
results = [key for key in self.flat_keys if key.startswith(text)]
|
||
|
|
if len(results) > 40:
|
||
|
|
# Collapse categories
|
||
|
|
if text[-1] != '.':
|
||
|
|
text = text + '.'
|
||
|
|
level = text.count('.') + 1
|
||
|
|
new_results = {'.'.join(key.split('.')[:level]) for key in results}
|
||
|
|
return list(new_results)
|
||
|
|
return results
|
||
|
|
|
||
|
|
def do_remove(self, arg):
|
||
|
|
'''
|
||
|
|
Remove a pending rename mapping. Takes the old name of the key, not the new one.
|
||
|
|
e.g. a single key: remove commands.execution.argument_error
|
||
|
|
e.g. a group: remove commands.enter_dungeon
|
||
|
|
'''
|
||
|
|
old = arg.split()[0]
|
||
|
|
if old in self.mappings:
|
||
|
|
self.mappings.pop(old)
|
||
|
|
else:
|
||
|
|
# Check if we are renaming a higher level
|
||
|
|
if not old.endswith('.'):
|
||
|
|
old = old + '.'
|
||
|
|
results = [key for key in self.mappings if key.startswith(old)]
|
||
|
|
if len(results) > 0:
|
||
|
|
# Ask for confirmation
|
||
|
|
print('Will remove the following pending mappings:')
|
||
|
|
print(JsonHelpers.pprint_keys(results))
|
||
|
|
print('Delete these mappings? [y/N]')
|
||
|
|
if self.prompt_yn():
|
||
|
|
for key in results:
|
||
|
|
self.mappings.pop(key)
|
||
|
|
else:
|
||
|
|
print('No pending rename mappings matched!')
|
||
|
|
|
||
|
|
def complete_remove(self, text: str, line: str, begidx: int, endidx: int) -> list:
|
||
|
|
return [key for key in self.mappings if key.startswith(text)]
|
||
|
|
|
||
|
|
def do_rename(self, _arg):
|
||
|
|
'Applies pending renames and overwrites language jsons.'
|
||
|
|
# Ask for confirmation
|
||
|
|
print('Will perform the following mappings:')
|
||
|
|
ppprint(self.mappings)
|
||
|
|
print('Perform and save these rename mappings? [y/N]')
|
||
|
|
if self.prompt_yn():
|
||
|
|
self.language_manager.rename_keys(self.mappings)
|
||
|
|
self.language_manager.update_all_languages_from_flattened()
|
||
|
|
print('Renamed keys, closing')
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
print('Do you instead wish to quit without saving? [yes/N]')
|
||
|
|
if self.prompt_yn(True):
|
||
|
|
print('Left rename shell without renaming')
|
||
|
|
return True
|
||
|
|
|
||
|
|
def prompt_yn(self, strict_yes=False):
|
||
|
|
if strict_yes:
|
||
|
|
return input('(yes/N) ').lower() == 'yes'
|
||
|
|
return input('(y/N) ').lower()[0] == 'y'
|
||
|
|
|
||
|
|
|
||
|
|
def main(args: argparse.Namespace):
|
||
|
|
# print(args)
|
||
|
|
language_manager = LanguageManager()
|
||
|
|
errors = None
|
||
|
|
if args.lint_report:
|
||
|
|
language_manager.lint_report()
|
||
|
|
missing = language_manager.used_keys - language_manager.key_sets[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)]
|
||
|
|
if len(missing) > 0:
|
||
|
|
errors = f'[ERROR] {len(missing)} keys missing from primary language json!\n{JsonHelpers.pprint_keys(missing)}'
|
||
|
|
if prefix := args.save_flattened:
|
||
|
|
language_manager.save_flattened_languages(prefix)
|
||
|
|
if args.update:
|
||
|
|
print('Updating secondary languages')
|
||
|
|
language_manager.update_secondary_languages()
|
||
|
|
if args.interactive_rename:
|
||
|
|
language_manager.load_jsons() # Previous actions may have changed them on-disk
|
||
|
|
try:
|
||
|
|
InteractiveRename(language_manager).cmdloop()
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
print('Left rename shell without renaming')
|
||
|
|
if errors:
|
||
|
|
print(errors)
|
||
|
|
exit(1)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
parser = argparse.ArgumentParser(description="Manage Grasscutter's language json files.")
|
||
|
|
parser.add_argument('-u', '--update', action='store_true',
|
||
|
|
help=f'Update secondary language files to conform to the layout of the primary language file ({PRIMARY_LANGUAGE}) and contain any new keys from it.')
|
||
|
|
parser.add_argument('-l', '--lint-report', action='store_true',
|
||
|
|
help='Prints a lint report, listing unused, missing, and untranslated keys among all language jsons.')
|
||
|
|
parser.add_argument('-f', '--save-flattened', const='./flat_', metavar='prefix', nargs='?',
|
||
|
|
help='Save copies of all the language jsons in a flattened key form.')
|
||
|
|
parser.add_argument('-i', '--interactive-rename', action='store_true',
|
||
|
|
help='Enter interactive rename mode, in which you can specify keys in flattened form to be renamed.')
|
||
|
|
args = parser.parse_args()
|
||
|
|
main(args)
|