From 59de1d58d4aa891cc19372dd6104ec65e1cf4731 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 11 Aug 2020 14:14:38 +0100 Subject: SL-13705 - modified-strings.sh added, for translation support --- scripts/code_tools/modified-strings.sh | 199 +++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 scripts/code_tools/modified-strings.sh (limited to 'scripts') diff --git a/scripts/code_tools/modified-strings.sh b/scripts/code_tools/modified-strings.sh new file mode 100644 index 0000000000..435dda3f5d --- /dev/null +++ b/scripts/code_tools/modified-strings.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# $LicenseInfo:firstyear=2014&license=viewerlgpl$ +# Second Life Viewer Source Code +# Copyright (C) 2011, Linden Research, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; +# version 2.1 of the License only. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +# +### +### Extract strings modified between some version and the current version +### + +Action=DEFAULT +Rev=master +DefaultXuiDir="indra/newview/skins/default/xui" +Verbose=false +ExitStatus=0 + +while [ $# -ne 0 ] +do + case ${1} in + ## + ## Show usage + ## + -h|--help) + Action=USAGE + ;; + + -v|--verbose) + Verbose=true + ;; + + ## + ## Select the revision to compare against + ## + -r) + if [ $# -lt 2 ] + then + echo "Must specify with ${1}" 1>&2 + Action=USAGE + ExitStatus=1 + break + else + Rev=${2} + shift # consume the switch ( for n values, consume n-1 ) + fi + ;; + + ## + ## handle an unknown switch + ## + -*) + Action=USAGE + ExitStatus=1 + break + ;; + + *) + if [ -z "${XuiDir}" ] + then + XuiDir=${1} + else + echo "Too many arguments supplied: $@" 1>&2 + Action=USAGE + ExitStatus=1 + break + fi + ;; + esac + + shift # always consume 1 +done + +progress() +{ + if $Verbose + then + echo $* 1>&2 + fi +} + +if [[ $ExitStatus -eq 0 && "${Action}" = "DEFAULT" ]] +then + if [[ ! -d "${XuiDir:=$DefaultXuiDir}" ]] + then + echo "No XUI directory found in '$XuiDir'" 1>&2 + Action=USAGE + ExitStatus=1 + fi +fi + +if [ "${Action}" = "USAGE" ] +then + cat <] [] + + where + --verbose shows progress messages on stderr (the command takes a while, so this is reassuring) + + -r specifies a git revision (branch, tag, commit, or relative specifier) + defaults to 'master' so that comparison is against the HEAD of the released viewer branch + + is the path to the root directory for XUI files + defaults to '$DefaultXuiDir' + + Emits a tab-separated file with these columns: + filename + the path of a file that has a string change (columns 2 and 3 are empty for lines with a filename) + name + the name attribute of a string or label whose value changed + English value + the current value of the string or label whose value changed + for strings, newlines are changed to '\n' and tab characters are changed to '\t' + + There is also a column for each of the language directories following the English. + +USAGE + exit $ExitStatus +fi + +stringval() # reads stdin and prints the escaped value of a string for the requested tag +{ + local tag=$1 + xmllint --xpath "string(/strings/string[@name=\"$tag\"])" - | perl -p -e 'chomp; s/\n/\\n/g; s/\t/\\t/g;' +} + +columns="file\tname\tEN" +for lang in $(ls -1 ${XuiDir}) +do + if [[ "$lang" != "en" && -d "${XuiDir}" && -f "${XuiDir}/$lang/strings.xml" ]] + then + columns+="\t$lang" + fi +done +echo -e "$columns" + +EnglishStrings="${XuiDir}/en/strings.xml" +progress -n "scanning $EnglishStrings " +echo -e "$EnglishStrings" +# loop over all tags in the current version of the strings file +cat "$EnglishStrings" | xmllint --xpath '/strings/string/@name' - | sed 's/ name="//; s/"$//;' \ +| while read name +do + progress -n "." + # fetch the $Rev and current values for each tag + old_stringval=$(git show "$Rev:$EnglishStrings" 2> /dev/null | stringval "$name") + new_stringval=$(cat "$EnglishStrings" | stringval "$name") + + if [[ "$old_stringval" != "$new_stringval" ]] + then + # the value is different, so print the tag and it's current value separated by a tab + echo -e "\t$name\t$new_stringval" + fi +done +progress "" + +# loop over all XUI files other than strings.xml finding labels +grep -rlw 'label' "${XuiDir}/en" | grep -v '/strings.xml' \ +| while read xuipath +do + progress -n "scanning $xuipath " + listed_file=false + # loop over all elements for which there is a label attribute, getting the name attribute value + xmllint --xpath '//*[@label]/@name' "$xuipath" 2> /dev/null | sed 's/ name="//; s/"$//;' \ + | while read name + do + progress -n "." + # get the old and new label attribute values for each name + old_label=$(git show "$Rev:$xuipath" 2> /dev/null | xmllint --xpath "string(//*[@name=\"${name}\"]/@label)" - 2> /dev/null) + new_label=$(cat "$xuipath" | xmllint --xpath "string(//*[@name=\"${name}\"]/@label)" - 2> /dev/null) + if [[ "$old_label" != "$new_label" ]] + then + if ! $listed_file + then + echo -e "$xuipath" + listed_file=true + fi + echo -e "\t$name\t$new_label" + fi + done + progress "" +done + -- cgit v1.2.3 From 1f9852f04c4c430631d10794cce4a3e8186470b5 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Wed, 12 Aug 2020 18:27:00 +0100 Subject: SL-13705 - modified_strings.py added, for translation support --- scripts/code_tools/modified_strings.py | 158 +++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 scripts/code_tools/modified_strings.py (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py new file mode 100644 index 0000000000..bb42628f55 --- /dev/null +++ b/scripts/code_tools/modified_strings.py @@ -0,0 +1,158 @@ +"""\ + +This module contains tools for scanning the SL codebase for translation-related strings. + +$LicenseInfo:firstyear=2020&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2020, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +from __future__ import print_function + +# packages required include: gitpython, pandas + +import xml.etree.ElementTree as ET +import argparse +import os +import sys +from git import Repo, Git # requires the gitpython package +import pandas as pd + +translate_attribs = [ + "title", + "short_title", + "value", + "label", + "label_selected", + "tool_tip", + "ignoretext", + "yestext", + "notext", + "canceltext", + "description", + "longdescription" +] + +def codify(val): + if isinstance(val, unicode): + return val.encode("utf-8") + else: + return unicode(val, 'utf-8').encode("utf-8") + +def failure(*msg): + print(*msg) + sys.exit(1) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="analyze viewer xui files") + parser.add_argument("--verbose", action="store_true", help="verbose flag") + parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") + parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") + parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en") + parser.add_argument("--lang", help="target language, default fr", default="fr") + #parser.add_argument("infilename", help="name of input file", nargs="?") + args = parser.parse_args() + + #root = ET.parse(args.infilename) + + #for child in root.iter("string"): + # print child.attrib["name"], "\t", unicode(child.text, 'utf-8').encode("utf-8") + # #print unicode(child.text, 'utf-8') + # #print u'\u0420\u043e\u0441\u0441\u0438\u044f'.encode("utf-8") + + if args.rev == args.rev_base: + failure("Revs are the same, nothing to compare") + + print("Finding changes in", args.rev, "not present in", args.rev_base) + + cwd = os.getcwd() + rootdir = Git(cwd).rev_parse("--show-toplevel") + repo = Repo(rootdir) + try: + mod_commit = repo.commit(args.rev) + except: + failure(args.rev,"is not a valid commit") + try: + base_commit = repo.commit(args.rev_base) + except: + failure(args.rev_base,"is not a valid commit") + + mod_tree = mod_commit.tree + base_tree = base_commit.tree + + all_attrib = set() + + try: + mod_xui_tree = mod_tree["indra/newview/skins/default/xui/{}".format(args.base_lang)] + except: + print("xui tree not found for language", args.base_lang) + sys.exit(1) + + data = [] + # For all files to be checked for translations + for mod_blob in mod_xui_tree.traverse(): + print(mod_blob.path) + filename = mod_blob.path + if mod_blob.type == "tree": # directory, skip + continue + + mod_contents = mod_blob.data_stream.read() + try: + base_blob = base_tree[filename] + base_contents = base_blob.data_stream.read() + except: + print("No matching base file found for", filename) + base_contents = '' + + mod_xml = ET.fromstring(mod_contents) + base_xml = ET.fromstring(base_contents) + + mod_dict = {} + for child in mod_xml.iter(): + if "name" in child.attrib: + name = child.attrib['name'] + mod_dict[name] = child + base_dict = {} + for child in base_xml.iter(): + if "name" in child.attrib: + name = child.attrib['name'] + base_dict[name] = child + for name in mod_dict.keys(): + if not name in base_dict or mod_dict[name].text != base_dict[name].text: + data.append([filename, name, "text", mod_dict[name].text,""]) + #print(" ", name, "text", codify(mod_dict[name].text)) + all_attrib = all_attrib.union(set(mod_dict[name].attrib.keys())) + for attr in translate_attribs: + if attr in mod_dict[name].attrib: + if name not in base_dict or attr not in base_dict[name] or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]: + val = mod_dict[name].attrib[attr] + data.append([filename, name, attr, mod_dict[name].attrib[attr],""]) + #print(" ", name, attr, codify(val)) + + cols = ["File", "Element", "Field", "EN", "Translation ({})".format(args.lang)] + df = pd.DataFrame(data, columns=cols) + df.to_excel("SL_Translations_{}.xlsx".format(args.lang.upper()), index=False) + + #print "all_attrib", all_attrib + + + + -- cgit v1.2.3 From f28437adc7ae856d55edfe77596f5ce7331778f4 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Thu, 13 Aug 2020 14:54:45 +0100 Subject: SL-13705 - modified_strings.py handles more cases --- scripts/code_tools/modified_strings.py | 341 ++++++++++++++++++--------------- 1 file changed, 183 insertions(+), 158 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index bb42628f55..dc3357fe8e 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -1,158 +1,183 @@ -"""\ - -This module contains tools for scanning the SL codebase for translation-related strings. - -$LicenseInfo:firstyear=2020&license=viewerlgpl$ -Second Life Viewer Source Code -Copyright (C) 2020, Linden Research, Inc. - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; -version 2.1 of the License only. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -$/LicenseInfo$ -""" - -from __future__ import print_function - -# packages required include: gitpython, pandas - -import xml.etree.ElementTree as ET -import argparse -import os -import sys -from git import Repo, Git # requires the gitpython package -import pandas as pd - -translate_attribs = [ - "title", - "short_title", - "value", - "label", - "label_selected", - "tool_tip", - "ignoretext", - "yestext", - "notext", - "canceltext", - "description", - "longdescription" -] - -def codify(val): - if isinstance(val, unicode): - return val.encode("utf-8") - else: - return unicode(val, 'utf-8').encode("utf-8") - -def failure(*msg): - print(*msg) - sys.exit(1) - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description="analyze viewer xui files") - parser.add_argument("--verbose", action="store_true", help="verbose flag") - parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") - parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") - parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en") - parser.add_argument("--lang", help="target language, default fr", default="fr") - #parser.add_argument("infilename", help="name of input file", nargs="?") - args = parser.parse_args() - - #root = ET.parse(args.infilename) - - #for child in root.iter("string"): - # print child.attrib["name"], "\t", unicode(child.text, 'utf-8').encode("utf-8") - # #print unicode(child.text, 'utf-8') - # #print u'\u0420\u043e\u0441\u0441\u0438\u044f'.encode("utf-8") - - if args.rev == args.rev_base: - failure("Revs are the same, nothing to compare") - - print("Finding changes in", args.rev, "not present in", args.rev_base) - - cwd = os.getcwd() - rootdir = Git(cwd).rev_parse("--show-toplevel") - repo = Repo(rootdir) - try: - mod_commit = repo.commit(args.rev) - except: - failure(args.rev,"is not a valid commit") - try: - base_commit = repo.commit(args.rev_base) - except: - failure(args.rev_base,"is not a valid commit") - - mod_tree = mod_commit.tree - base_tree = base_commit.tree - - all_attrib = set() - - try: - mod_xui_tree = mod_tree["indra/newview/skins/default/xui/{}".format(args.base_lang)] - except: - print("xui tree not found for language", args.base_lang) - sys.exit(1) - - data = [] - # For all files to be checked for translations - for mod_blob in mod_xui_tree.traverse(): - print(mod_blob.path) - filename = mod_blob.path - if mod_blob.type == "tree": # directory, skip - continue - - mod_contents = mod_blob.data_stream.read() - try: - base_blob = base_tree[filename] - base_contents = base_blob.data_stream.read() - except: - print("No matching base file found for", filename) - base_contents = '' - - mod_xml = ET.fromstring(mod_contents) - base_xml = ET.fromstring(base_contents) - - mod_dict = {} - for child in mod_xml.iter(): - if "name" in child.attrib: - name = child.attrib['name'] - mod_dict[name] = child - base_dict = {} - for child in base_xml.iter(): - if "name" in child.attrib: - name = child.attrib['name'] - base_dict[name] = child - for name in mod_dict.keys(): - if not name in base_dict or mod_dict[name].text != base_dict[name].text: - data.append([filename, name, "text", mod_dict[name].text,""]) - #print(" ", name, "text", codify(mod_dict[name].text)) - all_attrib = all_attrib.union(set(mod_dict[name].attrib.keys())) - for attr in translate_attribs: - if attr in mod_dict[name].attrib: - if name not in base_dict or attr not in base_dict[name] or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]: - val = mod_dict[name].attrib[attr] - data.append([filename, name, attr, mod_dict[name].attrib[attr],""]) - #print(" ", name, attr, codify(val)) - - cols = ["File", "Element", "Field", "EN", "Translation ({})".format(args.lang)] - df = pd.DataFrame(data, columns=cols) - df.to_excel("SL_Translations_{}.xlsx".format(args.lang.upper()), index=False) - - #print "all_attrib", all_attrib - - - - +"""\ + +This module contains tools for scanning the SL codebase for translation-related strings. + +$LicenseInfo:firstyear=2020&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2020, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +from __future__ import print_function + +# packages required include: gitpython, pandas + +import xml.etree.ElementTree as ET +import argparse +import os +import sys +from git import Repo, Git # requires the gitpython package +import pandas as pd + +translate_attribs = [ + "title", + "short_title", + "value", + "label", + "label_selected", + "tool_tip", + "ignoretext", + "yestext", + "notext", + "canceltext", + "description", + "longdescription" +] + +def codify_for_print(val): + if isinstance(val, unicode): + return val.encode("utf-8") + else: + return unicode(val, 'utf-8').encode("utf-8") + +# Returns a dict of { name => xml_node } +def read_xml_elements(blob): + try: + contents = blob.data_stream.read() + except: + contents = '' + xml = ET.fromstring(contents) + elts = {} + for child in xml.iter(): + if "name" in child.attrib: + name = child.attrib['name'] + elts[name] = child + return elts + +def failure(*msg): + print(*msg) + sys.exit(1) + +def can_translate(val): + if val is None: + return False + if val.isspace(): + return False + val = val.strip() + if val.isdigit(): + return False + return True + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="analyze viewer xui files") + parser.add_argument("-v","--verbose", action="store_true", help="verbose flag") + parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") + parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") + parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en") + parser.add_argument("--lang", help="target language, default fr", default="fr") + args = parser.parse_args() + + if args.rev == args.rev_base: + failure("Revs are the same, nothing to compare") + + print("Finding changes in", args.rev, "not present in", args.rev_base) + sys.stdout.flush() + + cwd = os.getcwd() + rootdir = Git(cwd).rev_parse("--show-toplevel") + repo = Repo(rootdir) + try: + mod_commit = repo.commit(args.rev) + except: + failure(args.rev,"is not a valid commit") + try: + base_commit = repo.commit(args.rev_base) + except: + failure(args.rev_base,"is not a valid commit") + + mod_tree = mod_commit.tree + base_tree = base_commit.tree + + xui_path = "indra/newview/skins/default/xui/{}".format(args.base_lang) + try: + mod_xui_tree = mod_tree[xui_path] + except: + failure("xui tree not found for language", args.base_lang) + + data = [] + # For all files to be checked for translations + for mod_blob in mod_xui_tree.traverse(): + filename = mod_blob.path + if mod_blob.type == "tree": # directory, skip + continue + + if args.verbose: + print(filename) + try: + base_blob = base_tree[filename] + except: + if args.verbose: + print("No matching base file found for", filename) + base_blob = None + + try: + transl_filename = filename.replace(args.base_lang, args.lang) + transl_blob = mod_tree[transl_filename] + except: + if args.verbose: + print("No matching translation file found at", transl_filename) + transl_blob = None + + mod_dict = read_xml_elements(mod_blob) + base_dict = read_xml_elements(base_blob) + transl_dict = read_xml_elements(transl_blob) + + rows = 0 + for name in mod_dict.keys(): + if not name in base_dict or mod_dict[name].text != base_dict[name].text: + val = mod_dict[name].text + if can_translate(val): + transl_val = "--" + if name in transl_dict: + transl_val = transl_dict[name].text + data.append([filename, name, "text", val, transl_val, ""]) + rows += 1 + for attr in translate_attribs: + if attr in mod_dict[name].attrib: + if name not in base_dict or attr not in base_dict[name].attrib or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]: + val = mod_dict[name].attrib[attr] + if can_translate(val): + transl_val = "--" + if name in transl_dict and attr in transl_dict[name].attrib: + transl_val = transl_dict[name].attrib[attr] + data.append([filename, name, attr, val, transl_val, ""]) + rows += 1 + if args.verbose and rows>0: + print(" ",rows,"rows added") + + outfile = "SL_Translations_{}.xlsx".format(args.lang.upper()) + cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(args.lang.upper()), "ENTER NEW TRANSLATION ({})".format(args.lang.upper())] + num_translations = len(data) + df = pd.DataFrame(data, columns=cols) + df.to_excel(outfile, index=False) + if num_translations>0: + print("Wrote", num_translations, "rows to file", outfile) + else: + print("Nothing to translate,", outfile, "is empty") -- cgit v1.2.3 From cef702d0c57e5bb4f0debfba677d1fd5a9694364 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 14 Aug 2020 14:11:34 +0100 Subject: SL-13705 - bug fixes and comments --- scripts/code_tools/modified_strings.py | 60 +++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 16 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index dc3357fe8e..77954637a4 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -1,6 +1,7 @@ +#!/usr/bin/env python """\ -This module contains tools for scanning the SL codebase for translation-related strings. +This script scans the SL codebase for translation-related strings. $LicenseInfo:firstyear=2020&license=viewerlgpl$ Second Life Viewer Source Code @@ -26,8 +27,6 @@ $/LicenseInfo$ from __future__ import print_function -# packages required include: gitpython, pandas - import xml.etree.ElementTree as ET import argparse import os @@ -61,6 +60,8 @@ def read_xml_elements(blob): try: contents = blob.data_stream.read() except: + # default - pretend we read a file with no elements of interest. + # Parser will complain if it gets no elements at all. contents = '' xml = ET.fromstring(contents) elts = {} @@ -83,23 +84,37 @@ def can_translate(val): if val.isdigit(): return False return True - + +usage_msg="""%(prog)s [options] + +Analyze the XUI configuration files to find text that may need to +be translated. Works by comparing two specified revisions, one +specified by --rev (default HEAD) and one specified by --rev_base +(default master). The script works by comparing xui contents of the +two revisions, and outputs a spreadsheet listing any areas of +difference. The target language must be specified using the --lang +option. Output is an excel file, which can be used as-is or imported +into google sheets. + +If the --rev revision already contains a translation for the text, it +will be included in the spreadsheet for reference. + +Normally you would want --rev_base to be the last revision to have +translations added, and --rev to be the tip of the current +project. + +""" + if __name__ == "__main__": - parser = argparse.ArgumentParser(description="analyze viewer xui files") + parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg) parser.add_argument("-v","--verbose", action="store_true", help="verbose flag") parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en") - parser.add_argument("--lang", help="target language, default fr", default="fr") + parser.add_argument("--lang", help="target language") args = parser.parse_args() - if args.rev == args.rev_base: - failure("Revs are the same, nothing to compare") - - print("Finding changes in", args.rev, "not present in", args.rev_base) - sys.stdout.flush() - cwd = os.getcwd() rootdir = Git(cwd).rev_parse("--show-toplevel") repo = Repo(rootdir) @@ -115,11 +130,23 @@ if __name__ == "__main__": mod_tree = mod_commit.tree base_tree = base_commit.tree - xui_path = "indra/newview/skins/default/xui/{}".format(args.base_lang) + xui_base = "indra/newview/skins/default/xui" + xui_base_tree = mod_tree[xui_base] + valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()] + if not args.lang or not args.lang.lower() in valid_langs: + failure("Please specify a target language using --lang. Valid values are", ",".join(sorted(valid_langs))) + + xui_path = "{}/{}".format(xui_base, args.base_lang) try: mod_xui_tree = mod_tree[xui_path] except: - failure("xui tree not found for language", args.base_lang) + failure("xui tree not found for base language", args.base_lang) + + if args.rev == args.rev_base: + failure("Revs are the same, nothing to compare") + + print("Finding changes in", args.rev, "not present in", args.rev_base) + sys.stdout.flush() data = [] # For all files to be checked for translations @@ -130,6 +157,7 @@ if __name__ == "__main__": if args.verbose: print(filename) + try: base_blob = base_tree[filename] except: @@ -138,11 +166,11 @@ if __name__ == "__main__": base_blob = None try: - transl_filename = filename.replace(args.base_lang, args.lang) + transl_filename = filename.replace("/xui/{}/".format(args.base_lang), "/xui/{}/".format(args.lang)) transl_blob = mod_tree[transl_filename] except: if args.verbose: - print("No matching translation file found at", transl_filename) + failure("No matching translation file found at", transl_filename) transl_blob = None mod_dict = read_xml_elements(mod_blob) -- cgit v1.2.3 From 932f66929db5e7a5afae1f547408388cdc680709 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Thu, 10 Sep 2020 18:58:41 +0100 Subject: SL-13705 - support multiple or all languages. more logic for excluding irrelevant fields --- scripts/code_tools/modified_strings.py | 130 ++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 42 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index 77954637a4..eee20cf83b 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -33,6 +33,7 @@ import os import sys from git import Repo, Git # requires the gitpython package import pandas as pd +import re translate_attribs = [ "title", @@ -75,14 +76,23 @@ def failure(*msg): print(*msg) sys.exit(1) -def can_translate(val): +def should_translate(filename, val): if val is None: return False + if "floater_test" in filename: + return False + if "TestString PleaseIgnore" in val: + return False + if len(val) == 0: + return False if val.isspace(): return False val = val.strip() if val.isdigit(): return False + if re.match(r"^\s*\d*\s*x\s*\d*\s*$", val): + print(val, "matches resolution string, will ignore") + return False return True usage_msg="""%(prog)s [options] @@ -105,36 +115,7 @@ project. """ -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg) - parser.add_argument("-v","--verbose", action="store_true", help="verbose flag") - parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") - parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") - parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en") - parser.add_argument("--lang", help="target language") - args = parser.parse_args() - - cwd = os.getcwd() - rootdir = Git(cwd).rev_parse("--show-toplevel") - repo = Repo(rootdir) - try: - mod_commit = repo.commit(args.rev) - except: - failure(args.rev,"is not a valid commit") - try: - base_commit = repo.commit(args.rev_base) - except: - failure(args.rev_base,"is not a valid commit") - - mod_tree = mod_commit.tree - base_tree = base_commit.tree - - xui_base = "indra/newview/skins/default/xui" - xui_base_tree = mod_tree[xui_base] - valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()] - if not args.lang or not args.lang.lower() in valid_langs: - failure("Please specify a target language using --lang. Valid values are", ",".join(sorted(valid_langs))) +def make_translation_spreadsheet(mod_tree, base_tree, lang, args): xui_path = "{}/{}".format(xui_base, args.base_lang) try: @@ -145,11 +126,10 @@ if __name__ == "__main__": if args.rev == args.rev_base: failure("Revs are the same, nothing to compare") - print("Finding changes in", args.rev, "not present in", args.rev_base) - sys.stdout.flush() data = [] # For all files to be checked for translations + all_en_strings = set() for mod_blob in mod_xui_tree.traverse(): filename = mod_blob.path if mod_blob.type == "tree": # directory, skip @@ -166,7 +146,7 @@ if __name__ == "__main__": base_blob = None try: - transl_filename = filename.replace("/xui/{}/".format(args.base_lang), "/xui/{}/".format(args.lang)) + transl_filename = filename.replace("/xui/{}/".format(args.base_lang), "/xui/{}/".format(lang)) transl_blob = mod_tree[transl_filename] except: if args.verbose: @@ -179,29 +159,43 @@ if __name__ == "__main__": rows = 0 for name in mod_dict.keys(): - if not name in base_dict or mod_dict[name].text != base_dict[name].text: + if not name in base_dict or mod_dict[name].text != base_dict[name].text or (args.missing and not name in transl_dict): val = mod_dict[name].text - if can_translate(val): + if should_translate(filename, val): transl_val = "--" if name in transl_dict: transl_val = transl_dict[name].text - data.append([filename, name, "text", val, transl_val, ""]) + if val in all_en_strings: + new_val = "(DUPLICATE)" + else: + new_val = "" + data.append([filename, name, "text", val, transl_val, new_val]) + all_en_strings.add(val) rows += 1 for attr in translate_attribs: if attr in mod_dict[name].attrib: - if name not in base_dict or attr not in base_dict[name].attrib or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]: + if name not in base_dict \ + or attr not in base_dict[name].attrib \ + or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr] \ + or (args.missing and (not name in transl_dict or not attr in transl_dict[name].attrib)): val = mod_dict[name].attrib[attr] - if can_translate(val): + if should_translate(filename, val): + show_val = val transl_val = "--" if name in transl_dict and attr in transl_dict[name].attrib: transl_val = transl_dict[name].attrib[attr] - data.append([filename, name, attr, val, transl_val, ""]) + if val in all_en_strings: + new_val = "(DUPLICATE)" + else: + new_val = "" + data.append([filename, name, attr, show_val, transl_val, new_val]) + all_en_strings.add(val) rows += 1 if args.verbose and rows>0: print(" ",rows,"rows added") - outfile = "SL_Translations_{}.xlsx".format(args.lang.upper()) - cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(args.lang.upper()), "ENTER NEW TRANSLATION ({})".format(args.lang.upper())] + outfile = "SL_Translations_{}.xlsx".format(lang.upper()) + cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper())] num_translations = len(data) df = pd.DataFrame(data, columns=cols) df.to_excel(outfile, index=False) @@ -209,3 +203,55 @@ if __name__ == "__main__": print("Wrote", num_translations, "rows to file", outfile) else: print("Nothing to translate,", outfile, "is empty") + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg) + parser.add_argument("-v","--verbose", action="store_true", help="verbose flag") + parser.add_argument("--missing", action="store_true", default = False, help="include all fields for which a translation does not exist") + parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") + parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") + parser.add_argument("--base_lang", help="base language, default en (normally leave unchanged - other values are only useful for testing)", default="en") + parser.add_argument("--lang", help="target languages, or all", nargs="+") + args = parser.parse_args() + + cwd = os.getcwd() + rootdir = Git(cwd).rev_parse("--show-toplevel") + repo = Repo(rootdir) + try: + mod_commit = repo.commit(args.rev) + except: + failure(args.rev,"is not a valid commit") + try: + base_commit = repo.commit(args.rev_base) + except: + failure(args.rev_base,"is not a valid commit") + + print("Will identify changes in", args.rev, "not present in", args.rev_base) + if args.missing: + print("Will also include any text for which no corresponding translation exists, regardless of when it was added") + sys.stdout.flush() + + mod_tree = mod_commit.tree + base_tree = base_commit.tree + + xui_base = "indra/newview/skins/default/xui" + xui_base_tree = mod_tree[xui_base] + valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()] + langs = [l.lower() for l in args.lang] + if "all" in args.lang: + langs = valid_langs + + for lang in langs: + if not lang in valid_langs: + failure("Unknown target language {}. Valid values are {} or all".format(lang,",".join(sorted(valid_langs)))) + + print("Target language(s) are", ",".join(sorted(langs))) + sys.stdout.flush() + + for lang in langs: + print("Creating spreadsheet for language", lang) + sys.stdout.flush() + + make_translation_spreadsheet(mod_tree, base_tree, lang, args) + -- cgit v1.2.3 From 3788fdbb03f85492f4bd26cb4637434ac3e8515e Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Thu, 10 Sep 2020 19:53:13 +0100 Subject: SL-13705 - fixes and spreadsheet format changes --- scripts/code_tools/modified_strings.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index eee20cf83b..3ea13a2bf5 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -169,7 +169,8 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): new_val = "(DUPLICATE)" else: new_val = "" - data.append([filename, name, "text", val, transl_val, new_val]) + field = "text" + data.append([val, transl_val, new_val, filename, name, field]) all_en_strings.add(val) rows += 1 for attr in translate_attribs: @@ -178,9 +179,12 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): or attr not in base_dict[name].attrib \ or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr] \ or (args.missing and (not name in transl_dict or not attr in transl_dict[name].attrib)): - val = mod_dict[name].attrib[attr] + elt = mod_dict[name] + val = elt.attrib[attr] + #if attr == "value" and elt.tag not in ["string","text"]: + # print("skipping value attribute", val, "tag", elt.tag, "in", filename) + # continue if should_translate(filename, val): - show_val = val transl_val = "--" if name in transl_dict and attr in transl_dict[name].attrib: transl_val = transl_dict[name].attrib[attr] @@ -188,14 +192,15 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): new_val = "(DUPLICATE)" else: new_val = "" - data.append([filename, name, attr, show_val, transl_val, new_val]) + field = attr + data.append([val, transl_val, new_val, filename, name, field]) all_en_strings.add(val) rows += 1 if args.verbose and rows>0: print(" ",rows,"rows added") outfile = "SL_Translations_{}.xlsx".format(lang.upper()) - cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper())] + cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"] num_translations = len(data) df = pd.DataFrame(data, columns=cols) df.to_excel(outfile, index=False) -- cgit v1.2.3 From 1eaa76f2f09557e1fc828775249d6eece236572d Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 11 Sep 2020 19:37:04 +0100 Subject: SL-13705 - quieter output --- scripts/code_tools/modified_strings.py | 48 +++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index 3ea13a2bf5..ec1c050a8f 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -76,13 +76,24 @@ def failure(*msg): print(*msg) sys.exit(1) -def should_translate(filename, val): +# return True iff any element of lis is "in" thing +def has_any(thing,lis): + for l in lis: + if l in thing: + return True + return False + +def should_translate(filename, elt, field, val): if val is None: return False - if "floater_test" in filename: + # Should translate apply recursively? + if "translate" in elt.attrib and elt.attrib["translate"] == "false": + return False + if has_any(filename,["floater_test","floater_aaa","floater_ui_preview"]): return False if "TestString PleaseIgnore" in val: return False + val = re.sub(r"\[.*?\]","",val) if len(val) == 0: return False if val.isspace(): @@ -90,9 +101,22 @@ def should_translate(filename, val): val = val.strip() if val.isdigit(): return False + if not re.search('\w+', val): + return False if re.match(r"^\s*\d*\s*x\s*\d*\s*$", val): - print(val, "matches resolution string, will ignore") + #print(val, "matches resolution string, will ignore") return False + # "value" is a hairball, mostly used to encode non-display info but a few exceptions + if field == "value": + if elt.text is not None and len(elt.text) > 0: + #print("value has text, ignoring", ET.tostring(elt)) + return False + if has_any(elt.attrib,["label"]): + return False + if elt.tag in ["string","text"]: + return True + #print("including value attribute", val, "tag", elt.tag,"in", ET.tostring(elt)) + return True return True usage_msg="""%(prog)s [options] @@ -150,7 +174,7 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): transl_blob = mod_tree[transl_filename] except: if args.verbose: - failure("No matching translation file found at", transl_filename) + print("No matching translation file found at", transl_filename) transl_blob = None mod_dict = read_xml_elements(mod_blob) @@ -160,8 +184,10 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): rows = 0 for name in mod_dict.keys(): if not name in base_dict or mod_dict[name].text != base_dict[name].text or (args.missing and not name in transl_dict): - val = mod_dict[name].text - if should_translate(filename, val): + elt = mod_dict[name] + val = elt.text + field = "text" + if should_translate(filename, elt, field, val): transl_val = "--" if name in transl_dict: transl_val = transl_dict[name].text @@ -169,7 +195,6 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): new_val = "(DUPLICATE)" else: new_val = "" - field = "text" data.append([val, transl_val, new_val, filename, name, field]) all_en_strings.add(val) rows += 1 @@ -181,10 +206,7 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): or (args.missing and (not name in transl_dict or not attr in transl_dict[name].attrib)): elt = mod_dict[name] val = elt.attrib[attr] - #if attr == "value" and elt.tag not in ["string","text"]: - # print("skipping value attribute", val, "tag", elt.tag, "in", filename) - # continue - if should_translate(filename, val): + if should_translate(filename, elt, attr, val): transl_val = "--" if name in transl_dict and attr in transl_dict[name].attrib: transl_val = transl_dict[name].attrib[attr] @@ -192,8 +214,8 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): new_val = "(DUPLICATE)" else: new_val = "" - field = attr - data.append([val, transl_val, new_val, filename, name, field]) + #attr = attr + ":" + ET.tostring(elt) + data.append([val, transl_val, new_val, filename, name, attr]) all_en_strings.add(val) rows += 1 if args.verbose and rows>0: -- cgit v1.2.3 From 822a4fb7d2645a4167b987995a190169133cff42 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 11 Sep 2020 21:47:37 +0100 Subject: SL-13705 - Additional cell/column properties specified in excel output --- scripts/code_tools/modified_strings.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index ec1c050a8f..b77115ed46 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -106,7 +106,7 @@ def should_translate(filename, elt, field, val): if re.match(r"^\s*\d*\s*x\s*\d*\s*$", val): #print(val, "matches resolution string, will ignore") return False - # "value" is a hairball, mostly used to encode non-display info but a few exceptions + # "value" attribute is a hairball, mostly used to encode non-display info but a few exceptions if field == "value": if elt.text is not None and len(elt.text) > 0: #print("value has text, ignoring", ET.tostring(elt)) @@ -218,18 +218,40 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): data.append([val, transl_val, new_val, filename, name, attr]) all_en_strings.add(val) rows += 1 - if args.verbose and rows>0: - print(" ",rows,"rows added") + save_as_excel(data, lang) + + +def save_as_excel(data, lang): + outfile = "SL_Translations_{}.xlsx".format(lang.upper()) - cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"] num_translations = len(data) + cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"] df = pd.DataFrame(data, columns=cols) - df.to_excel(outfile, index=False) + + writer = pd.ExcelWriter(outfile, engine='xlsxwriter') + df.to_excel(writer, index=False, sheet_name = "Sheet1") + + workbook = writer.book + worksheet = writer.sheets['Sheet1'] + + cell_format = workbook.add_format({'text_wrap': True}) + + # Translators primarily care about columns A-C + worksheet.set_column('A:C', 100, cell_format) + worksheet.set_column('D:D', 50, cell_format, {'hidden': True}) + worksheet.set_column('E:F', 30, cell_format, {'hidden': True}) + + # Lock the column header in place while scrolling + worksheet.freeze_panes(1, 0) + + writer.save() + if num_translations>0: print("Wrote", num_translations, "rows to file", outfile) else: print("Nothing to translate,", outfile, "is empty") + if __name__ == "__main__": -- cgit v1.2.3 From a1d24974e808c3a0fb6072d010ccabbc38703232 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Mon, 14 Sep 2020 20:34:51 +0100 Subject: SL-13705 - lock all columns except C in the excel file (apparently google sheets does not respect this setting on import) --- scripts/code_tools/modified_strings.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index b77115ed46..c259a0f984 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -235,14 +235,18 @@ def save_as_excel(data, lang): workbook = writer.book worksheet = writer.sheets['Sheet1'] - cell_format = workbook.add_format({'text_wrap': True}) - - # Translators primarily care about columns A-C - worksheet.set_column('A:C', 100, cell_format) - worksheet.set_column('D:D', 50, cell_format, {'hidden': True}) - worksheet.set_column('E:F', 30, cell_format, {'hidden': True}) - - # Lock the column header in place while scrolling + wrap_format = workbook.add_format({'text_wrap': True}) + wrap_unlocked_format = workbook.add_format({'text_wrap': True, 'locked': False}) + + # Translators primarily care about columns A-C, and should write + # only in column C. Can hide the others. Set widths. + worksheet.protect() + worksheet.set_column('A:B', 100, wrap_format) + worksheet.set_column('C:C', 100, wrap_unlocked_format) + worksheet.set_column('D:D', 50, wrap_format, {'hidden': True}) + worksheet.set_column('E:F', 30, wrap_format, {'hidden': True}) + + # Lock the top row (column headers) in place while scrolling worksheet.freeze_panes(1, 0) writer.save() -- cgit v1.2.3 From a74b366d22f82447919751c04815e070bc5307c7 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 15 Sep 2020 13:53:39 +0100 Subject: SL-13705 - single output file with one tab per translation language --- scripts/code_tools/modified_strings.py | 79 +++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 34 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index c259a0f984..70ca3e1ae6 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -139,7 +139,7 @@ project. """ -def make_translation_spreadsheet(mod_tree, base_tree, lang, args): +def make_translation_table(mod_tree, base_tree, lang, args): xui_path = "{}/{}".format(xui_base, args.base_lang) try: @@ -195,7 +195,7 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): new_val = "(DUPLICATE)" else: new_val = "" - data.append([val, transl_val, new_val, filename, name, field]) + data.append([val, transl_val, new_val, "", "", filename, name, field]) all_en_strings.add(val) rows += 1 for attr in translate_attribs: @@ -215,48 +215,47 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args): else: new_val = "" #attr = attr + ":" + ET.tostring(elt) - data.append([val, transl_val, new_val, filename, name, attr]) + data.append([val, transl_val, new_val, "", "", filename, name, attr]) all_en_strings.add(val) rows += 1 + return data - save_as_excel(data, lang) +def save_translation_file(all_data, outfile): - -def save_as_excel(data, lang): - - outfile = "SL_Translations_{}.xlsx".format(lang.upper()) - num_translations = len(data) - cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"] - df = pd.DataFrame(data, columns=cols) + langs = sorted(all_data.keys()) + print("Saving languages", ",".join(langs),"as",outfile) writer = pd.ExcelWriter(outfile, engine='xlsxwriter') - df.to_excel(writer, index=False, sheet_name = "Sheet1") workbook = writer.book - worksheet = writer.sheets['Sheet1'] - wrap_format = workbook.add_format({'text_wrap': True}) wrap_unlocked_format = workbook.add_format({'text_wrap': True, 'locked': False}) - # Translators primarily care about columns A-C, and should write - # only in column C. Can hide the others. Set widths. - worksheet.protect() - worksheet.set_column('A:B', 100, wrap_format) - worksheet.set_column('C:C', 100, wrap_unlocked_format) - worksheet.set_column('D:D', 50, wrap_format, {'hidden': True}) - worksheet.set_column('E:F', 30, wrap_format, {'hidden': True}) - - # Lock the top row (column headers) in place while scrolling - worksheet.freeze_panes(1, 0) - + for lang in langs: + data = all_data[lang] + num_translations = len(data) + cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "Translator Questions", "Notes", "File", "Element", "Field"] + df = pd.DataFrame(data, columns=cols) + df.to_excel(writer, index=False, sheet_name = lang.upper()) + + worksheet = writer.sheets[lang.upper()] + + # Translators primarily care about columns A-C, and should write + # only in column C. Hide the others. Set widths. + worksheet.protect() + worksheet.set_column('A:B', 60, wrap_format) + worksheet.set_column('C:C', 60, wrap_unlocked_format) + worksheet.set_column('D:E', 40, wrap_unlocked_format) + worksheet.set_column('F:F', 50, wrap_format, {'hidden': True}) + worksheet.set_column('G:H', 30, wrap_format, {'hidden': True}) + + # Lock the top row (column headers) in place while scrolling + worksheet.freeze_panes(1, 0) + print("Added", num_translations, "rows for language", lang) + + print("Writing", outfile) writer.save() - if num_translations>0: - print("Wrote", num_translations, "rows to file", outfile) - else: - print("Nothing to translate,", outfile, "is empty") - - if __name__ == "__main__": parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg) @@ -290,21 +289,33 @@ if __name__ == "__main__": xui_base = "indra/newview/skins/default/xui" xui_base_tree = mod_tree[xui_base] + + # Find target languages valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()] langs = [l.lower() for l in args.lang] if "all" in args.lang: langs = valid_langs - + langs = sorted(langs) for lang in langs: if not lang in valid_langs: failure("Unknown target language {}. Valid values are {} or all".format(lang,",".join(sorted(valid_langs)))) - print("Target language(s) are", ",".join(sorted(langs))) sys.stdout.flush() + outfile = "SL_Translations.xlsx" + try: + f = open(outfile,"a+") + f.close() + except: + failure("Can't write to output file",outfile,". Is it already open?") + + all_data = {} for lang in langs: print("Creating spreadsheet for language", lang) sys.stdout.flush() - make_translation_spreadsheet(mod_tree, base_tree, lang, args) + all_data[lang] = make_translation_table(mod_tree, base_tree, lang, args) + + print("Saving output file", outfile) + save_translation_file(all_data, outfile) -- cgit v1.2.3 From 713d896d3664c8266a7b08feea480735d2a8b390 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 15 Sep 2020 15:49:26 +0100 Subject: SL-13705 - added metadata tab to translation spreadsheet, showing invocation info --- scripts/code_tools/modified_strings.py | 70 +++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 26 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index 70ca3e1ae6..1dbd20f1b4 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -34,6 +34,30 @@ import sys from git import Repo, Git # requires the gitpython package import pandas as pd import re +from datetime import datetime + +usage_msg="""%(prog)s [options] + +Analyze the XUI configuration files to find text that may need to +be translated. Works by comparing two specified revisions, one +specified by --rev (default HEAD) and one specified by --rev_base +(default master). The script works by comparing xui contents of the +two revisions, and outputs a spreadsheet listing any areas of +difference. The target language must be specified using the --lang +option. Output is an excel file, which can be used as-is or imported +into google sheets. + +If the --rev revision already contains a translation for the text, it +will be included in the spreadsheet for reference. + +Normally you would want --rev_base to be the last revision to have +translations added, and --rev to be the tip of the current +project. You can find the last commit with translation work using "git log --grep INTL- | head" + +The --missing argument can be used to find all text with missing +translations, regardless of when it was added. If translations are being kept +reasonably current, you will normally not need this argument. +""" translate_attribs = [ "title", @@ -119,26 +143,6 @@ def should_translate(filename, elt, field, val): return True return True -usage_msg="""%(prog)s [options] - -Analyze the XUI configuration files to find text that may need to -be translated. Works by comparing two specified revisions, one -specified by --rev (default HEAD) and one specified by --rev_base -(default master). The script works by comparing xui contents of the -two revisions, and outputs a spreadsheet listing any areas of -difference. The target language must be specified using the --lang -option. Output is an excel file, which can be used as-is or imported -into google sheets. - -If the --rev revision already contains a translation for the text, it -will be included in the spreadsheet for reference. - -Normally you would want --rev_base to be the last revision to have -translations added, and --rev to be the tip of the current -project. - -""" - def make_translation_table(mod_tree, base_tree, lang, args): xui_path = "{}/{}".format(xui_base, args.base_lang) @@ -220,19 +224,20 @@ def make_translation_table(mod_tree, base_tree, lang, args): rows += 1 return data -def save_translation_file(all_data, outfile): +def save_translation_file(per_lang_data, aux_data, outfile): - langs = sorted(all_data.keys()) + langs = sorted(per_lang_data.keys()) print("Saving languages", ",".join(langs),"as",outfile) writer = pd.ExcelWriter(outfile, engine='xlsxwriter') workbook = writer.book wrap_format = workbook.add_format({'text_wrap': True}) + bold_wrap_format = workbook.add_format({'text_wrap': True, 'bold': True}) wrap_unlocked_format = workbook.add_format({'text_wrap': True, 'locked': False}) for lang in langs: - data = all_data[lang] + data = per_lang_data[lang] num_translations = len(data) cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "Translator Questions", "Notes", "File", "Element", "Field"] df = pd.DataFrame(data, columns=cols) @@ -253,6 +258,14 @@ def save_translation_file(all_data, outfile): worksheet.freeze_panes(1, 0) print("Added", num_translations, "rows for language", lang) + # Reference info, not for translation + for aux, data in aux_data.items(): + df = pd.DataFrame(data, columns = ["Key", "Value"]) + df.to_excel(writer, index=False, sheet_name=aux) + worksheet = writer.sheets[aux] + worksheet.set_column('A:A', 50, bold_wrap_format) + worksheet.set_column('B:B', 80, wrap_format) + print("Writing", outfile) writer.save() @@ -309,13 +322,18 @@ if __name__ == "__main__": except: failure("Can't write to output file",outfile,". Is it already open?") - all_data = {} + aux_data = { "REFERENCE": [["Command", " ".join(sys.argv)], + ["Date", str(datetime.now())], + ["Mod Commit", mod_commit.hexsha], + ["Base Commit", base_commit.hexsha], + ] } + per_lang_data = {} for lang in langs: print("Creating spreadsheet for language", lang) sys.stdout.flush() - all_data[lang] = make_translation_table(mod_tree, base_tree, lang, args) + per_lang_data[lang] = make_translation_table(mod_tree, base_tree, lang, args) print("Saving output file", outfile) - save_translation_file(all_data, outfile) + save_translation_file(per_lang_data, aux_data, outfile) -- cgit v1.2.3 From f20b29dcb780333cba4d6e895ceba549ddfbee80 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Wed, 16 Sep 2020 12:51:59 +0100 Subject: SL-13705 - language list defaults to the standard supported set of FR, ES, IT, PT, JA, DE. '--lang all' will also use this list --- scripts/code_tools/modified_strings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index 1dbd20f1b4..762f1b3845 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -277,7 +277,7 @@ if __name__ == "__main__": parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") parser.add_argument("--base_lang", help="base language, default en (normally leave unchanged - other values are only useful for testing)", default="en") - parser.add_argument("--lang", help="target languages, or all", nargs="+") + parser.add_argument("--lang", help="target languages, or all", nargs="+", default = ["all"]) args = parser.parse_args() cwd = os.getcwd() @@ -305,9 +305,10 @@ if __name__ == "__main__": # Find target languages valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()] + supported_langs = ["fr", "es", "it", "pt", "ja", "de"] langs = [l.lower() for l in args.lang] if "all" in args.lang: - langs = valid_langs + langs = supported_langs langs = sorted(langs) for lang in langs: if not lang in valid_langs: -- cgit v1.2.3 From bf72ffd84b82bf8b7cf28a09a8bc4134e7e5d05a Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 18 Sep 2020 16:22:54 +0100 Subject: SL-13705 - added --deleted option to collect info on no-longer-needed text in translated files. Processing needs to be automated as the list is unfeasibly long. --- scripts/code_tools/modified_strings.py | 87 +++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 12 deletions(-) (limited to 'scripts') diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py index 762f1b3845..6a763b6ec5 100644 --- a/scripts/code_tools/modified_strings.py +++ b/scripts/code_tools/modified_strings.py @@ -149,7 +149,7 @@ def make_translation_table(mod_tree, base_tree, lang, args): try: mod_xui_tree = mod_tree[xui_path] except: - failure("xui tree not found for base language", args.base_lang) + failure("xui tree not found for base language", args.base_lang,"or target lang", lang) if args.rev == args.rev_base: failure("Revs are the same, nothing to compare") @@ -222,8 +222,56 @@ def make_translation_table(mod_tree, base_tree, lang, args): data.append([val, transl_val, new_val, "", "", filename, name, attr]) all_en_strings.add(val) rows += 1 + return data +def find_deletions(mod_tree, base_tree, lang, args, f): + + transl_xui_path = "{}/{}".format(xui_base, lang) + try: + transl_xui_tree = mod_tree[transl_xui_path] + except: + failure("xui tree not found for base language", args.base_lang,"or target lang", lang) + + for transl_blob in transl_xui_tree.traverse(): + if transl_blob.type == "tree": # directory, skip + continue + transl_filename = transl_blob.path + mod_filename = transl_filename.replace("/xui/{}/".format(lang), "/xui/{}/".format(args.base_lang)) + #print("checking",transl_filename,"against",mod_filename) + try: + mod_blob = mod_tree[mod_filename] + except: + print(" delete file", transl_filename, file=f) + continue + mod_dict = read_xml_elements(mod_blob) + if len(mod_dict) == 0: + print(" delete file", transl_filename, file=f) + continue + transl_dict = read_xml_elements(transl_blob) + #print("mod vs transl", len(mod_dict), len(transl_dict)) + lines = 0 + for elt_key in transl_dict: + if not elt_key in mod_dict: + if lines == 0: + print(" in file", transl_filename, file=f) + lines += 1 + print(" delete element", elt_key, file=f) + else: + transl_elt = transl_dict[elt_key] + mod_elt = mod_dict[elt_key] + for a in transl_elt.attrib: + if not a in mod_elt.attrib: + if lines == 0: + print(" in file", transl_filename, file=f) + lines += 1 + print(" delete attribute", a, "from", elt_key, file=f) + if transl_elt.text and (not mod_elt.text): + if lines == 0: + print(" in file", transl_filename, file=f) + lines += 1 + print(" delete text from", elt_key, file=f) + def save_translation_file(per_lang_data, aux_data, outfile): langs = sorted(per_lang_data.keys()) @@ -274,10 +322,12 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg) parser.add_argument("-v","--verbose", action="store_true", help="verbose flag") parser.add_argument("--missing", action="store_true", default = False, help="include all fields for which a translation does not exist") + parser.add_argument("--deleted", action="store_true", default = False, help="show all translated entities which don't exist in english") + parser.add_argument("--skip_spreadsheet", action="store_true", default = False, help="skip creating the translation spreadsheet") parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD") parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master") parser.add_argument("--base_lang", help="base language, default en (normally leave unchanged - other values are only useful for testing)", default="en") - parser.add_argument("--lang", help="target languages, or all", nargs="+", default = ["all"]) + parser.add_argument("--lang", help="target languages, or 'all_valid' or 'supported'; default is 'supported'", nargs="+", default = ["supported"]) args = parser.parse_args() cwd = os.getcwd() @@ -304,15 +354,19 @@ if __name__ == "__main__": xui_base_tree = mod_tree[xui_base] # Find target languages + # all languages present in the codebase valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()] + # offically supported languages supported_langs = ["fr", "es", "it", "pt", "ja", "de"] langs = [l.lower() for l in args.lang] - if "all" in args.lang: + if "supported" in args.lang: langs = supported_langs + if "all_valid" in args.lang: + langs = valid_langs langs = sorted(langs) for lang in langs: if not lang in valid_langs: - failure("Unknown target language {}. Valid values are {} or all".format(lang,",".join(sorted(valid_langs)))) + failure("Unknown target language {}. Valid values are {}".format(lang,", ".join(sorted(valid_langs) + ["all_valid","supported"]))) print("Target language(s) are", ",".join(sorted(langs))) sys.stdout.flush() @@ -328,13 +382,22 @@ if __name__ == "__main__": ["Mod Commit", mod_commit.hexsha], ["Base Commit", base_commit.hexsha], ] } - per_lang_data = {} - for lang in langs: - print("Creating spreadsheet for language", lang) - sys.stdout.flush() - - per_lang_data[lang] = make_translation_table(mod_tree, base_tree, lang, args) - print("Saving output file", outfile) - save_translation_file(per_lang_data, aux_data, outfile) + if not args.skip_spreadsheet: + per_lang_data = {} + for lang in langs: + print("Creating spreadsheet for language", lang) + sys.stdout.flush() + + per_lang_data[lang] = make_translation_table(mod_tree, base_tree, lang, args) + + print("Saving output file", outfile) + save_translation_file(per_lang_data, aux_data, outfile) + + if args.deleted: + deletion_file = "Translate_deletions.txt" + print("Saving deletion info to", deletion_file) + with open(deletion_file,"w") as f: + for lang in langs: + find_deletions(mod_tree, base_tree, lang, args, f) -- cgit v1.2.3