From 59de1d58d4aa891cc19372dd6104ec65e1cf4731 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Tue, 11 Aug 2020 14:14:38 +0100
Subject: SL-13705 - modified-strings.sh added, for translation support
---
scripts/code_tools/modified-strings.sh | 199 +++++++++++++++++++++++++++++++++
1 file changed, 199 insertions(+)
create mode 100644 scripts/code_tools/modified-strings.sh
(limited to 'scripts')
diff --git a/scripts/code_tools/modified-strings.sh b/scripts/code_tools/modified-strings.sh
new file mode 100644
index 0000000000..435dda3f5d
--- /dev/null
+++ b/scripts/code_tools/modified-strings.sh
@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+# $LicenseInfo:firstyear=2014&license=viewerlgpl$
+# Second Life Viewer Source Code
+# Copyright (C) 2011, Linden Research, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation;
+# version 2.1 of the License only.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+#
+# Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
+#
+###
+### Extract strings modified between some version and the current version
+###
+
+Action=DEFAULT
+Rev=master
+DefaultXuiDir="indra/newview/skins/default/xui"
+Verbose=false
+ExitStatus=0
+
+while [ $# -ne 0 ]
+do
+ case ${1} in
+ ##
+ ## Show usage
+ ##
+ -h|--help)
+ Action=USAGE
+ ;;
+
+ -v|--verbose)
+ Verbose=true
+ ;;
+
+ ##
+ ## Select the revision to compare against
+ ##
+ -r)
+ if [ $# -lt 2 ]
+ then
+ echo "Must specify <revision> with ${1}" 1>&2
+ Action=USAGE
+ ExitStatus=1
+ break
+ else
+ Rev=${2}
+ shift # consume the switch ( for n values, consume n-1 )
+ fi
+ ;;
+
+ ##
+ ## handle an unknown switch
+ ##
+ -*)
+ Action=USAGE
+ ExitStatus=1
+ break
+ ;;
+
+ *)
+ if [ -z "${XuiDir}" ]
+ then
+ XuiDir=${1}
+ else
+ echo "Too many arguments supplied: $@" 1>&2
+ Action=USAGE
+ ExitStatus=1
+ break
+ fi
+ ;;
+ esac
+
+ shift # always consume 1
+done
+
+progress()
+{
+ if $Verbose
+ then
+ echo $* 1>&2
+ fi
+}
+
+if [[ $ExitStatus -eq 0 && "${Action}" = "DEFAULT" ]]
+then
+ if [[ ! -d "${XuiDir:=$DefaultXuiDir}" ]]
+ then
+ echo "No XUI directory found in '$XuiDir'" 1>&2
+ Action=USAGE
+ ExitStatus=1
+ fi
+fi
+
+if [ "${Action}" = "USAGE" ]
+then
+ cat <<USAGE
+
+Usage:
+
+ modified-strings.sh [ { -v | --verbose } ] [-r <revision>] [<path-to-xui>]
+
+ where
+ --verbose shows progress messages on stderr (the command takes a while, so this is reassuring)
+
+ -r <revision> specifies a git revision (branch, tag, commit, or relative specifier)
+ defaults to 'master' so that comparison is against the HEAD of the released viewer branch
+
+ <path-to-xui> is the path to the root directory for XUI files
+ defaults to '$DefaultXuiDir'
+
+ Emits a tab-separated file with these columns:
+ filename
+ the path of a file that has a string change (columns 2 and 3 are empty for lines with a filename)
+ name
+ the name attribute of a string or label whose value changed
+ English value
+ the current value of the string or label whose value changed
+ for strings, newlines are changed to '\n' and tab characters are changed to '\t'
+
+ There is also a column for each of the language directories following the English.
+
+USAGE
+ exit $ExitStatus
+fi
+
+stringval() # reads stdin and prints the escaped value of a string for the requested tag
+{
+ local tag=$1
+ xmllint --xpath "string(/strings/string[@name=\"$tag\"])" - | perl -p -e 'chomp; s/\n/\\n/g; s/\t/\\t/g;'
+}
+
+columns="file\tname\tEN"
+for lang in $(ls -1 ${XuiDir})
+do
+ if [[ "$lang" != "en" && -d "${XuiDir}" && -f "${XuiDir}/$lang/strings.xml" ]]
+ then
+ columns+="\t$lang"
+ fi
+done
+echo -e "$columns"
+
+EnglishStrings="${XuiDir}/en/strings.xml"
+progress -n "scanning $EnglishStrings "
+echo -e "$EnglishStrings"
+# loop over all tags in the current version of the strings file
+cat "$EnglishStrings" | xmllint --xpath '/strings/string/@name' - | sed 's/ name="//; s/"$//;' \
+| while read name
+do
+ progress -n "."
+ # fetch the $Rev and current values for each tag
+ old_stringval=$(git show "$Rev:$EnglishStrings" 2> /dev/null | stringval "$name")
+ new_stringval=$(cat "$EnglishStrings" | stringval "$name")
+
+ if [[ "$old_stringval" != "$new_stringval" ]]
+ then
+ # the value is different, so print the tag and it's current value separated by a tab
+ echo -e "\t$name\t$new_stringval"
+ fi
+done
+progress ""
+
+# loop over all XUI files other than strings.xml finding labels
+grep -rlw 'label' "${XuiDir}/en" | grep -v '/strings.xml' \
+| while read xuipath
+do
+ progress -n "scanning $xuipath "
+ listed_file=false
+ # loop over all elements for which there is a label attribute, getting the name attribute value
+ xmllint --xpath '//*[@label]/@name' "$xuipath" 2> /dev/null | sed 's/ name="//; s/"$//;' \
+ | while read name
+ do
+ progress -n "."
+ # get the old and new label attribute values for each name
+ old_label=$(git show "$Rev:$xuipath" 2> /dev/null | xmllint --xpath "string(//*[@name=\"${name}\"]/@label)" - 2> /dev/null)
+ new_label=$(cat "$xuipath" | xmllint --xpath "string(//*[@name=\"${name}\"]/@label)" - 2> /dev/null)
+ if [[ "$old_label" != "$new_label" ]]
+ then
+ if ! $listed_file
+ then
+ echo -e "$xuipath"
+ listed_file=true
+ fi
+ echo -e "\t$name\t$new_label"
+ fi
+ done
+ progress ""
+done
+
--
cgit v1.2.3
From 1f9852f04c4c430631d10794cce4a3e8186470b5 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Wed, 12 Aug 2020 18:27:00 +0100
Subject: SL-13705 - modified_strings.py added, for translation support
---
scripts/code_tools/modified_strings.py | 158 +++++++++++++++++++++++++++++++++
1 file changed, 158 insertions(+)
create mode 100644 scripts/code_tools/modified_strings.py
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
new file mode 100644
index 0000000000..bb42628f55
--- /dev/null
+++ b/scripts/code_tools/modified_strings.py
@@ -0,0 +1,158 @@
+"""\
+
+This module contains tools for scanning the SL codebase for translation-related strings.
+
+$LicenseInfo:firstyear=2020&license=viewerlgpl$
+Second Life Viewer Source Code
+Copyright (C) 2020, Linden Research, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation;
+version 2.1 of the License only.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
+$/LicenseInfo$
+"""
+
+from __future__ import print_function
+
+# packages required include: gitpython, pandas
+
+import xml.etree.ElementTree as ET
+import argparse
+import os
+import sys
+from git import Repo, Git # requires the gitpython package
+import pandas as pd
+
+translate_attribs = [
+ "title",
+ "short_title",
+ "value",
+ "label",
+ "label_selected",
+ "tool_tip",
+ "ignoretext",
+ "yestext",
+ "notext",
+ "canceltext",
+ "description",
+ "longdescription"
+]
+
+def codify(val):
+ if isinstance(val, unicode):
+ return val.encode("utf-8")
+ else:
+ return unicode(val, 'utf-8').encode("utf-8")
+
+def failure(*msg):
+ print(*msg)
+ sys.exit(1)
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description="analyze viewer xui files")
+ parser.add_argument("--verbose", action="store_true", help="verbose flag")
+ parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
+ parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
+ parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en")
+ parser.add_argument("--lang", help="target language, default fr", default="fr")
+ #parser.add_argument("infilename", help="name of input file", nargs="?")
+ args = parser.parse_args()
+
+ #root = ET.parse(args.infilename)
+
+ #for child in root.iter("string"):
+ # print child.attrib["name"], "\t", unicode(child.text, 'utf-8').encode("utf-8")
+ # #print unicode(child.text, 'utf-8')
+ # #print u'\u0420\u043e\u0441\u0441\u0438\u044f'.encode("utf-8")
+
+ if args.rev == args.rev_base:
+ failure("Revs are the same, nothing to compare")
+
+ print("Finding changes in", args.rev, "not present in", args.rev_base)
+
+ cwd = os.getcwd()
+ rootdir = Git(cwd).rev_parse("--show-toplevel")
+ repo = Repo(rootdir)
+ try:
+ mod_commit = repo.commit(args.rev)
+ except:
+ failure(args.rev,"is not a valid commit")
+ try:
+ base_commit = repo.commit(args.rev_base)
+ except:
+ failure(args.rev_base,"is not a valid commit")
+
+ mod_tree = mod_commit.tree
+ base_tree = base_commit.tree
+
+ all_attrib = set()
+
+ try:
+ mod_xui_tree = mod_tree["indra/newview/skins/default/xui/{}".format(args.base_lang)]
+ except:
+ print("xui tree not found for language", args.base_lang)
+ sys.exit(1)
+
+ data = []
+ # For all files to be checked for translations
+ for mod_blob in mod_xui_tree.traverse():
+ print(mod_blob.path)
+ filename = mod_blob.path
+ if mod_blob.type == "tree": # directory, skip
+ continue
+
+ mod_contents = mod_blob.data_stream.read()
+ try:
+ base_blob = base_tree[filename]
+ base_contents = base_blob.data_stream.read()
+ except:
+ print("No matching base file found for", filename)
+ base_contents = '<?xml version="1.0" encoding="utf-8" standalone="yes" ?><strings></strings>'
+
+ mod_xml = ET.fromstring(mod_contents)
+ base_xml = ET.fromstring(base_contents)
+
+ mod_dict = {}
+ for child in mod_xml.iter():
+ if "name" in child.attrib:
+ name = child.attrib['name']
+ mod_dict[name] = child
+ base_dict = {}
+ for child in base_xml.iter():
+ if "name" in child.attrib:
+ name = child.attrib['name']
+ base_dict[name] = child
+ for name in mod_dict.keys():
+ if not name in base_dict or mod_dict[name].text != base_dict[name].text:
+ data.append([filename, name, "text", mod_dict[name].text,""])
+ #print(" ", name, "text", codify(mod_dict[name].text))
+ all_attrib = all_attrib.union(set(mod_dict[name].attrib.keys()))
+ for attr in translate_attribs:
+ if attr in mod_dict[name].attrib:
+ if name not in base_dict or attr not in base_dict[name] or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]:
+ val = mod_dict[name].attrib[attr]
+ data.append([filename, name, attr, mod_dict[name].attrib[attr],""])
+ #print(" ", name, attr, codify(val))
+
+ cols = ["File", "Element", "Field", "EN", "Translation ({})".format(args.lang)]
+ df = pd.DataFrame(data, columns=cols)
+ df.to_excel("SL_Translations_{}.xlsx".format(args.lang.upper()), index=False)
+
+ #print "all_attrib", all_attrib
+
+
+
+
--
cgit v1.2.3
From f28437adc7ae856d55edfe77596f5ce7331778f4 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Thu, 13 Aug 2020 14:54:45 +0100
Subject: SL-13705 - modified_strings.py handles more cases
---
scripts/code_tools/modified_strings.py | 341 ++++++++++++++++++---------------
1 file changed, 183 insertions(+), 158 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index bb42628f55..dc3357fe8e 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -1,158 +1,183 @@
-"""\
-
-This module contains tools for scanning the SL codebase for translation-related strings.
-
-$LicenseInfo:firstyear=2020&license=viewerlgpl$
-Second Life Viewer Source Code
-Copyright (C) 2020, Linden Research, Inc.
-
-This library is free software; you can redistribute it and/or
-modify it under the terms of the GNU Lesser General Public
-License as published by the Free Software Foundation;
-version 2.1 of the License only.
-
-This library is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
-$/LicenseInfo$
-"""
-
-from __future__ import print_function
-
-# packages required include: gitpython, pandas
-
-import xml.etree.ElementTree as ET
-import argparse
-import os
-import sys
-from git import Repo, Git # requires the gitpython package
-import pandas as pd
-
-translate_attribs = [
- "title",
- "short_title",
- "value",
- "label",
- "label_selected",
- "tool_tip",
- "ignoretext",
- "yestext",
- "notext",
- "canceltext",
- "description",
- "longdescription"
-]
-
-def codify(val):
- if isinstance(val, unicode):
- return val.encode("utf-8")
- else:
- return unicode(val, 'utf-8').encode("utf-8")
-
-def failure(*msg):
- print(*msg)
- sys.exit(1)
-
-if __name__ == "__main__":
-
- parser = argparse.ArgumentParser(description="analyze viewer xui files")
- parser.add_argument("--verbose", action="store_true", help="verbose flag")
- parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
- parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
- parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en")
- parser.add_argument("--lang", help="target language, default fr", default="fr")
- #parser.add_argument("infilename", help="name of input file", nargs="?")
- args = parser.parse_args()
-
- #root = ET.parse(args.infilename)
-
- #for child in root.iter("string"):
- # print child.attrib["name"], "\t", unicode(child.text, 'utf-8').encode("utf-8")
- # #print unicode(child.text, 'utf-8')
- # #print u'\u0420\u043e\u0441\u0441\u0438\u044f'.encode("utf-8")
-
- if args.rev == args.rev_base:
- failure("Revs are the same, nothing to compare")
-
- print("Finding changes in", args.rev, "not present in", args.rev_base)
-
- cwd = os.getcwd()
- rootdir = Git(cwd).rev_parse("--show-toplevel")
- repo = Repo(rootdir)
- try:
- mod_commit = repo.commit(args.rev)
- except:
- failure(args.rev,"is not a valid commit")
- try:
- base_commit = repo.commit(args.rev_base)
- except:
- failure(args.rev_base,"is not a valid commit")
-
- mod_tree = mod_commit.tree
- base_tree = base_commit.tree
-
- all_attrib = set()
-
- try:
- mod_xui_tree = mod_tree["indra/newview/skins/default/xui/{}".format(args.base_lang)]
- except:
- print("xui tree not found for language", args.base_lang)
- sys.exit(1)
-
- data = []
- # For all files to be checked for translations
- for mod_blob in mod_xui_tree.traverse():
- print(mod_blob.path)
- filename = mod_blob.path
- if mod_blob.type == "tree": # directory, skip
- continue
-
- mod_contents = mod_blob.data_stream.read()
- try:
- base_blob = base_tree[filename]
- base_contents = base_blob.data_stream.read()
- except:
- print("No matching base file found for", filename)
- base_contents = '<?xml version="1.0" encoding="utf-8" standalone="yes" ?><strings></strings>'
-
- mod_xml = ET.fromstring(mod_contents)
- base_xml = ET.fromstring(base_contents)
-
- mod_dict = {}
- for child in mod_xml.iter():
- if "name" in child.attrib:
- name = child.attrib['name']
- mod_dict[name] = child
- base_dict = {}
- for child in base_xml.iter():
- if "name" in child.attrib:
- name = child.attrib['name']
- base_dict[name] = child
- for name in mod_dict.keys():
- if not name in base_dict or mod_dict[name].text != base_dict[name].text:
- data.append([filename, name, "text", mod_dict[name].text,""])
- #print(" ", name, "text", codify(mod_dict[name].text))
- all_attrib = all_attrib.union(set(mod_dict[name].attrib.keys()))
- for attr in translate_attribs:
- if attr in mod_dict[name].attrib:
- if name not in base_dict or attr not in base_dict[name] or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]:
- val = mod_dict[name].attrib[attr]
- data.append([filename, name, attr, mod_dict[name].attrib[attr],""])
- #print(" ", name, attr, codify(val))
-
- cols = ["File", "Element", "Field", "EN", "Translation ({})".format(args.lang)]
- df = pd.DataFrame(data, columns=cols)
- df.to_excel("SL_Translations_{}.xlsx".format(args.lang.upper()), index=False)
-
- #print "all_attrib", all_attrib
-
-
-
-
+"""\
+
+This module contains tools for scanning the SL codebase for translation-related strings.
+
+$LicenseInfo:firstyear=2020&license=viewerlgpl$
+Second Life Viewer Source Code
+Copyright (C) 2020, Linden Research, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation;
+version 2.1 of the License only.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
+$/LicenseInfo$
+"""
+
+from __future__ import print_function
+
+# packages required include: gitpython, pandas
+
+import xml.etree.ElementTree as ET
+import argparse
+import os
+import sys
+from git import Repo, Git # requires the gitpython package
+import pandas as pd
+
+translate_attribs = [
+ "title",
+ "short_title",
+ "value",
+ "label",
+ "label_selected",
+ "tool_tip",
+ "ignoretext",
+ "yestext",
+ "notext",
+ "canceltext",
+ "description",
+ "longdescription"
+]
+
+def codify_for_print(val):
+ if isinstance(val, unicode):
+ return val.encode("utf-8")
+ else:
+ return unicode(val, 'utf-8').encode("utf-8")
+
+# Returns a dict of { name => xml_node }
+def read_xml_elements(blob):
+ try:
+ contents = blob.data_stream.read()
+ except:
+ contents = '<?xml version="1.0" encoding="utf-8" standalone="yes" ?><strings></strings>'
+ xml = ET.fromstring(contents)
+ elts = {}
+ for child in xml.iter():
+ if "name" in child.attrib:
+ name = child.attrib['name']
+ elts[name] = child
+ return elts
+
+def failure(*msg):
+ print(*msg)
+ sys.exit(1)
+
+def can_translate(val):
+ if val is None:
+ return False
+ if val.isspace():
+ return False
+ val = val.strip()
+ if val.isdigit():
+ return False
+ return True
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description="analyze viewer xui files")
+ parser.add_argument("-v","--verbose", action="store_true", help="verbose flag")
+ parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
+ parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
+ parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en")
+ parser.add_argument("--lang", help="target language, default fr", default="fr")
+ args = parser.parse_args()
+
+ if args.rev == args.rev_base:
+ failure("Revs are the same, nothing to compare")
+
+ print("Finding changes in", args.rev, "not present in", args.rev_base)
+ sys.stdout.flush()
+
+ cwd = os.getcwd()
+ rootdir = Git(cwd).rev_parse("--show-toplevel")
+ repo = Repo(rootdir)
+ try:
+ mod_commit = repo.commit(args.rev)
+ except:
+ failure(args.rev,"is not a valid commit")
+ try:
+ base_commit = repo.commit(args.rev_base)
+ except:
+ failure(args.rev_base,"is not a valid commit")
+
+ mod_tree = mod_commit.tree
+ base_tree = base_commit.tree
+
+ xui_path = "indra/newview/skins/default/xui/{}".format(args.base_lang)
+ try:
+ mod_xui_tree = mod_tree[xui_path]
+ except:
+ failure("xui tree not found for language", args.base_lang)
+
+ data = []
+ # For all files to be checked for translations
+ for mod_blob in mod_xui_tree.traverse():
+ filename = mod_blob.path
+ if mod_blob.type == "tree": # directory, skip
+ continue
+
+ if args.verbose:
+ print(filename)
+ try:
+ base_blob = base_tree[filename]
+ except:
+ if args.verbose:
+ print("No matching base file found for", filename)
+ base_blob = None
+
+ try:
+ transl_filename = filename.replace(args.base_lang, args.lang)
+ transl_blob = mod_tree[transl_filename]
+ except:
+ if args.verbose:
+ print("No matching translation file found at", transl_filename)
+ transl_blob = None
+
+ mod_dict = read_xml_elements(mod_blob)
+ base_dict = read_xml_elements(base_blob)
+ transl_dict = read_xml_elements(transl_blob)
+
+ rows = 0
+ for name in mod_dict.keys():
+ if not name in base_dict or mod_dict[name].text != base_dict[name].text:
+ val = mod_dict[name].text
+ if can_translate(val):
+ transl_val = "--"
+ if name in transl_dict:
+ transl_val = transl_dict[name].text
+ data.append([filename, name, "text", val, transl_val, ""])
+ rows += 1
+ for attr in translate_attribs:
+ if attr in mod_dict[name].attrib:
+ if name not in base_dict or attr not in base_dict[name].attrib or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]:
+ val = mod_dict[name].attrib[attr]
+ if can_translate(val):
+ transl_val = "--"
+ if name in transl_dict and attr in transl_dict[name].attrib:
+ transl_val = transl_dict[name].attrib[attr]
+ data.append([filename, name, attr, val, transl_val, ""])
+ rows += 1
+ if args.verbose and rows>0:
+ print(" ",rows,"rows added")
+
+ outfile = "SL_Translations_{}.xlsx".format(args.lang.upper())
+ cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(args.lang.upper()), "ENTER NEW TRANSLATION ({})".format(args.lang.upper())]
+ num_translations = len(data)
+ df = pd.DataFrame(data, columns=cols)
+ df.to_excel(outfile, index=False)
+ if num_translations>0:
+ print("Wrote", num_translations, "rows to file", outfile)
+ else:
+ print("Nothing to translate,", outfile, "is empty")
--
cgit v1.2.3
From cef702d0c57e5bb4f0debfba677d1fd5a9694364 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Fri, 14 Aug 2020 14:11:34 +0100
Subject: SL-13705 - bug fixes and comments
---
scripts/code_tools/modified_strings.py | 60 +++++++++++++++++++++++++---------
1 file changed, 44 insertions(+), 16 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index dc3357fe8e..77954637a4 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -1,6 +1,7 @@
+#!/usr/bin/env python
"""\
-This module contains tools for scanning the SL codebase for translation-related strings.
+This script scans the SL codebase for translation-related strings.
$LicenseInfo:firstyear=2020&license=viewerlgpl$
Second Life Viewer Source Code
@@ -26,8 +27,6 @@ $/LicenseInfo$
from __future__ import print_function
-# packages required include: gitpython, pandas
-
import xml.etree.ElementTree as ET
import argparse
import os
@@ -61,6 +60,8 @@ def read_xml_elements(blob):
try:
contents = blob.data_stream.read()
except:
+ # default - pretend we read a file with no elements of interest.
+ # Parser will complain if it gets no elements at all.
contents = '<?xml version="1.0" encoding="utf-8" standalone="yes" ?><strings></strings>'
xml = ET.fromstring(contents)
elts = {}
@@ -83,23 +84,37 @@ def can_translate(val):
if val.isdigit():
return False
return True
-
+
+usage_msg="""%(prog)s [options]
+
+Analyze the XUI configuration files to find text that may need to
+be translated. Works by comparing two specified revisions, one
+specified by --rev (default HEAD) and one specified by --rev_base
+(default master). The script works by comparing xui contents of the
+two revisions, and outputs a spreadsheet listing any areas of
+difference. The target language must be specified using the --lang
+option. Output is an excel file, which can be used as-is or imported
+into google sheets.
+
+If the --rev revision already contains a translation for the text, it
+will be included in the spreadsheet for reference.
+
+Normally you would want --rev_base to be the last revision to have
+translations added, and --rev to be the tip of the current
+project.
+
+"""
+
if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="analyze viewer xui files")
+ parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg)
parser.add_argument("-v","--verbose", action="store_true", help="verbose flag")
parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en")
- parser.add_argument("--lang", help="target language, default fr", default="fr")
+ parser.add_argument("--lang", help="target language")
args = parser.parse_args()
- if args.rev == args.rev_base:
- failure("Revs are the same, nothing to compare")
-
- print("Finding changes in", args.rev, "not present in", args.rev_base)
- sys.stdout.flush()
-
cwd = os.getcwd()
rootdir = Git(cwd).rev_parse("--show-toplevel")
repo = Repo(rootdir)
@@ -115,11 +130,23 @@ if __name__ == "__main__":
mod_tree = mod_commit.tree
base_tree = base_commit.tree
- xui_path = "indra/newview/skins/default/xui/{}".format(args.base_lang)
+ xui_base = "indra/newview/skins/default/xui"
+ xui_base_tree = mod_tree[xui_base]
+ valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()]
+ if not args.lang or not args.lang.lower() in valid_langs:
+ failure("Please specify a target language using --lang. Valid values are", ",".join(sorted(valid_langs)))
+
+ xui_path = "{}/{}".format(xui_base, args.base_lang)
try:
mod_xui_tree = mod_tree[xui_path]
except:
- failure("xui tree not found for language", args.base_lang)
+ failure("xui tree not found for base language", args.base_lang)
+
+ if args.rev == args.rev_base:
+ failure("Revs are the same, nothing to compare")
+
+ print("Finding changes in", args.rev, "not present in", args.rev_base)
+ sys.stdout.flush()
data = []
# For all files to be checked for translations
@@ -130,6 +157,7 @@ if __name__ == "__main__":
if args.verbose:
print(filename)
+
try:
base_blob = base_tree[filename]
except:
@@ -138,11 +166,11 @@ if __name__ == "__main__":
base_blob = None
try:
- transl_filename = filename.replace(args.base_lang, args.lang)
+ transl_filename = filename.replace("/xui/{}/".format(args.base_lang), "/xui/{}/".format(args.lang))
transl_blob = mod_tree[transl_filename]
except:
if args.verbose:
- print("No matching translation file found at", transl_filename)
+ failure("No matching translation file found at", transl_filename)
transl_blob = None
mod_dict = read_xml_elements(mod_blob)
--
cgit v1.2.3
From 932f66929db5e7a5afae1f547408388cdc680709 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Thu, 10 Sep 2020 18:58:41 +0100
Subject: SL-13705 - support multiple or all languages. more logic for
excluding irrelevant fields
---
scripts/code_tools/modified_strings.py | 130 ++++++++++++++++++++++-----------
1 file changed, 88 insertions(+), 42 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index 77954637a4..eee20cf83b 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -33,6 +33,7 @@ import os
import sys
from git import Repo, Git # requires the gitpython package
import pandas as pd
+import re
translate_attribs = [
"title",
@@ -75,14 +76,23 @@ def failure(*msg):
print(*msg)
sys.exit(1)
-def can_translate(val):
+def should_translate(filename, val):
if val is None:
return False
+ if "floater_test" in filename:
+ return False
+ if "TestString PleaseIgnore" in val:
+ return False
+ if len(val) == 0:
+ return False
if val.isspace():
return False
val = val.strip()
if val.isdigit():
return False
+ if re.match(r"^\s*\d*\s*x\s*\d*\s*$", val):
+ print(val, "matches resolution string, will ignore")
+ return False
return True
usage_msg="""%(prog)s [options]
@@ -105,36 +115,7 @@ project.
"""
-if __name__ == "__main__":
-
- parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg)
- parser.add_argument("-v","--verbose", action="store_true", help="verbose flag")
- parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
- parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
- parser.add_argument("--base_lang", help="base language, default en (useful only for testing)", default="en")
- parser.add_argument("--lang", help="target language")
- args = parser.parse_args()
-
- cwd = os.getcwd()
- rootdir = Git(cwd).rev_parse("--show-toplevel")
- repo = Repo(rootdir)
- try:
- mod_commit = repo.commit(args.rev)
- except:
- failure(args.rev,"is not a valid commit")
- try:
- base_commit = repo.commit(args.rev_base)
- except:
- failure(args.rev_base,"is not a valid commit")
-
- mod_tree = mod_commit.tree
- base_tree = base_commit.tree
-
- xui_base = "indra/newview/skins/default/xui"
- xui_base_tree = mod_tree[xui_base]
- valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()]
- if not args.lang or not args.lang.lower() in valid_langs:
- failure("Please specify a target language using --lang. Valid values are", ",".join(sorted(valid_langs)))
+def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
xui_path = "{}/{}".format(xui_base, args.base_lang)
try:
@@ -145,11 +126,10 @@ if __name__ == "__main__":
if args.rev == args.rev_base:
failure("Revs are the same, nothing to compare")
- print("Finding changes in", args.rev, "not present in", args.rev_base)
- sys.stdout.flush()
data = []
# For all files to be checked for translations
+ all_en_strings = set()
for mod_blob in mod_xui_tree.traverse():
filename = mod_blob.path
if mod_blob.type == "tree": # directory, skip
@@ -166,7 +146,7 @@ if __name__ == "__main__":
base_blob = None
try:
- transl_filename = filename.replace("/xui/{}/".format(args.base_lang), "/xui/{}/".format(args.lang))
+ transl_filename = filename.replace("/xui/{}/".format(args.base_lang), "/xui/{}/".format(lang))
transl_blob = mod_tree[transl_filename]
except:
if args.verbose:
@@ -179,29 +159,43 @@ if __name__ == "__main__":
rows = 0
for name in mod_dict.keys():
- if not name in base_dict or mod_dict[name].text != base_dict[name].text:
+ if not name in base_dict or mod_dict[name].text != base_dict[name].text or (args.missing and not name in transl_dict):
val = mod_dict[name].text
- if can_translate(val):
+ if should_translate(filename, val):
transl_val = "--"
if name in transl_dict:
transl_val = transl_dict[name].text
- data.append([filename, name, "text", val, transl_val, ""])
+ if val in all_en_strings:
+ new_val = "(DUPLICATE)"
+ else:
+ new_val = ""
+ data.append([filename, name, "text", val, transl_val, new_val])
+ all_en_strings.add(val)
rows += 1
for attr in translate_attribs:
if attr in mod_dict[name].attrib:
- if name not in base_dict or attr not in base_dict[name].attrib or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr]:
+ if name not in base_dict \
+ or attr not in base_dict[name].attrib \
+ or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr] \
+ or (args.missing and (not name in transl_dict or not attr in transl_dict[name].attrib)):
val = mod_dict[name].attrib[attr]
- if can_translate(val):
+ if should_translate(filename, val):
+ show_val = val
transl_val = "--"
if name in transl_dict and attr in transl_dict[name].attrib:
transl_val = transl_dict[name].attrib[attr]
- data.append([filename, name, attr, val, transl_val, ""])
+ if val in all_en_strings:
+ new_val = "(DUPLICATE)"
+ else:
+ new_val = ""
+ data.append([filename, name, attr, show_val, transl_val, new_val])
+ all_en_strings.add(val)
rows += 1
if args.verbose and rows>0:
print(" ",rows,"rows added")
- outfile = "SL_Translations_{}.xlsx".format(args.lang.upper())
- cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(args.lang.upper()), "ENTER NEW TRANSLATION ({})".format(args.lang.upper())]
+ outfile = "SL_Translations_{}.xlsx".format(lang.upper())
+ cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper())]
num_translations = len(data)
df = pd.DataFrame(data, columns=cols)
df.to_excel(outfile, index=False)
@@ -209,3 +203,55 @@ if __name__ == "__main__":
print("Wrote", num_translations, "rows to file", outfile)
else:
print("Nothing to translate,", outfile, "is empty")
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg)
+ parser.add_argument("-v","--verbose", action="store_true", help="verbose flag")
+ parser.add_argument("--missing", action="store_true", default = False, help="include all fields for which a translation does not exist")
+ parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
+ parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
+ parser.add_argument("--base_lang", help="base language, default en (normally leave unchanged - other values are only useful for testing)", default="en")
+ parser.add_argument("--lang", help="target languages, or all", nargs="+")
+ args = parser.parse_args()
+
+ cwd = os.getcwd()
+ rootdir = Git(cwd).rev_parse("--show-toplevel")
+ repo = Repo(rootdir)
+ try:
+ mod_commit = repo.commit(args.rev)
+ except:
+ failure(args.rev,"is not a valid commit")
+ try:
+ base_commit = repo.commit(args.rev_base)
+ except:
+ failure(args.rev_base,"is not a valid commit")
+
+ print("Will identify changes in", args.rev, "not present in", args.rev_base)
+ if args.missing:
+ print("Will also include any text for which no corresponding translation exists, regardless of when it was added")
+ sys.stdout.flush()
+
+ mod_tree = mod_commit.tree
+ base_tree = base_commit.tree
+
+ xui_base = "indra/newview/skins/default/xui"
+ xui_base_tree = mod_tree[xui_base]
+ valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()]
+ langs = [l.lower() for l in args.lang]
+ if "all" in args.lang:
+ langs = valid_langs
+
+ for lang in langs:
+ if not lang in valid_langs:
+ failure("Unknown target language {}. Valid values are {} or all".format(lang,",".join(sorted(valid_langs))))
+
+ print("Target language(s) are", ",".join(sorted(langs)))
+ sys.stdout.flush()
+
+ for lang in langs:
+ print("Creating spreadsheet for language", lang)
+ sys.stdout.flush()
+
+ make_translation_spreadsheet(mod_tree, base_tree, lang, args)
+
--
cgit v1.2.3
From 3788fdbb03f85492f4bd26cb4637434ac3e8515e Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Thu, 10 Sep 2020 19:53:13 +0100
Subject: SL-13705 - fixes and spreadsheet format changes
---
scripts/code_tools/modified_strings.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index eee20cf83b..3ea13a2bf5 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -169,7 +169,8 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
new_val = "(DUPLICATE)"
else:
new_val = ""
- data.append([filename, name, "text", val, transl_val, new_val])
+ field = "text"
+ data.append([val, transl_val, new_val, filename, name, field])
all_en_strings.add(val)
rows += 1
for attr in translate_attribs:
@@ -178,9 +179,12 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
or attr not in base_dict[name].attrib \
or mod_dict[name].attrib[attr] != base_dict[name].attrib[attr] \
or (args.missing and (not name in transl_dict or not attr in transl_dict[name].attrib)):
- val = mod_dict[name].attrib[attr]
+ elt = mod_dict[name]
+ val = elt.attrib[attr]
+ #if attr == "value" and elt.tag not in ["string","text"]:
+ # print("skipping value attribute", val, "tag", elt.tag, "in", filename)
+ # continue
if should_translate(filename, val):
- show_val = val
transl_val = "--"
if name in transl_dict and attr in transl_dict[name].attrib:
transl_val = transl_dict[name].attrib[attr]
@@ -188,14 +192,15 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
new_val = "(DUPLICATE)"
else:
new_val = ""
- data.append([filename, name, attr, show_val, transl_val, new_val])
+ field = attr
+ data.append([val, transl_val, new_val, filename, name, field])
all_en_strings.add(val)
rows += 1
if args.verbose and rows>0:
print(" ",rows,"rows added")
outfile = "SL_Translations_{}.xlsx".format(lang.upper())
- cols = ["File", "Element", "Field", "EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper())]
+ cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"]
num_translations = len(data)
df = pd.DataFrame(data, columns=cols)
df.to_excel(outfile, index=False)
--
cgit v1.2.3
From 1eaa76f2f09557e1fc828775249d6eece236572d Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Fri, 11 Sep 2020 19:37:04 +0100
Subject: SL-13705 - quieter output
---
scripts/code_tools/modified_strings.py | 48 +++++++++++++++++++++++++---------
1 file changed, 35 insertions(+), 13 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index 3ea13a2bf5..ec1c050a8f 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -76,13 +76,24 @@ def failure(*msg):
print(*msg)
sys.exit(1)
-def should_translate(filename, val):
+# return True iff any element of lis is "in" thing
+def has_any(thing,lis):
+ for l in lis:
+ if l in thing:
+ return True
+ return False
+
+def should_translate(filename, elt, field, val):
if val is None:
return False
- if "floater_test" in filename:
+ # Should translate apply recursively?
+ if "translate" in elt.attrib and elt.attrib["translate"] == "false":
+ return False
+ if has_any(filename,["floater_test","floater_aaa","floater_ui_preview"]):
return False
if "TestString PleaseIgnore" in val:
return False
+ val = re.sub(r"\[.*?\]","",val)
if len(val) == 0:
return False
if val.isspace():
@@ -90,9 +101,22 @@ def should_translate(filename, val):
val = val.strip()
if val.isdigit():
return False
+ if not re.search('\w+', val):
+ return False
if re.match(r"^\s*\d*\s*x\s*\d*\s*$", val):
- print(val, "matches resolution string, will ignore")
+ #print(val, "matches resolution string, will ignore")
return False
+ # "value" is a hairball, mostly used to encode non-display info but a few exceptions
+ if field == "value":
+ if elt.text is not None and len(elt.text) > 0:
+ #print("value has text, ignoring", ET.tostring(elt))
+ return False
+ if has_any(elt.attrib,["label"]):
+ return False
+ if elt.tag in ["string","text"]:
+ return True
+ #print("including value attribute", val, "tag", elt.tag,"in", ET.tostring(elt))
+ return True
return True
usage_msg="""%(prog)s [options]
@@ -150,7 +174,7 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
transl_blob = mod_tree[transl_filename]
except:
if args.verbose:
- failure("No matching translation file found at", transl_filename)
+ print("No matching translation file found at", transl_filename)
transl_blob = None
mod_dict = read_xml_elements(mod_blob)
@@ -160,8 +184,10 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
rows = 0
for name in mod_dict.keys():
if not name in base_dict or mod_dict[name].text != base_dict[name].text or (args.missing and not name in transl_dict):
- val = mod_dict[name].text
- if should_translate(filename, val):
+ elt = mod_dict[name]
+ val = elt.text
+ field = "text"
+ if should_translate(filename, elt, field, val):
transl_val = "--"
if name in transl_dict:
transl_val = transl_dict[name].text
@@ -169,7 +195,6 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
new_val = "(DUPLICATE)"
else:
new_val = ""
- field = "text"
data.append([val, transl_val, new_val, filename, name, field])
all_en_strings.add(val)
rows += 1
@@ -181,10 +206,7 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
or (args.missing and (not name in transl_dict or not attr in transl_dict[name].attrib)):
elt = mod_dict[name]
val = elt.attrib[attr]
- #if attr == "value" and elt.tag not in ["string","text"]:
- # print("skipping value attribute", val, "tag", elt.tag, "in", filename)
- # continue
- if should_translate(filename, val):
+ if should_translate(filename, elt, attr, val):
transl_val = "--"
if name in transl_dict and attr in transl_dict[name].attrib:
transl_val = transl_dict[name].attrib[attr]
@@ -192,8 +214,8 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
new_val = "(DUPLICATE)"
else:
new_val = ""
- field = attr
- data.append([val, transl_val, new_val, filename, name, field])
+ #attr = attr + ":" + ET.tostring(elt)
+ data.append([val, transl_val, new_val, filename, name, attr])
all_en_strings.add(val)
rows += 1
if args.verbose and rows>0:
--
cgit v1.2.3
From 822a4fb7d2645a4167b987995a190169133cff42 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Fri, 11 Sep 2020 21:47:37 +0100
Subject: SL-13705 - Additional cell/column properties specified in excel
output
---
scripts/code_tools/modified_strings.py | 32 +++++++++++++++++++++++++++-----
1 file changed, 27 insertions(+), 5 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index ec1c050a8f..b77115ed46 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -106,7 +106,7 @@ def should_translate(filename, elt, field, val):
if re.match(r"^\s*\d*\s*x\s*\d*\s*$", val):
#print(val, "matches resolution string, will ignore")
return False
- # "value" is a hairball, mostly used to encode non-display info but a few exceptions
+ # "value" attribute is a hairball, mostly used to encode non-display info but a few exceptions
if field == "value":
if elt.text is not None and len(elt.text) > 0:
#print("value has text, ignoring", ET.tostring(elt))
@@ -218,18 +218,40 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
data.append([val, transl_val, new_val, filename, name, attr])
all_en_strings.add(val)
rows += 1
- if args.verbose and rows>0:
- print(" ",rows,"rows added")
+ save_as_excel(data, lang)
+
+
+def save_as_excel(data, lang):
+
outfile = "SL_Translations_{}.xlsx".format(lang.upper())
- cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"]
num_translations = len(data)
+ cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"]
df = pd.DataFrame(data, columns=cols)
- df.to_excel(outfile, index=False)
+
+ writer = pd.ExcelWriter(outfile, engine='xlsxwriter')
+ df.to_excel(writer, index=False, sheet_name = "Sheet1")
+
+ workbook = writer.book
+ worksheet = writer.sheets['Sheet1']
+
+ cell_format = workbook.add_format({'text_wrap': True})
+
+ # Translators primarily care about columns A-C
+ worksheet.set_column('A:C', 100, cell_format)
+ worksheet.set_column('D:D', 50, cell_format, {'hidden': True})
+ worksheet.set_column('E:F', 30, cell_format, {'hidden': True})
+
+ # Lock the column header in place while scrolling
+ worksheet.freeze_panes(1, 0)
+
+ writer.save()
+
if num_translations>0:
print("Wrote", num_translations, "rows to file", outfile)
else:
print("Nothing to translate,", outfile, "is empty")
+
if __name__ == "__main__":
--
cgit v1.2.3
From a1d24974e808c3a0fb6072d010ccabbc38703232 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Mon, 14 Sep 2020 20:34:51 +0100
Subject: SL-13705 - lock all columns except C in the excel file (apparently
google sheets does not respect this setting on import)
---
scripts/code_tools/modified_strings.py | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index b77115ed46..c259a0f984 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -235,14 +235,18 @@ def save_as_excel(data, lang):
workbook = writer.book
worksheet = writer.sheets['Sheet1']
- cell_format = workbook.add_format({'text_wrap': True})
-
- # Translators primarily care about columns A-C
- worksheet.set_column('A:C', 100, cell_format)
- worksheet.set_column('D:D', 50, cell_format, {'hidden': True})
- worksheet.set_column('E:F', 30, cell_format, {'hidden': True})
-
- # Lock the column header in place while scrolling
+ wrap_format = workbook.add_format({'text_wrap': True})
+ wrap_unlocked_format = workbook.add_format({'text_wrap': True, 'locked': False})
+
+ # Translators primarily care about columns A-C, and should write
+ # only in column C. Can hide the others. Set widths.
+ worksheet.protect()
+ worksheet.set_column('A:B', 100, wrap_format)
+ worksheet.set_column('C:C', 100, wrap_unlocked_format)
+ worksheet.set_column('D:D', 50, wrap_format, {'hidden': True})
+ worksheet.set_column('E:F', 30, wrap_format, {'hidden': True})
+
+ # Lock the top row (column headers) in place while scrolling
worksheet.freeze_panes(1, 0)
writer.save()
--
cgit v1.2.3
From a74b366d22f82447919751c04815e070bc5307c7 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Tue, 15 Sep 2020 13:53:39 +0100
Subject: SL-13705 - single output file with one tab per translation language
---
scripts/code_tools/modified_strings.py | 79 +++++++++++++++++++---------------
1 file changed, 45 insertions(+), 34 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index c259a0f984..70ca3e1ae6 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -139,7 +139,7 @@ project.
"""
-def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
+def make_translation_table(mod_tree, base_tree, lang, args):
xui_path = "{}/{}".format(xui_base, args.base_lang)
try:
@@ -195,7 +195,7 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
new_val = "(DUPLICATE)"
else:
new_val = ""
- data.append([val, transl_val, new_val, filename, name, field])
+ data.append([val, transl_val, new_val, "", "", filename, name, field])
all_en_strings.add(val)
rows += 1
for attr in translate_attribs:
@@ -215,48 +215,47 @@ def make_translation_spreadsheet(mod_tree, base_tree, lang, args):
else:
new_val = ""
#attr = attr + ":" + ET.tostring(elt)
- data.append([val, transl_val, new_val, filename, name, attr])
+ data.append([val, transl_val, new_val, "", "", filename, name, attr])
all_en_strings.add(val)
rows += 1
+ return data
- save_as_excel(data, lang)
+def save_translation_file(all_data, outfile):
-
-def save_as_excel(data, lang):
-
- outfile = "SL_Translations_{}.xlsx".format(lang.upper())
- num_translations = len(data)
- cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "File", "Element", "Field"]
- df = pd.DataFrame(data, columns=cols)
+ langs = sorted(all_data.keys())
+ print("Saving languages", ",".join(langs),"as",outfile)
writer = pd.ExcelWriter(outfile, engine='xlsxwriter')
- df.to_excel(writer, index=False, sheet_name = "Sheet1")
workbook = writer.book
- worksheet = writer.sheets['Sheet1']
-
wrap_format = workbook.add_format({'text_wrap': True})
wrap_unlocked_format = workbook.add_format({'text_wrap': True, 'locked': False})
- # Translators primarily care about columns A-C, and should write
- # only in column C. Can hide the others. Set widths.
- worksheet.protect()
- worksheet.set_column('A:B', 100, wrap_format)
- worksheet.set_column('C:C', 100, wrap_unlocked_format)
- worksheet.set_column('D:D', 50, wrap_format, {'hidden': True})
- worksheet.set_column('E:F', 30, wrap_format, {'hidden': True})
-
- # Lock the top row (column headers) in place while scrolling
- worksheet.freeze_panes(1, 0)
-
+ for lang in langs:
+ data = all_data[lang]
+ num_translations = len(data)
+ cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "Translator Questions", "Notes", "File", "Element", "Field"]
+ df = pd.DataFrame(data, columns=cols)
+ df.to_excel(writer, index=False, sheet_name = lang.upper())
+
+ worksheet = writer.sheets[lang.upper()]
+
+ # Translators primarily care about columns A-C, and should write
+ # only in column C. Hide the others. Set widths.
+ worksheet.protect()
+ worksheet.set_column('A:B', 60, wrap_format)
+ worksheet.set_column('C:C', 60, wrap_unlocked_format)
+ worksheet.set_column('D:E', 40, wrap_unlocked_format)
+ worksheet.set_column('F:F', 50, wrap_format, {'hidden': True})
+ worksheet.set_column('G:H', 30, wrap_format, {'hidden': True})
+
+ # Lock the top row (column headers) in place while scrolling
+ worksheet.freeze_panes(1, 0)
+ print("Added", num_translations, "rows for language", lang)
+
+ print("Writing", outfile)
writer.save()
- if num_translations>0:
- print("Wrote", num_translations, "rows to file", outfile)
- else:
- print("Nothing to translate,", outfile, "is empty")
-
-
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg)
@@ -290,21 +289,33 @@ if __name__ == "__main__":
xui_base = "indra/newview/skins/default/xui"
xui_base_tree = mod_tree[xui_base]
+
+ # Find target languages
valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()]
langs = [l.lower() for l in args.lang]
if "all" in args.lang:
langs = valid_langs
-
+ langs = sorted(langs)
for lang in langs:
if not lang in valid_langs:
failure("Unknown target language {}. Valid values are {} or all".format(lang,",".join(sorted(valid_langs))))
-
print("Target language(s) are", ",".join(sorted(langs)))
sys.stdout.flush()
+ outfile = "SL_Translations.xlsx"
+ try:
+ f = open(outfile,"a+")
+ f.close()
+ except:
+ failure("Can't write to output file",outfile,". Is it already open?")
+
+ all_data = {}
for lang in langs:
print("Creating spreadsheet for language", lang)
sys.stdout.flush()
- make_translation_spreadsheet(mod_tree, base_tree, lang, args)
+ all_data[lang] = make_translation_table(mod_tree, base_tree, lang, args)
+
+ print("Saving output file", outfile)
+ save_translation_file(all_data, outfile)
--
cgit v1.2.3
From 713d896d3664c8266a7b08feea480735d2a8b390 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Tue, 15 Sep 2020 15:49:26 +0100
Subject: SL-13705 - added metadata tab to translation spreadsheet, showing
invocation info
---
scripts/code_tools/modified_strings.py | 70 +++++++++++++++++++++-------------
1 file changed, 44 insertions(+), 26 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index 70ca3e1ae6..1dbd20f1b4 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -34,6 +34,30 @@ import sys
from git import Repo, Git # requires the gitpython package
import pandas as pd
import re
+from datetime import datetime
+
+usage_msg="""%(prog)s [options]
+
+Analyze the XUI configuration files to find text that may need to
+be translated. Works by comparing two specified revisions, one
+specified by --rev (default HEAD) and one specified by --rev_base
+(default master). The script works by comparing xui contents of the
+two revisions, and outputs a spreadsheet listing any areas of
+difference. The target language must be specified using the --lang
+option. Output is an excel file, which can be used as-is or imported
+into google sheets.
+
+If the --rev revision already contains a translation for the text, it
+will be included in the spreadsheet for reference.
+
+Normally you would want --rev_base to be the last revision to have
+translations added, and --rev to be the tip of the current
+project. You can find the last commit with translation work using "git log --grep INTL- | head"
+
+The --missing argument can be used to find all text with missing
+translations, regardless of when it was added. If translations are being kept
+reasonably current, you will normally not need this argument.
+"""
translate_attribs = [
"title",
@@ -119,26 +143,6 @@ def should_translate(filename, elt, field, val):
return True
return True
-usage_msg="""%(prog)s [options]
-
-Analyze the XUI configuration files to find text that may need to
-be translated. Works by comparing two specified revisions, one
-specified by --rev (default HEAD) and one specified by --rev_base
-(default master). The script works by comparing xui contents of the
-two revisions, and outputs a spreadsheet listing any areas of
-difference. The target language must be specified using the --lang
-option. Output is an excel file, which can be used as-is or imported
-into google sheets.
-
-If the --rev revision already contains a translation for the text, it
-will be included in the spreadsheet for reference.
-
-Normally you would want --rev_base to be the last revision to have
-translations added, and --rev to be the tip of the current
-project.
-
-"""
-
def make_translation_table(mod_tree, base_tree, lang, args):
xui_path = "{}/{}".format(xui_base, args.base_lang)
@@ -220,19 +224,20 @@ def make_translation_table(mod_tree, base_tree, lang, args):
rows += 1
return data
-def save_translation_file(all_data, outfile):
+def save_translation_file(per_lang_data, aux_data, outfile):
- langs = sorted(all_data.keys())
+ langs = sorted(per_lang_data.keys())
print("Saving languages", ",".join(langs),"as",outfile)
writer = pd.ExcelWriter(outfile, engine='xlsxwriter')
workbook = writer.book
wrap_format = workbook.add_format({'text_wrap': True})
+ bold_wrap_format = workbook.add_format({'text_wrap': True, 'bold': True})
wrap_unlocked_format = workbook.add_format({'text_wrap': True, 'locked': False})
for lang in langs:
- data = all_data[lang]
+ data = per_lang_data[lang]
num_translations = len(data)
cols = ["EN", "Previous Translation ({})".format(lang.upper()), "ENTER NEW TRANSLATION ({})".format(lang.upper()), "Translator Questions", "Notes", "File", "Element", "Field"]
df = pd.DataFrame(data, columns=cols)
@@ -253,6 +258,14 @@ def save_translation_file(all_data, outfile):
worksheet.freeze_panes(1, 0)
print("Added", num_translations, "rows for language", lang)
+ # Reference info, not for translation
+ for aux, data in aux_data.items():
+ df = pd.DataFrame(data, columns = ["Key", "Value"])
+ df.to_excel(writer, index=False, sheet_name=aux)
+ worksheet = writer.sheets[aux]
+ worksheet.set_column('A:A', 50, bold_wrap_format)
+ worksheet.set_column('B:B', 80, wrap_format)
+
print("Writing", outfile)
writer.save()
@@ -309,13 +322,18 @@ if __name__ == "__main__":
except:
failure("Can't write to output file",outfile,". Is it already open?")
- all_data = {}
+ aux_data = { "REFERENCE": [["Command", " ".join(sys.argv)],
+ ["Date", str(datetime.now())],
+ ["Mod Commit", mod_commit.hexsha],
+ ["Base Commit", base_commit.hexsha],
+ ] }
+ per_lang_data = {}
for lang in langs:
print("Creating spreadsheet for language", lang)
sys.stdout.flush()
- all_data[lang] = make_translation_table(mod_tree, base_tree, lang, args)
+ per_lang_data[lang] = make_translation_table(mod_tree, base_tree, lang, args)
print("Saving output file", outfile)
- save_translation_file(all_data, outfile)
+ save_translation_file(per_lang_data, aux_data, outfile)
--
cgit v1.2.3
From f20b29dcb780333cba4d6e895ceba549ddfbee80 Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Wed, 16 Sep 2020 12:51:59 +0100
Subject: SL-13705 - language list defaults to the standard supported set of
FR, ES, IT, PT, JA, DE. '--lang all' will also use this list
---
scripts/code_tools/modified_strings.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index 1dbd20f1b4..762f1b3845 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -277,7 +277,7 @@ if __name__ == "__main__":
parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
parser.add_argument("--base_lang", help="base language, default en (normally leave unchanged - other values are only useful for testing)", default="en")
- parser.add_argument("--lang", help="target languages, or all", nargs="+")
+ parser.add_argument("--lang", help="target languages, or all", nargs="+", default = ["all"])
args = parser.parse_args()
cwd = os.getcwd()
@@ -305,9 +305,10 @@ if __name__ == "__main__":
# Find target languages
valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()]
+ supported_langs = ["fr", "es", "it", "pt", "ja", "de"]
langs = [l.lower() for l in args.lang]
if "all" in args.lang:
- langs = valid_langs
+ langs = supported_langs
langs = sorted(langs)
for lang in langs:
if not lang in valid_langs:
--
cgit v1.2.3
From bf72ffd84b82bf8b7cf28a09a8bc4134e7e5d05a Mon Sep 17 00:00:00 2001
From: "Brad Payne (Vir Linden)" <vir@lindenlab.com>
Date: Fri, 18 Sep 2020 16:22:54 +0100
Subject: SL-13705 - added --deleted option to collect info on no-longer-needed
text in translated files. Processing needs to be automated as the list is
unfeasibly long.
---
scripts/code_tools/modified_strings.py | 87 +++++++++++++++++++++++++++++-----
1 file changed, 75 insertions(+), 12 deletions(-)
(limited to 'scripts')
diff --git a/scripts/code_tools/modified_strings.py b/scripts/code_tools/modified_strings.py
index 762f1b3845..6a763b6ec5 100644
--- a/scripts/code_tools/modified_strings.py
+++ b/scripts/code_tools/modified_strings.py
@@ -149,7 +149,7 @@ def make_translation_table(mod_tree, base_tree, lang, args):
try:
mod_xui_tree = mod_tree[xui_path]
except:
- failure("xui tree not found for base language", args.base_lang)
+ failure("xui tree not found for base language", args.base_lang,"or target lang", lang)
if args.rev == args.rev_base:
failure("Revs are the same, nothing to compare")
@@ -222,8 +222,56 @@ def make_translation_table(mod_tree, base_tree, lang, args):
data.append([val, transl_val, new_val, "", "", filename, name, attr])
all_en_strings.add(val)
rows += 1
+
return data
+def find_deletions(mod_tree, base_tree, lang, args, f):
+
+ transl_xui_path = "{}/{}".format(xui_base, lang)
+ try:
+ transl_xui_tree = mod_tree[transl_xui_path]
+ except:
+ failure("xui tree not found for base language", args.base_lang,"or target lang", lang)
+
+ for transl_blob in transl_xui_tree.traverse():
+ if transl_blob.type == "tree": # directory, skip
+ continue
+ transl_filename = transl_blob.path
+ mod_filename = transl_filename.replace("/xui/{}/".format(lang), "/xui/{}/".format(args.base_lang))
+ #print("checking",transl_filename,"against",mod_filename)
+ try:
+ mod_blob = mod_tree[mod_filename]
+ except:
+ print(" delete file", transl_filename, file=f)
+ continue
+ mod_dict = read_xml_elements(mod_blob)
+ if len(mod_dict) == 0:
+ print(" delete file", transl_filename, file=f)
+ continue
+ transl_dict = read_xml_elements(transl_blob)
+ #print("mod vs transl", len(mod_dict), len(transl_dict))
+ lines = 0
+ for elt_key in transl_dict:
+ if not elt_key in mod_dict:
+ if lines == 0:
+ print(" in file", transl_filename, file=f)
+ lines += 1
+ print(" delete element", elt_key, file=f)
+ else:
+ transl_elt = transl_dict[elt_key]
+ mod_elt = mod_dict[elt_key]
+ for a in transl_elt.attrib:
+ if not a in mod_elt.attrib:
+ if lines == 0:
+ print(" in file", transl_filename, file=f)
+ lines += 1
+ print(" delete attribute", a, "from", elt_key, file=f)
+ if transl_elt.text and (not mod_elt.text):
+ if lines == 0:
+ print(" in file", transl_filename, file=f)
+ lines += 1
+ print(" delete text from", elt_key, file=f)
+
def save_translation_file(per_lang_data, aux_data, outfile):
langs = sorted(per_lang_data.keys())
@@ -274,10 +322,12 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="analyze viewer xui files for needed translations", usage=usage_msg)
parser.add_argument("-v","--verbose", action="store_true", help="verbose flag")
parser.add_argument("--missing", action="store_true", default = False, help="include all fields for which a translation does not exist")
+ parser.add_argument("--deleted", action="store_true", default = False, help="show all translated entities which don't exist in english")
+ parser.add_argument("--skip_spreadsheet", action="store_true", default = False, help="skip creating the translation spreadsheet")
parser.add_argument("--rev", help="revision with modified strings, default HEAD", default="HEAD")
parser.add_argument("--rev_base", help="previous revision to compare against, default master", default="master")
parser.add_argument("--base_lang", help="base language, default en (normally leave unchanged - other values are only useful for testing)", default="en")
- parser.add_argument("--lang", help="target languages, or all", nargs="+", default = ["all"])
+ parser.add_argument("--lang", help="target languages, or 'all_valid' or 'supported'; default is 'supported'", nargs="+", default = ["supported"])
args = parser.parse_args()
cwd = os.getcwd()
@@ -304,15 +354,19 @@ if __name__ == "__main__":
xui_base_tree = mod_tree[xui_base]
# Find target languages
+ # all languages present in the codebase
valid_langs = [tree.name.lower() for tree in xui_base_tree if tree.name.lower() != args.base_lang.lower()]
+ # offically supported languages
supported_langs = ["fr", "es", "it", "pt", "ja", "de"]
langs = [l.lower() for l in args.lang]
- if "all" in args.lang:
+ if "supported" in args.lang:
langs = supported_langs
+ if "all_valid" in args.lang:
+ langs = valid_langs
langs = sorted(langs)
for lang in langs:
if not lang in valid_langs:
- failure("Unknown target language {}. Valid values are {} or all".format(lang,",".join(sorted(valid_langs))))
+ failure("Unknown target language {}. Valid values are {}".format(lang,", ".join(sorted(valid_langs) + ["all_valid","supported"])))
print("Target language(s) are", ",".join(sorted(langs)))
sys.stdout.flush()
@@ -328,13 +382,22 @@ if __name__ == "__main__":
["Mod Commit", mod_commit.hexsha],
["Base Commit", base_commit.hexsha],
] }
- per_lang_data = {}
- for lang in langs:
- print("Creating spreadsheet for language", lang)
- sys.stdout.flush()
-
- per_lang_data[lang] = make_translation_table(mod_tree, base_tree, lang, args)
- print("Saving output file", outfile)
- save_translation_file(per_lang_data, aux_data, outfile)
+ if not args.skip_spreadsheet:
+ per_lang_data = {}
+ for lang in langs:
+ print("Creating spreadsheet for language", lang)
+ sys.stdout.flush()
+
+ per_lang_data[lang] = make_translation_table(mod_tree, base_tree, lang, args)
+
+ print("Saving output file", outfile)
+ save_translation_file(per_lang_data, aux_data, outfile)
+
+ if args.deleted:
+ deletion_file = "Translate_deletions.txt"
+ print("Saving deletion info to", deletion_file)
+ with open(deletion_file,"w") as f:
+ for lang in langs:
+ find_deletions(mod_tree, base_tree, lang, args, f)
--
cgit v1.2.3