From 60d62c767be3576cb674a817b90028951d6ee3b0 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 16 Mar 2021 16:18:56 +0100 Subject: SL-14999 - cppfeatures test file for examples and compiler conformance verification --- indra/newview/CMakeLists.txt | 4 + indra/newview/tests/cppfeatures_test.cpp | 127 +++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 indra/newview/tests/cppfeatures_test.cpp diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index 3439951e80..9c11f8aa87 100644 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -2520,6 +2520,10 @@ if (LL_TESTS) ${BOOST_CONTEXT_LIBRARY} ) + LL_ADD_INTEGRATION_TEST(cppfeatures + "" + "${test_libs}" + ) LL_ADD_INTEGRATION_TEST(llsechandler_basic llsechandler_basic.cpp "${test_libs}" diff --git a/indra/newview/tests/cppfeatures_test.cpp b/indra/newview/tests/cppfeatures_test.cpp new file mode 100644 index 0000000000..15aa1cd6d6 --- /dev/null +++ b/indra/newview/tests/cppfeatures_test.cpp @@ -0,0 +1,127 @@ +/** + * @file cppfeatures_test + * @author Vir + * @date 2021-03 + * @brief cpp features + * + * $LicenseInfo:firstyear=2021&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2021, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +// Tests related to newer C++ features, for verifying support across compilers and platforms + +#include "linden_common.h" +#include "../test/lltut.h" + +namespace tut +{ + +struct cpp_features_test {}; +typedef test_group cpp_features_test_t; +typedef cpp_features_test_t::object cpp_features_test_object_t; +tut::cpp_features_test_t tut_cpp_features_test("LLCPPFeatures"); + +// bracket initializers +// Can initialize containers or values using curly brackets +template<> template<> +void cpp_features_test_object_t::test<1>() +{ + S32 explicit_val{3}; + ensure(explicit_val==3); + + S32 default_val{}; + ensure(default_val==0); + + std::vector fibs{1,1,2,3,5}; + ensure(fibs[4]==5); +} + +// auto +// +// https://en.cppreference.com/w/cpp/language/auto +// +// Can use auto in place of a more complex type specification, if the compiler can infer the type +template<> template<> +void cpp_features_test_object_t::test<2>() +{ + std::vector numbers{3,6,9}; + + // auto element + auto& aval = numbers[1]; + ensure("auto element", aval==6); + + // auto iterator (non-const) + auto it = numbers.rbegin(); + *it += 1; + S32 val = *it; + ensure("auto iterator", val==10); +} + +// range for +// +// https://en.cppreference.com/w/cpp/language/range-for +// +// Can iterate over containers without explicit iterator +template<> template<> +void cpp_features_test_object_t::test<3>() +{ + + // Traditional iterator for with container + // + // Problems: + // * Have to create a new variable for the iterator, which is unrelated to the problem you're trying to solve. + // * Redundant and somewhat fragile. Have to make sure begin() and end() are both from the right container. + std::vector numbers{3,6,9}; + for (auto it = numbers.begin(); it != numbers.end(); ++it) + { + auto& n = *it; + n *= 2; + } + ensure("iterator for vector", numbers[2]==18); + + // Range for with container + // + // Under the hood, this is doing the same thing as the traditional + // for loop above. Still uses begin() and end() but you don't have + // to access them directly. + std::vector numbersb{3,6,9}; + for (auto& n: numbersb) + { + n *= 2; + } + ensure("range for vector", numbersb[2]==18); + + // Range for over a C-style array. + // + // This is handy because the language determines the range automatically. + // Getting this right manually is a little trickier. + S32 pows[] = {1,2,4,8,16}; + S32 sum{}; + for (const auto& v: pows) + { + sum += v; + } + ensure("for C-array", sum==31); +} + + + +} -- cgit v1.2.3 From 47b3078c93a763f67103ff67f0fe8b6ac5794370 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Wed, 17 Mar 2021 13:54:18 +0100 Subject: SL-14999 - C++ feature tests --- indra/newview/tests/cppfeatures_test.cpp | 372 ++++++++++++++++++++----------- 1 file changed, 245 insertions(+), 127 deletions(-) diff --git a/indra/newview/tests/cppfeatures_test.cpp b/indra/newview/tests/cppfeatures_test.cpp index 15aa1cd6d6..8399bb12ff 100644 --- a/indra/newview/tests/cppfeatures_test.cpp +++ b/indra/newview/tests/cppfeatures_test.cpp @@ -1,127 +1,245 @@ -/** - * @file cppfeatures_test - * @author Vir - * @date 2021-03 - * @brief cpp features - * - * $LicenseInfo:firstyear=2021&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2021, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -// Tests related to newer C++ features, for verifying support across compilers and platforms - -#include "linden_common.h" -#include "../test/lltut.h" - -namespace tut -{ - -struct cpp_features_test {}; -typedef test_group cpp_features_test_t; -typedef cpp_features_test_t::object cpp_features_test_object_t; -tut::cpp_features_test_t tut_cpp_features_test("LLCPPFeatures"); - -// bracket initializers -// Can initialize containers or values using curly brackets -template<> template<> -void cpp_features_test_object_t::test<1>() -{ - S32 explicit_val{3}; - ensure(explicit_val==3); - - S32 default_val{}; - ensure(default_val==0); - - std::vector fibs{1,1,2,3,5}; - ensure(fibs[4]==5); -} - -// auto -// -// https://en.cppreference.com/w/cpp/language/auto -// -// Can use auto in place of a more complex type specification, if the compiler can infer the type -template<> template<> -void cpp_features_test_object_t::test<2>() -{ - std::vector numbers{3,6,9}; - - // auto element - auto& aval = numbers[1]; - ensure("auto element", aval==6); - - // auto iterator (non-const) - auto it = numbers.rbegin(); - *it += 1; - S32 val = *it; - ensure("auto iterator", val==10); -} - -// range for -// -// https://en.cppreference.com/w/cpp/language/range-for -// -// Can iterate over containers without explicit iterator -template<> template<> -void cpp_features_test_object_t::test<3>() -{ - - // Traditional iterator for with container - // - // Problems: - // * Have to create a new variable for the iterator, which is unrelated to the problem you're trying to solve. - // * Redundant and somewhat fragile. Have to make sure begin() and end() are both from the right container. - std::vector numbers{3,6,9}; - for (auto it = numbers.begin(); it != numbers.end(); ++it) - { - auto& n = *it; - n *= 2; - } - ensure("iterator for vector", numbers[2]==18); - - // Range for with container - // - // Under the hood, this is doing the same thing as the traditional - // for loop above. Still uses begin() and end() but you don't have - // to access them directly. - std::vector numbersb{3,6,9}; - for (auto& n: numbersb) - { - n *= 2; - } - ensure("range for vector", numbersb[2]==18); - - // Range for over a C-style array. - // - // This is handy because the language determines the range automatically. - // Getting this right manually is a little trickier. - S32 pows[] = {1,2,4,8,16}; - S32 sum{}; - for (const auto& v: pows) - { - sum += v; - } - ensure("for C-array", sum==31); -} - - - -} +/** + * @file cppfeatures_test + * @author Vir + * @date 2021-03 + * @brief cpp features + * + * $LicenseInfo:firstyear=2021&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2021, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +// Tests related to newer C++ features, for verifying support across compilers and platforms + +#include "linden_common.h" +#include "../test/lltut.h" + +namespace tut +{ + +struct cpp_features_test {}; +typedef test_group cpp_features_test_t; +typedef cpp_features_test_t::object cpp_features_test_object_t; +tut::cpp_features_test_t tut_cpp_features_test("LLCPPFeatures"); + +// bracket initializers +// Can initialize containers or values using curly brackets +template<> template<> +void cpp_features_test_object_t::test<1>() +{ + S32 explicit_val{3}; + ensure(explicit_val==3); + + S32 default_val{}; + ensure(default_val==0); + + std::vector fibs{1,1,2,3,5}; + ensure(fibs[4]==5); +} + +// auto +// +// https://en.cppreference.com/w/cpp/language/auto +// +// Can use auto in place of a more complex type specification, if the compiler can infer the type +template<> template<> +void cpp_features_test_object_t::test<2>() +{ + std::vector numbers{3,6,9}; + + // auto element + auto& aval = numbers[1]; + ensure("auto element", aval==6); + + // auto iterator (non-const) + auto it = numbers.rbegin(); + *it += 1; + S32 val = *it; + ensure("auto iterator", val==10); +} + +// range for +// +// https://en.cppreference.com/w/cpp/language/range-for +// +// Can iterate over containers without explicit iterator +template<> template<> +void cpp_features_test_object_t::test<3>() +{ + + // Traditional iterator for with container + // + // Problems: + // * Have to create a new variable for the iterator, which is unrelated to the problem you're trying to solve. + // * Redundant and somewhat fragile. Have to make sure begin() and end() are both from the right container. + std::vector numbers{3,6,9}; + for (auto it = numbers.begin(); it != numbers.end(); ++it) + { + auto& n = *it; + n *= 2; + } + ensure("iterator for vector", numbers[2]==18); + + // Range for with container + // + // Under the hood, this is doing the same thing as the traditional + // for loop above. Still uses begin() and end() but you don't have + // to access them directly. + std::vector numbersb{3,6,9}; + for (auto& n: numbersb) + { + n *= 2; + } + ensure("range for vector", numbersb[2]==18); + + // Range for over a C-style array. + // + // This is handy because the language determines the range automatically. + // Getting this right manually is a little trickier. + S32 pows[] = {1,2,4,8,16}; + S32 sum{}; + for (const auto& v: pows) + { + sum += v; + } + ensure("for C-array", sum==31); +} + +// override specifier +// +// https://en.cppreference.com/w/cpp/language/override +// +// Specify that a particular class function is an override of a virtual function. +// Benefits: +// * Makes code somewhat easier to read by showing intent. +// * Prevents mistakes where you think something is an override but it doesn't actually match the declaration in the parent class. +// Drawbacks: +// * Some compilers require that any class using override must use it consistently for all functions. +// This makes switching a class to use override a lot more work. + +class Foo +{ +public: + virtual bool is_happy() const = 0; +}; + +class Bar: public Foo +{ +public: + bool is_happy() const override { return true; } + // Override would fail: non-const declaration doesn't match parent + // bool is_happy() override { return true; } + // Override would fail: wrong name + // bool is_happx() override { return true; } +}; + +template<> template<> +void cpp_features_test_object_t::test<4>() +{ + Bar b; + ensure("override", b.is_happy()); +} + +// final +// +// https://en.cppreference.com/w/cpp/language/final: "Specifies that a +// virtual function cannot be overridden in a derived class or that a +// class cannot be inherited from." + +class Vehicle +{ +public: + virtual bool has_wheels() const = 0; +}; + +class WheeledVehicle: public Vehicle +{ +public: + virtual bool has_wheels() const final override { return true; } +}; + +class Bicycle: public WheeledVehicle +{ +public: + // Error: can't override final version in WheeledVehicle + // virtual bool has_wheels() override const { return true; } +}; + +template<> template<> +void cpp_features_test_object_t::test<5>() +{ + Bicycle bi; + ensure("final", bi.has_wheels()); +} + +// deleted function declaration +// +// https://en.cppreference.com/w/cpp/language/function#Deleted_functions +// +// Typical case: copy constructor doesn't make sense for a particular class, so you want to make +// sure the no one tries to copy-construct an instance of the class, and that the +// compiler won't generate a copy constructor for you automatically. +// Traditional fix is to declare a +// copy constructor but never implement it, giving you a link-time error if anyone tries to use it. +// Now you can explicitly declare a function to be deleted, which has at least two advantages over +// the old way: +// * Makes the intention clear +// * Creates an error sooner, at compile time + +class DoNotCopy +{ +public: + DoNotCopy() {} + DoNotCopy(const DoNotCopy& ref) = delete; +}; + +template<> template<> +void cpp_features_test_object_t::test<6>() +{ + DoNotCopy nc; // OK, default constructor + //DoNotCopy nc2(nc); // No, can't copy + //DoNotCopy nc3 = nc; // No, this also calls copy constructor (even though it looks like an assignment) +} + +// defaulted function declaration +// +// https://en.cppreference.com/w/cpp/language/function#Function_definition +// +// What about the complementary case to the deleted function declaration, where you want a copy constructor +// and are happy with the default implementation the compiler will make (memberwise copy). +// Now you can explicitly declare that too. +// Usage: I guess it makes the intent clearer, but otherwise not obviously useful. +class DefaultCopyOK +{ +public: + DefaultCopyOK() {} + DefaultCopyOK(const DefaultCopyOK&) = default; +}; + +template<> template<> +void cpp_features_test_object_t::test<7>() +{ + DefaultCopyOK d; // OK + DefaultCopyOK d2(d); // OK + DefaultCopyOK d3 = d; // OK +} + + +} // namespace tut -- cgit v1.2.3 From 1a3c5c57396c8faacd18adb2bd830f6907c4d66f Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Wed, 17 Mar 2021 16:04:32 +0100 Subject: SL-14999 - test update --- indra/newview/tests/cppfeatures_test.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/indra/newview/tests/cppfeatures_test.cpp b/indra/newview/tests/cppfeatures_test.cpp index 8399bb12ff..67671dc65e 100644 --- a/indra/newview/tests/cppfeatures_test.cpp +++ b/indra/newview/tests/cppfeatures_test.cpp @@ -229,8 +229,11 @@ void cpp_features_test_object_t::test<6>() class DefaultCopyOK { public: - DefaultCopyOK() {} + DefaultCopyOK(): mVal(123) {} DefaultCopyOK(const DefaultCopyOK&) = default; + S32 val() const { return mVal; } +private: + S32 mVal; }; template<> template<> @@ -239,6 +242,9 @@ void cpp_features_test_object_t::test<7>() DefaultCopyOK d; // OK DefaultCopyOK d2(d); // OK DefaultCopyOK d3 = d; // OK + ensure("default copy d", d.val()==123); + ensure("default copy d2", d.val()==d2.val()); + ensure("default copy d3", d.val()==d3.val()); } -- cgit v1.2.3 From 41a10cceb45f88d3ab433aa129eafd2ffa40256d Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Wed, 17 Mar 2021 20:53:36 +0100 Subject: SL-14999 - more c++ feature tests --- indra/newview/tests/cppfeatures_test.cpp | 89 ++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/indra/newview/tests/cppfeatures_test.cpp b/indra/newview/tests/cppfeatures_test.cpp index 67671dc65e..bfaa83140d 100644 --- a/indra/newview/tests/cppfeatures_test.cpp +++ b/indra/newview/tests/cppfeatures_test.cpp @@ -247,5 +247,94 @@ void cpp_features_test_object_t::test<7>() ensure("default copy d3", d.val()==d3.val()); } +// initialize class members inline +// +// https://en.cppreference.com/w/cpp/language/data_members#Member_initialization +// +// Default class member values can be set where they are declared, using either brackets or = + +// It is preferred to skip creating a constructor if all the work can be done by inline initialization: +// http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines.html#c45-dont-define-a-default-constructor-that-only-initializes-data-members-use-in-class-member-initializers-instead +// +class InitInline +{ +public: + S32 mFoo{10}; +}; + +class InitInlineWithConstructor +{ +public: + // Here mFoo is not specified, so you will get the default value of 10. + // mBar is specified, so 25 will override the default value. + InitInlineWithConstructor(): + mBar(25) + {} + + // Default values set using two different styles, same effect. + S32 mFoo{10}; + S32 mBar = 20; +}; + +template<> template<> +void cpp_features_test_object_t::test<8>() +{ + InitInline ii; + ensure("init member inline 1", ii.mFoo==10); + + InitInlineWithConstructor iici; + ensure("init member inline 2", iici.mFoo=10); + ensure("init member inline 3", iici.mBar==25); +} + +// constexpr +// +// https://en.cppreference.com/w/cpp/language/constexpr +// +// Various things can be computed at compile time, and flagged as constexpr. +constexpr S32 compute2() { return 2; } + +constexpr S32 ce_factorial(S32 n) +{ + if (n<=0) + { + return 1; + } + else + { + return n*ce_factorial(n-1); + } +} + +template<> template<> +void cpp_features_test_object_t::test<9>() +{ + S32 val = compute2(); + ensure("constexpr 1", val==2); + + // Compile-time factorial. You used to need complex templates to do something this useless. + S32 fac5 = ce_factorial(5); + ensure("constexpr 2", fac5==120); +} + +// static assert +// +// https://en.cppreference.com/w/cpp/language/static_assert +// +// You can add asserts to be checked at compile time. The thing to be checked must be a constexpr. +// There are two forms: +// * static_assert(expr); +// * static_assert(expr, message); +// +// Currently only the 2-parameter form works on windows. The 1-parameter form needs a flag we don't set. + +template<> template<> +void cpp_features_test_object_t::test<10>() +{ + // static_assert(ce_factorial(6)==720); No, needs a flag we don't currently set. + static_assert(ce_factorial(6)==720, "bad factorial"); // OK +} + + } // namespace tut -- cgit v1.2.3 From 026f4c6024ced642e7fff1949a38657b40a92c45 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 26 Mar 2021 14:41:58 +0100 Subject: SL-14999 - more cpp features tests --- indra/newview/tests/cppfeatures_test.cpp | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/indra/newview/tests/cppfeatures_test.cpp b/indra/newview/tests/cppfeatures_test.cpp index bfaa83140d..9f052c11c4 100644 --- a/indra/newview/tests/cppfeatures_test.cpp +++ b/indra/newview/tests/cppfeatures_test.cpp @@ -335,6 +335,49 @@ void cpp_features_test_object_t::test<10>() static_assert(ce_factorial(6)==720, "bad factorial"); // OK } +// type aliases +// +// https://en.cppreference.com/w/cpp/language/type_alias +// +// You can use the "using" statement to create simpler templates that +// are aliases for more complex ones. "Template typedef" + +// This makes stringmap an alias for std::map +template +using stringmap = std::map; + +template<> template<> +void cpp_features_test_object_t::test<11>() +{ + stringmap name_counts{ {"alice", 3}, {"bob", 2} }; + ensure("type alias", name_counts["bob"]==2); +} + + +// nullptr + +// enums + +// std::unique_ptr and make_unique + +// std::shared_ptr and make_shared + +// lambdas + +// perfect forwarding + +// variadic templates + +// std::thread + +// std::mutex + +// thread_local + +// rvalue reference && + +// move semantics +// std::move } // namespace tut -- cgit v1.2.3 From 61ab98212073f0093f044bc78a6ccefac0f63761 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 26 Mar 2021 14:42:40 +0100 Subject: SL-15031 - ViewerStats analysis script --- scripts/metrics/viewerstats.py | 92 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/metrics/viewerstats.py diff --git a/scripts/metrics/viewerstats.py b/scripts/metrics/viewerstats.py new file mode 100644 index 0000000000..1e54d2cfd0 --- /dev/null +++ b/scripts/metrics/viewerstats.py @@ -0,0 +1,92 @@ +#!runpy.sh + +"""\ + +This module contains code for analyzing ViewerStats data as uploaded by the viewer. + +$LicenseInfo:firstyear=2021&license=viewerlgpl$ +Second Life Viewer Source Code +Copyright (C) 2021, Linden Research, Inc. + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; +version 2.1 of the License only. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +$/LicenseInfo$ +""" + +import argparse +import numpy as np +import pandas as pd +import json +from collections import Counter, defaultdict + +def show_stats_by_key(recs,indices): + cnt = Counter() + per_key_cnt = defaultdict(Counter) + for r in recs: + try: + d = r + for idx in indices: + d = d[idx] + for k,v in d.items(): + if isinstance(v,dict): + continue + cnt[k] += 1 + if isinstance(v,list): + v = tuple(v) + per_key_cnt[k][v] += 1 + except Exception as e: + print "err", e + print "d", d, "k", k, "v", v + raise + mc = cnt.most_common(100) + print "=========================" + keyprefix = "" + if len(indices)>0: + keyprefix = ".".join(indices) + "." + for i,m in enumerate(mc): + k = m[0] + bigc = m[1] + kmc = per_key_cnt[k].most_common(5) + print i, keyprefix+str(k), bigc + for v in kmc: + print " ", "value",v[0],"count",v[1] + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="process tab-separated table containing viewerstats logs") + parser.add_argument("--verbose", action="store_true",help="verbose flag") + parser.add_argument("--preferences", action="store_true", help="analyze preference info") + parser.add_argument("--column", help="name of column containing viewerstats info") + parser.add_argument("infiles", nargs="+", help="name of .tsv files to process") + args = parser.parse_args() + + for fname in args.infiles: + print "process", fname + df = pd.read_csv(fname,sep='\t') + #print "DF", df.describe() + jstrs = df['RAW_LOG:BODY'] + #print "JSTRS", jstrs.describe() + recs = [] + for i,jstr in enumerate(jstrs): + recs.append(json.loads(jstr)) + show_stats_by_key(recs,[]) + show_stats_by_key(recs,["agent"]) + if args.preferences: + show_stats_by_key(recs,["preferences","settings"]) + + + + -- cgit v1.2.3 From e0b414921590152c51a1efc31d0c22c3957d437f Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Fri, 26 Mar 2021 14:43:05 +0100 Subject: SL-15031 - chmod flag --- scripts/metrics/viewerstats.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/metrics/viewerstats.py diff --git a/scripts/metrics/viewerstats.py b/scripts/metrics/viewerstats.py old mode 100644 new mode 100755 -- cgit v1.2.3 From 2e255caae1cb4f82c9f7f3d9233c9865230d599b Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 30 Mar 2021 16:59:17 +0100 Subject: SL-15031 - ViewerStats analysis compare to settings.xml --- scripts/metrics/viewerstats.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/scripts/metrics/viewerstats.py b/scripts/metrics/viewerstats.py index 1e54d2cfd0..740540b3dd 100755 --- a/scripts/metrics/viewerstats.py +++ b/scripts/metrics/viewerstats.py @@ -31,8 +31,9 @@ import numpy as np import pandas as pd import json from collections import Counter, defaultdict +from llbase import llsd -def show_stats_by_key(recs,indices): +def show_stats_by_key(recs,indices,settings_sd = None): cnt = Counter() per_key_cnt = defaultdict(Counter) for r in recs: @@ -51,7 +52,7 @@ def show_stats_by_key(recs,indices): print "err", e print "d", d, "k", k, "v", v raise - mc = cnt.most_common(100) + mc = cnt.most_common() print "=========================" keyprefix = "" if len(indices)>0: @@ -59,10 +60,30 @@ def show_stats_by_key(recs,indices): for i,m in enumerate(mc): k = m[0] bigc = m[1] + unset_cnt = len(recs) - bigc kmc = per_key_cnt[k].most_common(5) print i, keyprefix+str(k), bigc + if settings_sd is not None and k in settings_sd and "Value" in settings_sd[k]: + print " ", "default",settings_sd[k]["Value"],"count",unset_cnt for v in kmc: print " ", "value",v[0],"count",v[1] + if settings_sd is not None: + print "Total keys in settings", len(settings_sd.keys()) + unused_keys = list(set(settings_sd.keys()) - set(cnt.keys())) + print "\nUnused_keys", len(unused_keys) + print "======================" + print "\n".join(sorted(unused_keys)) + unrec_keys = list(set(cnt.keys()) - set(settings_sd.keys())) + print "\nUnrecognized keys", len(unrec_keys) + print "======================" + print "\n".join(sorted(unrec_keys)) + +def parse_settings_xml(): + # assume we're in scripts/metrics + fname = "../../indra/newview/app_settings/settings.xml" + with open(fname,"r") as f: + contents = f.read() + return llsd.parse_xml(contents) if __name__ == "__main__": @@ -85,7 +106,10 @@ if __name__ == "__main__": show_stats_by_key(recs,[]) show_stats_by_key(recs,["agent"]) if args.preferences: - show_stats_by_key(recs,["preferences","settings"]) + settings_sd = parse_settings_xml() + #for skey,svals in settings_sd.items(): + # print skey, "=>", svals + show_stats_by_key(recs,["preferences","settings"],settings_sd) -- cgit v1.2.3 From 6c1dc74b75fc3d8967c54a3f95e5ce57ba4c4716 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Thu, 1 Apr 2021 15:55:57 +0100 Subject: SL-15031 - include settings_per_account stats --- scripts/metrics/viewerstats.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/scripts/metrics/viewerstats.py b/scripts/metrics/viewerstats.py index 740540b3dd..1e2b8bc4ff 100755 --- a/scripts/metrics/viewerstats.py +++ b/scripts/metrics/viewerstats.py @@ -70,17 +70,28 @@ def show_stats_by_key(recs,indices,settings_sd = None): if settings_sd is not None: print "Total keys in settings", len(settings_sd.keys()) unused_keys = list(set(settings_sd.keys()) - set(cnt.keys())) - print "\nUnused_keys", len(unused_keys) + unused_keys_non_str = [k for k in unused_keys if settings_sd[k]["Type"] != "String"] + unused_keys_str = [k for k in unused_keys if settings_sd[k]["Type"] == "String"] + + # Things that no one in the sample has set to a non-default value. Possible candidates for removal. + print "\nUnused_keys_non_str", len(unused_keys_non_str) + print "======================" + print "\n".join(sorted(unused_keys_non_str)) + + # Strings are not currently logged, so we have no info on usage. + print "\nString keys (usage unknown)", len(unused_keys_str) print "======================" - print "\n".join(sorted(unused_keys)) + print "\n".join(sorted(unused_keys_str)) + + # Things that someone has set but that aren't recognized settings. unrec_keys = list(set(cnt.keys()) - set(settings_sd.keys())) print "\nUnrecognized keys", len(unrec_keys) print "======================" print "\n".join(sorted(unrec_keys)) -def parse_settings_xml(): +def parse_settings_xml(fname): # assume we're in scripts/metrics - fname = "../../indra/newview/app_settings/settings.xml" + fname = "../../indra/newview/app_settings/" + fname with open(fname,"r") as f: contents = f.read() return llsd.parse_xml(contents) @@ -106,11 +117,16 @@ if __name__ == "__main__": show_stats_by_key(recs,[]) show_stats_by_key(recs,["agent"]) if args.preferences: - settings_sd = parse_settings_xml() + print "\nSETTINGS.XML" + settings_sd = parse_settings_xml("settings.xml") #for skey,svals in settings_sd.items(): # print skey, "=>", svals show_stats_by_key(recs,["preferences","settings"],settings_sd) - + print + + print "\nSETTINGS_PER_ACCOUNT.XML" + settings_pa_sd = parse_settings_xml("settings_per_account.xml") + show_stats_by_key(recs,["preferences","settings_per_account"],settings_pa_sd) -- cgit v1.2.3 From 9013a04eddc9631c5040d50ad766e5708801dcab Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Thu, 1 Apr 2021 16:51:27 +0100 Subject: SL-14999 - note --- indra/newview/tests/cppfeatures_test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/indra/newview/tests/cppfeatures_test.cpp b/indra/newview/tests/cppfeatures_test.cpp index 9f052c11c4..923bb1e1b2 100644 --- a/indra/newview/tests/cppfeatures_test.cpp +++ b/indra/newview/tests/cppfeatures_test.cpp @@ -353,10 +353,11 @@ void cpp_features_test_object_t::test<11>() ensure("type alias", name_counts["bob"]==2); } +// Other possibilities: // nullptr -// enums +// class enums // std::unique_ptr and make_unique @@ -380,4 +381,6 @@ void cpp_features_test_object_t::test<11>() // std::move +// string_view + } // namespace tut -- cgit v1.2.3 From 704aeb09d81f0fa37e343460edb1ab11ba3dbc8c Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Tue, 6 Apr 2021 16:32:19 +0100 Subject: SL-15031 - option to remove unreferenced settings --- scripts/metrics/viewerstats.py | 55 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/metrics/viewerstats.py b/scripts/metrics/viewerstats.py index 1e2b8bc4ff..c34ed440c9 100755 --- a/scripts/metrics/viewerstats.py +++ b/scripts/metrics/viewerstats.py @@ -32,8 +32,11 @@ import pandas as pd import json from collections import Counter, defaultdict from llbase import llsd +import io +import re def show_stats_by_key(recs,indices,settings_sd = None): + result = () cnt = Counter() per_key_cnt = defaultdict(Counter) for r in recs: @@ -89,6 +92,9 @@ def show_stats_by_key(recs,indices,settings_sd = None): print "======================" print "\n".join(sorted(unrec_keys)) + result = (settings_sd.keys(), unused_keys_str, unused_keys_non_str, unrec_keys) + return result + def parse_settings_xml(fname): # assume we're in scripts/metrics fname = "../../indra/newview/app_settings/" + fname @@ -96,11 +102,40 @@ def parse_settings_xml(fname): contents = f.read() return llsd.parse_xml(contents) +def read_settings_xml(fname): + # assume we're in scripts/metrics + fname = "../../indra/newview/app_settings/" + fname + contents = None + with open(fname,"r") as f: + contents = f.read() + return contents + +def write_settings_xml(fname, contents): + # assume we're in scripts/metrics + fname = "../../indra/newview/app_settings/" + fname + with open(fname,"w") as f: + f.write(llsd.format_pretty_xml(contents)) + f.close() + +def write_raw_settings_xml(fname, string): + # assume we're in scripts/metrics + fname = "../../indra/newview/app_settings/" + fname + with io.open(fname,"w", newline='\n') as f: + f.write(string.decode('latin1')) + f.close() + +def remove_settings(string, to_remove): + for r in to_remove: + subs_str = r"" + r + r"<.*?" + string = re.sub(subs_str,"",string,flags=re.S|re.DOTALL) + return string + if __name__ == "__main__": parser = argparse.ArgumentParser(description="process tab-separated table containing viewerstats logs") parser.add_argument("--verbose", action="store_true",help="verbose flag") parser.add_argument("--preferences", action="store_true", help="analyze preference info") + parser.add_argument("--remove_unused", action="store_true", help="remove unused preferences") parser.add_argument("--column", help="name of column containing viewerstats info") parser.add_argument("infiles", nargs="+", help="name of .tsv files to process") args = parser.parse_args() @@ -121,12 +156,30 @@ if __name__ == "__main__": settings_sd = parse_settings_xml("settings.xml") #for skey,svals in settings_sd.items(): # print skey, "=>", svals - show_stats_by_key(recs,["preferences","settings"],settings_sd) + (all_str,_,_,_) = show_stats_by_key(recs,["preferences","settings"],settings_sd) print print "\nSETTINGS_PER_ACCOUNT.XML" settings_pa_sd = parse_settings_xml("settings_per_account.xml") show_stats_by_key(recs,["preferences","settings_per_account"],settings_pa_sd) + if args.remove_unused: + # quotestrings created by + # % find . -name '*.cpp' -o -name '*.hpp' -o -name '*.h' -o -name '*cmd_line.xml' | xargs grep -ohP '[\">][a-zA-Z0-9]*[\"<]' | sort | uniq > ../scripts/metrics/quotestrings.out + with open("quotestrings.out", "r") as f: + used_strings = f.readlines() + used_strings = [u[1:-2] for u in used_strings] + #print "\n".join(used_strings) + unref_strings = list(set(all_str)-set(used_strings)) + print "\nUNREF_IN_CODE " + str(len(unref_strings)) + "\n" + print "\n".join(unref_strings) + settings_str = read_settings_xml("settings.xml") + # Do this via direct string munging to generate minimal changeset + settings_edited = remove_settings(settings_str,unref_strings) + write_raw_settings_xml("settings.xml",settings_edited) + + + + -- cgit v1.2.3 From d2f76612a7e47073e5e780a99866afc81efbbda9 Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Wed, 7 Apr 2021 19:48:27 +0100 Subject: SL-15031 - more viewerstats.py support for cleaning up unused settings --- scripts/metrics/viewerstats.py | 77 ++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/scripts/metrics/viewerstats.py b/scripts/metrics/viewerstats.py index c34ed440c9..f7be3d967e 100755 --- a/scripts/metrics/viewerstats.py +++ b/scripts/metrics/viewerstats.py @@ -34,6 +34,8 @@ from collections import Counter, defaultdict from llbase import llsd import io import re +import os +import sys def show_stats_by_key(recs,indices,settings_sd = None): result = () @@ -102,7 +104,7 @@ def parse_settings_xml(fname): contents = f.read() return llsd.parse_xml(contents) -def read_settings_xml(fname): +def read_raw_settings_xml(fname): # assume we're in scripts/metrics fname = "../../indra/newview/app_settings/" + fname contents = None @@ -126,10 +128,38 @@ def write_raw_settings_xml(fname, string): def remove_settings(string, to_remove): for r in to_remove: - subs_str = r"" + r + r"<.*?" + subs_str = r"" + r + r"<.*?\n" string = re.sub(subs_str,"",string,flags=re.S|re.DOTALL) return string +def get_used_strings(root_dir): + used_str = set() + skipped_ext = set() + for dir_name, sub_dir_list, file_list in os.walk(root_dir): + for fname in file_list: + if fname in ["settings.xml", "settings.xml.edit", "settings_per_account.xml"]: + print "skip", fname + continue + (base,ext) = os.path.splitext(fname) + #if ext not in [".cpp", ".hpp", ".h", ".xml"]: + # skipped_ext.add(ext) + # continue + + full_name = os.path.join(dir_name,fname) + + with open(full_name,"r") as f: + #print full_name + lines = f.readlines() + for l in lines: + ms = re.findall(r'[>\"]([A-Za-z0-9_]+)[\"<]',l) + for m in ms: + #print "used_str",m + used_str.add(m) + print "skipped extensions", skipped_ext + print "got used_str", len(used_str) + return used_str + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="process tab-separated table containing viewerstats logs") @@ -159,24 +189,35 @@ if __name__ == "__main__": (all_str,_,_,_) = show_stats_by_key(recs,["preferences","settings"],settings_sd) print - print "\nSETTINGS_PER_ACCOUNT.XML" - settings_pa_sd = parse_settings_xml("settings_per_account.xml") - show_stats_by_key(recs,["preferences","settings_per_account"],settings_pa_sd) + #print "\nSETTINGS_PER_ACCOUNT.XML" + #settings_pa_sd = parse_settings_xml("settings_per_account.xml") + #show_stats_by_key(recs,["preferences","settings_per_account"],settings_pa_sd) if args.remove_unused: - # quotestrings created by - # % find . -name '*.cpp' -o -name '*.hpp' -o -name '*.h' -o -name '*cmd_line.xml' | xargs grep -ohP '[\">][a-zA-Z0-9]*[\"<]' | sort | uniq > ../scripts/metrics/quotestrings.out - with open("quotestrings.out", "r") as f: - used_strings = f.readlines() - used_strings = [u[1:-2] for u in used_strings] - #print "\n".join(used_strings) - unref_strings = list(set(all_str)-set(used_strings)) - print "\nUNREF_IN_CODE " + str(len(unref_strings)) + "\n" - print "\n".join(unref_strings) - settings_str = read_settings_xml("settings.xml") - # Do this via direct string munging to generate minimal changeset - settings_edited = remove_settings(settings_str,unref_strings) - write_raw_settings_xml("settings.xml",settings_edited) + # walk codebase looking for strings + all_str_set = set(all_str) + used_strings = get_used_strings("../../indra") + used_strings_set = set(used_strings) + unref_strings = all_str_set-used_strings_set + # Some settings names are generated by appending to a prefix. Need to look for this case. + prefix_used = set() + print "checking unref_strings", len(unref_strings) + for u in unref_strings: + for k in range(6,len(u)): + prefix = u[0:k] + if prefix in all_str_set and prefix in used_strings_set: + prefix_used.add(u) + #print "PREFIX_USED",u,prefix + print "PREFIX_USED", len(prefix_used), ",".join(list(prefix_used)) + print + unref_strings = unref_strings - prefix_used + + print "\nUNREF_IN_CODE " + str(len(unref_strings)) + "\n" + print "\n".join(list(unref_strings)) + settings_str = read_raw_settings_xml("settings.xml") + # Do this via direct string munging to generate minimal changeset + settings_edited = remove_settings(settings_str,unref_strings) + write_raw_settings_xml("settings.xml.edit",settings_edited) -- cgit v1.2.3