From 8d7cde22c31a59d0503230334b1d68e858364c9a Mon Sep 17 00:00:00 2001 From: Signal Linden Date: Tue, 11 Oct 2022 15:10:04 -0700 Subject: Replace llbase with llsd module --- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..be7ec12094 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1706,7 +1706,7 @@ namespace tut // scanner. import_llsd("import os.path\n" "import sys\n" - "from llbase import llsd\n") + "import llsd\n") {} ~TestPythonCompatible() {} -- cgit v1.2.3 From 6516c9d07db42beba5ba9c0c41a33925794a249c Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Jun 2023 07:44:42 -0400 Subject: SL-18837: NamedTempFile back to std::function, use boost::phoenix << It seems the problem addressed by aab769e wasn't some synergy between Boost.Phoenix and Boost.Function, but rather the lack of a Phoenix header file introducing operator<<(). --- indra/llcommon/tests/llsdserialize_test.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index d7c11c5021..a0b8519508 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -50,6 +50,7 @@ typedef U32 uint32_t; #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" +#include using namespace boost::phoenix; #include "../llsd.h" -- cgit v1.2.3 From c4366378b61cacb9d87eb2917302fa17e9207491 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Jun 2023 10:04:26 -0400 Subject: SL-18837: Ditch Boost.Phoenix implicit lambda syntax. It's cool to be able to write 'arg1 << "stuff" << var ...;' for a lambda accepting a std::ostream reference, but cascading compile errors mean it's no longer worth trying to make that work -- given actual C++ lambdas. Also clean up a lingering BOOST_FOREACH() and a boost::bind() while at it. --- indra/llcommon/tests/llsdserialize_test.cpp | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index a0b8519508..08bf7fbc51 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -45,13 +45,6 @@ typedef U32 uint32_t; #endif #include "boost/range.hpp" -#include "boost/foreach.hpp" -#include "boost/function.hpp" -#include "boost/bind.hpp" -#include "boost/phoenix/bind/bind_function.hpp" -#include "boost/phoenix/core/argument.hpp" -#include -using namespace boost::phoenix; #include "../llsd.h" #include "../llsdserialize.h" @@ -1802,7 +1795,7 @@ namespace tut // helper for test<3> static void writeLLSDArray(std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item: llsd::inArray(array)) { LLSDSerialize::toNotation(item, out); // It's important to separate with newlines because Python's llsd @@ -1842,21 +1835,21 @@ namespace tut // Create an llsdXXXXXX file containing 'data' serialized to // notation. NamedTempFile file("llsd", - // NamedTempFile's boost::function constructor + // NamedTempFile's function constructor // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [&](std::ostream& out){ writeLLSDArray(out, cdata); }); python("read C++ notation", - placeholders::arg1 << + [&](std::ostream& out){ out << import_llsd << "def parse_each(iterable):\n" " for item in iterable:\n" " yield llsd.parse(item)\n" << pydata << // Don't forget raw-string syntax for Windows pathnames. - "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); + "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n";}); } template<> template<> @@ -1870,7 +1863,7 @@ namespace tut NamedTempFile file("llsd", ""); python("write Python notation", - placeholders::arg1 << + [&](std::ostream& out){ out << import_llsd << "DATA = [\n" " 17,\n" @@ -1884,7 +1877,7 @@ namespace tut // N.B. Using 'print' implicitly adds newlines. "with open(r'" << file.getName() << "', 'w') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " print(llsd.format_notation(item).decode(), file=f)\n";}); std::ifstream inf(file.getName().c_str()); LLSD item; -- cgit v1.2.3 From 95aa00f7427b7d19ab502862b3018012c1bf1904 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 7 Sep 2023 13:40:46 -0400 Subject: SL-18837: Fix minor merge glitch. --- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 2de7df6f36..76e9ecc293 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -2081,7 +2081,7 @@ namespace tut " for item in DATA:\n" " serialized = llsd." << pyformatter << "(item)\n" " f.write(lenformat.pack(len(serialized)))\n" - " f.write(serialized)\n"); + " f.write(serialized)\n";}); std::ifstream inf(file.getName().c_str()); LLSD item; -- cgit v1.2.3 From a45c9f68c3e2600f48b25cc5cc74ef47cd83005b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 8 Sep 2023 08:58:32 -0400 Subject: SL-18837: Add debugging output to llsdserialize_test.cpp. --- indra/llcommon/tests/llsdserialize_test.cpp | 90 ++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 76e9ecc293..ca63e74c6c 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -55,7 +55,9 @@ typedef U32 uint32_t; #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include #include +#include typedef std::function FormatterFunction; typedef std::function ParserFunction; @@ -65,6 +67,81 @@ std::vector string_to_vector(const std::string& str) return std::vector(str.begin(), str.end()); } +// Format a given byte string as 2-digit hex values, no separators +// Usage: std::cout << hexdump(somestring) << ... +class hexdump +{ +public: + hexdump(const char* data, size_t len): + hexdump(reinterpret_cast(data), len) + {} + + hexdump(const U8* data, size_t len): + mData(data, data + len) + {} + + hexdump(const hexdump&) = delete; + hexdump& operator=(const hexdump&) = delete; + + friend std::ostream& operator<<(std::ostream& out, const hexdump& self) + { + auto oldfmt{ out.flags() }; + auto oldfill{ out.fill() }; + out.setf(std::ios_base::hex, std::ios_base::basefield); + out.fill('0'); + for (auto c : self.mData) + { + out << std::setw(2) << unsigned(c); + } + out.setf(oldfmt, std::ios_base::basefield); + out.fill(oldfill); + return out; + } + +private: + std::vector mData; +}; + +// Format a given byte string as a mix of printable characters and, for each +// non-printable character, "\xnn" +// Usage: std::cout << hexmix(somestring) << ... +class hexmix +{ +public: + hexmix(const char* data, size_t len): + mData(data, len) + {} + + hexmix(const hexmix&) = delete; + hexmix& operator=(const hexmix&) = delete; + + friend std::ostream& operator<<(std::ostream& out, const hexmix& self) + { + auto oldfmt{ out.flags() }; + auto oldfill{ out.fill() }; + out.setf(std::ios_base::hex, std::ios_base::basefield); + out.fill('0'); + for (auto c : self.mData) + { + // std::isprint() must be passed an unsigned char! + if (std::isprint(static_cast(c))) + { + out << c; + } + else + { + out << "\\x" << std::setw(2) << unsigned(c); + } + } + out.setf(oldfmt, std::ios_base::basefield); + out.fill(oldfill); + return out; + } + +private: + std::string mData; +}; + namespace tut { struct sd_xml_data @@ -1909,7 +1986,14 @@ namespace tut auto buffstr{ buffer.str() }; int bufflen{ static_cast(buffstr.length()) }; out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); + LL_DEBUGS("topy") << "Wrote length: " + << hexdump(reinterpret_cast(&bufflen), + sizeof(bufflen)) + << LL_ENDL; out.write(buffstr.c_str(), buffstr.length()); + LL_DEBUGS("topy") << "Wrote data: " + << hexmix(buffstr.c_str(), buffstr.length()) + << LL_ENDL; } } @@ -1938,8 +2022,8 @@ namespace tut " else:\n" " raise AssertionError('Too many data items')\n"; - // Create an llsdXXXXXX file containing 'data' serialized to - // notation. + // Create an llsdXXXXXX file containing 'data' serialized per + // FormatterFunction. NamedTempFile file("llsd", // NamedTempFile's function constructor // takes a callable. To this callable it passes the @@ -1958,11 +2042,13 @@ namespace tut "lenformat = struct.Struct('i')\n" "def parse_each(inf):\n" " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " print('Read length:', ''.join(('%02x' % b) for b in rawlen))\n" " len = lenformat.unpack(rawlen)[0]\n" // Since llsd.parse() has no max_bytes argument, instead of // passing the input stream directly to parse(), read the item // into a distinct bytes object and parse that. " data = inf.read(len)\n" + " print('Read data: ', repr(data))\n" " try:\n" " frombytes = llsd.parse(data)\n" " except llsd.LLSDParseError as err:\n" -- cgit v1.2.3 From c7546ea65e55143ff3d2d82d8c289bbac7fffe0f Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 8 Sep 2023 14:14:09 -0400 Subject: SL-18837: Make llsdserialize_test debug output conditional. Move hexdump() and hexmix() stream formatters to new hexdump.h for potential use by other tests. In toPythonUsing() helper function, add a temp file to receive Python script debug output, and direct debug output to that file. On test failure, dump the contents of that file to the log. Give NamedTempFile::peep() an optional target std::ostream; refactor implementation as peep_via() that accepts a callable to process each text line. Add operator<<() to stream the contents of a NamedTempFile object to ostream -- but don't use that with LL_DEBUGS(), as it flattens the file contents into a single log line. Instead add peep_log(), which streams each individual text line to LL_DEBUGS(). --- indra/llcommon/tests/llsdserialize_test.cpp | 202 +++++++++++----------------- 1 file changed, 75 insertions(+), 127 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index ca63e74c6c..ac40125f75 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -52,12 +52,12 @@ typedef U32 uint32_t; #include "llformat.h" #include "llmemorystream.h" +#include "../test/hexdump.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" -#include +#include "StringVec.h" #include -#include typedef std::function FormatterFunction; typedef std::function ParserFunction; @@ -67,81 +67,6 @@ std::vector string_to_vector(const std::string& str) return std::vector(str.begin(), str.end()); } -// Format a given byte string as 2-digit hex values, no separators -// Usage: std::cout << hexdump(somestring) << ... -class hexdump -{ -public: - hexdump(const char* data, size_t len): - hexdump(reinterpret_cast(data), len) - {} - - hexdump(const U8* data, size_t len): - mData(data, data + len) - {} - - hexdump(const hexdump&) = delete; - hexdump& operator=(const hexdump&) = delete; - - friend std::ostream& operator<<(std::ostream& out, const hexdump& self) - { - auto oldfmt{ out.flags() }; - auto oldfill{ out.fill() }; - out.setf(std::ios_base::hex, std::ios_base::basefield); - out.fill('0'); - for (auto c : self.mData) - { - out << std::setw(2) << unsigned(c); - } - out.setf(oldfmt, std::ios_base::basefield); - out.fill(oldfill); - return out; - } - -private: - std::vector mData; -}; - -// Format a given byte string as a mix of printable characters and, for each -// non-printable character, "\xnn" -// Usage: std::cout << hexmix(somestring) << ... -class hexmix -{ -public: - hexmix(const char* data, size_t len): - mData(data, len) - {} - - hexmix(const hexmix&) = delete; - hexmix& operator=(const hexmix&) = delete; - - friend std::ostream& operator<<(std::ostream& out, const hexmix& self) - { - auto oldfmt{ out.flags() }; - auto oldfill{ out.fill() }; - out.setf(std::ios_base::hex, std::ios_base::basefield); - out.fill('0'); - for (auto c : self.mData) - { - // std::isprint() must be passed an unsigned char! - if (std::isprint(static_cast(c))) - { - out << c; - } - else - { - out << "\\x" << std::setw(2) << unsigned(c); - } - } - out.setf(oldfmt, std::ios_base::basefield); - out.fill(oldfill); - return out; - } - -private: - std::string mData; -}; - namespace tut { struct sd_xml_data @@ -1868,16 +1793,12 @@ namespace tut // helper for TestPythonCompatible static std::string import_llsd("import os.path\n" "import sys\n" - "try:\n" - // new freestanding llsd package - " import llsd\n" - "except ImportError:\n" - // older llbase.llsd module - " from llbase import llsd\n"); + "import llsd\n"); // helper for TestPythonCompatible - template - void python(const std::string& desc, const CONTENT& script, int expect=0) + template + void python_expect(const std::string& desc, const CONTENT& script, int expect=0, + ARGS&&... args) { auto PYTHON(LLStringUtil::getenv("PYTHON")); ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); @@ -1888,7 +1809,8 @@ namespace tut std::string q("\""); std::string qPYTHON(q + PYTHON + q); std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), + std::forward(args)..., NULL); if (rc == -1) { char buffer[256]; @@ -1904,6 +1826,10 @@ namespace tut LLProcess::Params params; params.executable = PYTHON; params.args.add(scriptfile.getName()); + for (const std::string& arg : StringVec{ std::forward(args)... }) + { + params.args.add(arg); + } LLProcessPtr py(LLProcess::create(params)); ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); // Implementing timeout would mean messing with alarm() and @@ -1938,6 +1864,14 @@ namespace tut #endif } + // helper for TestPythonCompatible + template + void python(const std::string& desc, const CONTENT& script, ARGS&&... args) + { + // plain python() expects rc 0 + python_expect(desc, script, 0, std::forward(args)...); + } + struct TestPythonCompatible { TestPythonCompatible() {} @@ -1952,10 +1886,10 @@ namespace tut void TestPythonCompatibleObject::test<1>() { set_test_name("verify python()"); - python("hello", - "import sys\n" - "sys.exit(17)\n", - 17); // expect nonzero rc + python_expect("hello", + "import sys\n" + "sys.exit(17)\n", + 17); // expect nonzero rc } template<> template<> @@ -1986,14 +1920,14 @@ namespace tut auto buffstr{ buffer.str() }; int bufflen{ static_cast(buffstr.length()) }; out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); - LL_DEBUGS("topy") << "Wrote length: " - << hexdump(reinterpret_cast(&bufflen), - sizeof(bufflen)) - << LL_ENDL; + LL_DEBUGS() << "Wrote length: " + << hexdump(reinterpret_cast(&bufflen), + sizeof(bufflen)) + << LL_ENDL; out.write(buffstr.c_str(), buffstr.length()); - LL_DEBUGS("topy") << "Wrote data: " - << hexmix(buffstr.c_str(), buffstr.length()) - << LL_ENDL; + LL_DEBUGS() << "Wrote data: " + << hexmix(buffstr.c_str(), buffstr.length()) + << LL_ENDL; } } @@ -2033,36 +1967,50 @@ namespace tut (std::ostream& out) { writeLLSDArray(serialize, out, cdata); }); - python("read C++ " + desc, - [&](std::ostream& out){ out << - import_llsd << - "from functools import partial\n" - "import io\n" - "import struct\n" - "lenformat = struct.Struct('i')\n" - "def parse_each(inf):\n" - " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" - " print('Read length:', ''.join(('%02x' % b) for b in rawlen))\n" - " len = lenformat.unpack(rawlen)[0]\n" - // Since llsd.parse() has no max_bytes argument, instead of - // passing the input stream directly to parse(), read the item - // into a distinct bytes object and parse that. - " data = inf.read(len)\n" - " print('Read data: ', repr(data))\n" - " try:\n" - " frombytes = llsd.parse(data)\n" - " except llsd.LLSDParseError as err:\n" - " print(f'*** {err}')\n" - " print(f'Bad content:\\n{data!r}')\n" - " raise\n" - // Also try parsing from a distinct stream. - " stream = io.BytesIO(data)\n" - " fromstream = llsd.parse(stream)\n" - " assert frombytes == fromstream\n" - " yield frombytes\n" - << pydata << - // Don't forget raw-string syntax for Windows pathnames. - "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n";}); + // 'debug' starts empty because it's intended as an output file + NamedTempFile debug("debug", ""); + + try + { + python("read C++ " + desc, + [&](std::ostream& out){ out << + import_llsd << + "from functools import partial\n" + "import io\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " print('Read length:', ''.join(('%02x' % b) for b in rawlen),\n" + " file=debug)\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " print('Read data: ', repr(data), file=debug)\n" + " try:\n" + " frombytes = llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" + << pydata << + // Don't forget raw-string syntax for Windows pathnames. + "debug = open(r'" << debug.getName() << "', 'w')\n" + "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n";}); + } + catch (const failure&) + { + LL_DEBUGS() << "Script debug output:" << LL_ENDL; + debug.peep_log(); + throw; + } } template<> template<> -- cgit v1.2.3