From abe62c23d74d5319691a49881719b03cc9b5b090 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 22 Nov 2022 16:21:44 -0500 Subject: SL-18330: Make LLSDSerialize::deserialize() default to notation. LLSDSerialize::serialize() emits a header string, e.g. "" for notation format. Until now, LLSDSerialize::deserialize() has required that header to properly decode the input stream. But none of LLSDBinaryFormatter, LLSDXMLFormatter or LLSDNotationFormatter emit that header themselves. Nor do any of the Python llsd.format_binary(), format_xml() or format_notation() functions. Until now, you could not use LLSD::deserialize() to parse an arbitrary-format LLSD stream serialized by anything but LLSDSerialize::serialize(). Change LLSDSerialize::deserialize() so that if no header is recognized, instead of failing, it attempts to parse as notation. Add tests to exercise this case. The tricky part about this processing is that deserialize() necessarily reads some number of bytes from the input stream first, to try to recognize the header. If it fails to do so, it must prepend the bytes it has already read to the rest of the input stream since they're probably the beginning of the serialized data. To support this use case, introduce cat_streambuf, a std::streambuf subclass that (virtually) concatenates other std::streambuf instances. When read by a std::istream, the sequence of underlying std::streambufs appears to the consumer as a single continuous stream. --- indra/llcommon/tests/llsdserialize_test.cpp | 192 +++++++++++++++++++--------- 1 file changed, 130 insertions(+), 62 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..0334699c7f 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -60,6 +60,7 @@ using namespace boost::phoenix; #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include std::vector string_to_vector(const std::string& str) { @@ -112,7 +113,7 @@ namespace tut mSD = LLUUID::null; expected = "\n"; xml_test("null uuid", expected); - + mSD = LLUUID("c96f9b1e-f589-4100-9774-d98643ce0bed"); expected = "c96f9b1e-f589-4100-9774-d98643ce0bed\n"; xml_test("uuid", expected); @@ -136,7 +137,7 @@ namespace tut expected = "aGVsbG8=\n"; xml_test("binary", expected); } - + template<> template<> void sd_xml_object::test<2>() { @@ -225,7 +226,7 @@ namespace tut expected = "bazfoobar\n"; xml_test("2 element map", expected); } - + template<> template<> void sd_xml_object::test<6>() { @@ -241,7 +242,7 @@ namespace tut expected = "Nnw2fGFzZGZoYXBweWJveHw2MGU0NGVjNS0zMDVjLTQzYzItOWExOS1iNGI4OWIxYWUyYTZ8NjBlNDRlYzUtMzA1Yy00M2MyLTlhMTktYjRiODliMWFlMmE2fDYwZTQ0ZWM1LTMwNWMtNDNjMi05YTE5LWI0Yjg5YjFhZTJhNnwwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDB8N2ZmZmZmZmZ8N2ZmZmZmZmZ8MHwwfDgyMDAwfDQ1MGZlMzk0LTI5MDQtYzlhZC0yMTRjLWEwN2ViN2ZlZWMyOXwoTm8gRGVzY3JpcHRpb24pfDB8MTB8MA==\n"; xml_test("binary", expected); } - + class TestLLSDSerializeData { public: @@ -250,9 +251,34 @@ namespace tut void doRoundTripTests(const std::string&); void checkRoundTrip(const std::string&, const LLSD& v); - - LLPointer mFormatter; - LLPointer mParser; + + void setFormatterParser(LLPointer formatter, LLPointer parser) + { + mFormatter = [formatter](const LLSD& data, std::ostream& str) + { + formatter->format(data, str); + }; + // this lambda must be mutable since otherwise the bound 'parser' + // is assumed to point to a const LLSDParser + mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) mutable + { + // reset() call is needed since test code re-uses parser object + parser->reset(); + return (parser->parse(istr, data, max_bytes) > 0); + }; + } + + void setParser(bool (*parser)(LLSD&, std::istream&, size_t)) + { + // why does LLSDSerialize::deserialize() reverse the parse() params?? + mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) + { + return (parser(data, istr, max_bytes) > 0); + }; + } + + std::function mFormatter; + std::function mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -265,12 +291,11 @@ namespace tut void TestLLSDSerializeData::checkRoundTrip(const std::string& msg, const LLSD& v) { - std::stringstream stream; - mFormatter->format(v, stream); + std::stringstream stream; + mFormatter(v, stream); //LL_INFOS() << "checkRoundTrip: length " << stream.str().length() << LL_ENDL; LLSD w; - mParser->reset(); // reset() call is needed since test code re-uses mParser - mParser->parse(stream, w, stream.str().size()); + mParser(stream, w, stream.str().size()); try { @@ -299,52 +324,52 @@ namespace tut fillmap(root[key], width, depth - 1); } } - + void TestLLSDSerializeData::doRoundTripTests(const std::string& msg) { LLSD v; checkRoundTrip(msg + " undefined", v); - + v = true; checkRoundTrip(msg + " true bool", v); - + v = false; checkRoundTrip(msg + " false bool", v); - + v = 1; checkRoundTrip(msg + " positive int", v); - + v = 0; checkRoundTrip(msg + " zero int", v); - + v = -1; checkRoundTrip(msg + " negative int", v); - + v = 1234.5f; checkRoundTrip(msg + " positive float", v); - + v = 0.0f; checkRoundTrip(msg + " zero float", v); - + v = -1234.5f; checkRoundTrip(msg + " negative float", v); - + // FIXME: need a NaN test - + v = LLUUID::null; checkRoundTrip(msg + " null uuid", v); - + LLUUID newUUID; newUUID.generate(); v = newUUID; checkRoundTrip(msg + " new uuid", v); - + v = ""; checkRoundTrip(msg + " empty string", v); - + v = "some string"; checkRoundTrip(msg + " non-empty string", v); - + v = "Second Life is a 3-D virtual world entirely built and owned by its residents. " "Since opening to the public in 2003, it has grown explosively and today is " @@ -372,7 +397,7 @@ namespace tut for (U32 block = 0x000000; block <= 0x10ffff; block += block_size) { std::ostringstream out; - + for (U32 c = block; c < block + block_size; ++c) { if (c <= 0x000001f @@ -386,7 +411,7 @@ namespace tut if (0x00fdd0 <= c && c <= 0x00fdef) { continue; } if ((c & 0x00fffe) == 0x00fffe) { continue; } // see Unicode standard, section 15.8 - + if (c <= 0x00007f) { out << (char)(c & 0x7f); @@ -410,55 +435,55 @@ namespace tut out << (char)(0x80 | ((c >> 0) & 0x3f)); } } - + v = out.str(); std::ostringstream blockmsg; blockmsg << msg << " unicode string block 0x" << std::hex << block; checkRoundTrip(blockmsg.str(), v); } - + LLDate epoch; v = epoch; checkRoundTrip(msg + " epoch date", v); - + LLDate aDay("2002-12-07T05:07:15.00Z"); v = aDay; checkRoundTrip(msg + " date", v); - + LLURI path("http://slurl.com/secondlife/Ambleside/57/104/26/"); v = path; checkRoundTrip(msg + " url", v); - + const char source[] = "it must be a blue moon again"; std::vector data; // note, includes terminating '\0' copy(&source[0], &source[sizeof(source)], back_inserter(data)); - + v = data; checkRoundTrip(msg + " binary", v); - + v = LLSD::emptyMap(); checkRoundTrip(msg + " empty map", v); - + v = LLSD::emptyMap(); v["name"] = "luke"; //v.insert("name", "luke"); v["age"] = 3; //v.insert("age", 3); checkRoundTrip(msg + " map", v); - + v.clear(); v["a"]["1"] = true; v["b"]["0"] = false; checkRoundTrip(msg + " nested maps", v); - + v = LLSD::emptyArray(); checkRoundTrip(msg + " empty array", v); - + v = LLSD::emptyArray(); v.append("ali"); v.append(28); checkRoundTrip(msg + " array", v); - + v.clear(); v[0][0] = true; v[1][0] = false; @@ -468,7 +493,7 @@ namespace tut fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } - + typedef tut::test_group TestLLSDSerializeGroup; typedef TestLLSDSerializeGroup::object TestLLSDSerializeObject; TestLLSDSerializeGroup gTestLLSDSerializeGroup("llsd serialization"); @@ -476,35 +501,78 @@ namespace tut template<> template<> void TestLLSDSerializeObject::test<1>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY), + new LLSDNotationParser()); doRoundTripTests("pretty binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<2>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); doRoundTripTests("raw binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<3>() { - mFormatter = new LLSDXMLFormatter(); - mParser = new LLSDXMLParser(); + setFormatterParser(new LLSDXMLFormatter(), new LLSDXMLParser()); doRoundTripTests("xml serialization"); } template<> template<> void TestLLSDSerializeObject::test<4>() { - mFormatter = new LLSDBinaryFormatter(); - mParser = new LLSDBinaryParser(); + setFormatterParser(new LLSDBinaryFormatter(), new LLSDBinaryParser()); doRoundTripTests("binary serialization"); } + template<> template<> + void TestLLSDSerializeObject::test<5>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_BINARY); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_BINARY)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<6>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_XML); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_XML)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<7>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_NOTATION); + }; + setParser(LLSDSerialize::deserialize); + // In this test, serialize(LLSD_NOTATION) emits a header recognized by + // deserialize(). + doRoundTripTests("serialize(LLSD_NOTATION)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<8>() + { + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDNotationFormatter does not + // emit an llsd/notation header. + doRoundTripTests("LLSDNotationFormatter -> deserialize"); + }; /** * @class TestLLSDParsing @@ -555,7 +623,7 @@ namespace tut public: TestLLSDXMLParsing() {} }; - + typedef tut::test_group TestLLSDXMLParsingGroup; typedef TestLLSDXMLParsingGroup::object TestLLSDXMLParsingObject; TestLLSDXMLParsingGroup gTestLLSDXMLParsingGroup("llsd XML parsing"); @@ -586,8 +654,8 @@ namespace tut LLSD(), LLSDParser::PARSE_FAILURE); } - - + + template<> template<> void TestLLSDXMLParsingObject::test<2>() { @@ -596,7 +664,7 @@ namespace tut v["amy"] = 23; v["bob"] = LLSD(); v["cam"] = 1.23; - + ensureParse( "unknown data type", "" @@ -607,16 +675,16 @@ namespace tut v, v.size() + 1); } - + template<> template<> void TestLLSDXMLParsingObject::test<3>() { // test handling of nested bad data - + LLSD v; v["amy"] = 23; v["cam"] = 1.23; - + ensureParse( "map with html", "" @@ -626,7 +694,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["cam"] = 1.23; @@ -639,7 +707,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["bob"] = LLSD::emptyMap(); @@ -661,7 +729,7 @@ namespace tut v[0] = 23; v[1] = LLSD(); v[2] = 1.23; - + ensureParse( "array value of html", "" @@ -671,7 +739,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v[0] = 23; v[1] = LLSD::emptyMap(); @@ -1225,7 +1293,7 @@ namespace tut vec[0] = 'a'; vec[1] = 'b'; vec[2] = 'c'; vec[3] = '3'; vec[4] = '2'; vec[5] = '1'; LLSD value = vec; - + vec.resize(11); vec[0] = 'b'; // for binary vec[5] = 'a'; vec[6] = 'b'; vec[7] = 'c'; @@ -1909,6 +1977,6 @@ namespace tut "This string\n" "has several\n" "lines."); - + } } -- cgit v1.2.3 From 42e0787446d35bf967ff088148f8ad5f902e929b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 23 Nov 2022 17:15:15 -0500 Subject: SL-18330: LLSDSerialize::deserialize() w/o hdr uses XML or notation Absent a header from LLSDSerialize::serialize(), make deserialize() distinguish between XML or notation by recognizing an initial '<'. --- indra/llcommon/tests/llsdserialize_test.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 0334699c7f..bbbc32214b 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -574,6 +574,34 @@ namespace tut doRoundTripTests("LLSDNotationFormatter -> deserialize"); }; + template<> template<> + void TestLLSDSerializeObject::test<9>() + { + setFormatterParser(new LLSDXMLFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDXMLParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDXMLFormatter does not + // emit an LLSD/XML header. + doRoundTripTests("LLSDXMLFormatter -> deserialize"); + }; + +/*==========================================================================*| + // We do not expect this test to succeed. Without a header, neither + // notation LLSD nor binary LLSD reliably start with a distinct character, + // the way XML LLSD starts with '<'. By convention, we default to notation + // rather than binary. + template<> template<> + void TestLLSDSerializeObject::test<10>() + { + setFormatterParser(new LLSDBinaryFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDBinaryParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDBinaryFormatter does not + // emit an LLSD/Binary header. + doRoundTripTests("LLSDBinaryFormatter -> deserialize"); + }; +|*==========================================================================*/ + /** * @class TestLLSDParsing * @brief Base class for of a parse tester. -- cgit v1.2.3 From b180e4de23cb54ec385e2d999fc5fdd4ea804ba4 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 29 Nov 2022 14:03:02 -0500 Subject: SL-18330: WIP: Send LLLeap to child as binary LLSD; generic parser. Since parsing binary LLSD is faster than parsing notation LLSD, send data from the viewer to the LEAP plugin child process's stdin in binary instead of notation. Similarly, instead of parsing the child process's stdout using specifically a notation parser, use the generic LLSDSerialize::deserialize() LLSD parser. Add more LLSDSerialize Python compatibility tests. --- indra/llcommon/tests/llsdserialize_test.cpp | 286 +++++++++++++++++++--------- 1 file changed, 195 insertions(+), 91 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index bbbc32214b..601f2c580d 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -46,7 +46,6 @@ typedef U32 uint32_t; #include "boost/range.hpp" #include "boost/foreach.hpp" -#include "boost/function.hpp" #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" @@ -62,6 +61,9 @@ using namespace boost::phoenix; #include "stringize.h" #include +typedef std::function FormatterFunction; +typedef std::function ParserFunction; + std::vector string_to_vector(const std::string& str) { return std::vector(str.begin(), str.end()); @@ -277,8 +279,8 @@ namespace tut }; } - std::function mFormatter; - std::function mParser; + FormatterFunction mFormatter; + ParserFunction mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -1790,85 +1792,83 @@ namespace tut ensureBinaryAndXML("map", test); } - struct TestPythonCompatible + // helper for TestPythonCompatible + static std::string import_llsd("import os.path\n" + "import sys\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n"); + + // helper for TestPythonCompatible + template + void python(const std::string& desc, const CONTENT& script, int expect=0) { - TestPythonCompatible(): - // Note the peculiar insertion of __FILE__ into this string. Since - // this script is being written into a platform-dependent temp - // directory, we can't locate indra/lib/python relative to - // Python's __file__. Use __FILE__ instead, navigating relative - // to this C++ source file. Use Python raw-string syntax so - // Windows pathname backslashes won't mislead Python's string - // scanner. - import_llsd("import os.path\n" - "import sys\n" - "from llbase import llsd\n") - {} - ~TestPythonCompatible() {} + auto PYTHON(LLStringUtil::getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - std::string import_llsd; + NamedTempFile scriptfile("py", script); - template - void python(const std::string& desc, const CONTENT& script, int expect=0) +#if LL_WINDOWS + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) { - auto PYTHON(LLStringUtil::getenv("PYTHON")); - ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - - NamedTempFile scriptfile("py", script); + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } -#if LL_WINDOWS - std::string q("\""); - std::string qPYTHON(q + PYTHON + q); - std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); - if (rc == -1) - { - char buffer[256]; - strerror_s(buffer, errno); // C++ can infer the buffer size! :-O - ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); - } - else +#else // LL_DARWIN, LL_LINUX + LLProcess::Params params; + params.executable = PYTHON; + params.args.add(scriptfile.getName()); + LLProcessPtr py(LLProcess::create(params)); + ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py->getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) { - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); } - -#else // LL_DARWIN, LL_LINUX - LLProcess::Params params; - params.executable = PYTHON; - params.args.add(scriptfile.getName()); - LLProcessPtr py(LLProcess::create(params)); - ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); - // Implementing timeout would mean messing with alarm() and - // catching SIGALRM... later maybe... - int status(0); - if (waitpid(py->getProcessID(), &status, 0) == -1) + else if (WIFSIGNALED(status)) { - int waitpid_errno(errno); - ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " - "waitpid() errno " << waitpid_errno), - waitpid_errno, ECHILD); + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); } else { - if (WIFEXITED(status)) - { - int rc(WEXITSTATUS(status)); - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), - rc, expect); - } - else if (WIFSIGNALED(status)) - { - ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), - false); - } - else - { - ensure(STRINGIZE(desc << " script produced impossible status " << status), - false); - } + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); } -#endif } +#endif + } + + struct TestPythonCompatible + { + TestPythonCompatible() {} + ~TestPythonCompatible() {} }; typedef tut::test_group TestPythonCompatibleGroup; @@ -1894,12 +1894,13 @@ namespace tut "print('Running on', sys.platform)\n"); } - // helper for test<3> - static void writeLLSDArray(std::ostream& out, const LLSD& array) + // helper for test<3> - test<7> + static void writeLLSDArray(const FormatterFunction& serialize, + std::ostream& out, const LLSD& array) { BOOST_FOREACH(LLSD item, llsd::inArray(array)) { - LLSDSerialize::toNotation(item, out); + serialize(item, out); // It's important to separate with newlines because Python's llsd // module doesn't support parsing from a file stream, only from a // string, so we have to know how much of the file to read into a @@ -1908,11 +1909,10 @@ namespace tut } } - template<> template<> - void TestPythonCompatibleObject::test<3>() + // helper for test<3> - test<7> + static void toPythonUsing(const std::string& desc, + const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - LLSD cdata(LLSDArray(17)(3.14) ("This string\n" "has several\n" @@ -1941,9 +1941,11 @@ namespace tut // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [serialize, cdata] + (std::ostream& out) + { writeLLSDArray(serialize, out, cdata); }); - python("read C++ notation", + python("read C++ " + desc, placeholders::arg1 << import_llsd << "def parse_each(iterable):\n" @@ -1954,17 +1956,70 @@ namespace tut "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } + template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_XML)"); + toPythonUsing("LLSD_XML", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_XML); }); + } + template<> template<> void TestPythonCompatibleObject::test<4>() { - set_test_name("verify sequence from Python"); + set_test_name("to Python using LLSDSerialize::serialize(LLSD_NOTATION)"); + toPythonUsing("LLSD_NOTATION", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_NOTATION); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<5>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_BINARY)"); + toPythonUsing("LLSD_BINARY", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_BINARY); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<6>() + { + set_test_name("to Python using LLSDSerialize::toXML()"); + toPythonUsing("toXML()", LLSDSerialize::toXML); + } + + template<> template<> + void TestPythonCompatibleObject::test<7>() + { + set_test_name("to Python using LLSDSerialize::toNotation()"); + toPythonUsing("toNotation()", LLSDSerialize::toNotation); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("to Python using LLSDSerialize::toBinary()"); + // We don't expect this to work because, without a header, + // llsd.parse() will assume notation rather than binary. + toPythonUsing("toBinary()", LLSDSerialize::toBinary); + } +|*==========================================================================*/ + // helper for test<8> - test<12> + void fromPythonUsing(const std::string& pyformatter, + const ParserFunction& parse= + [](std::istream& istr, LLSD& data, size_t max_bytes) + { return LLSDSerialize::deserialize(data, istr, max_bytes); }) + { // Create an empty data file. This is just a placeholder for our // script to write into. Create it to establish a unique name that // we know. NamedTempFile file("llsd", ""); - python("write Python notation", + python("Python " + pyformatter, placeholders::arg1 << import_llsd << "DATA = [\n" @@ -1977,9 +2032,9 @@ namespace tut "]\n" // Don't forget raw-string syntax for Windows pathnames. // N.B. Using 'print' implicitly adds newlines. - "with open(r'" << file.getName() << "', 'w') as f:\n" + "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " print(llsd." << pyformatter << "(item), file=f)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; @@ -1989,22 +2044,71 @@ namespace tut // want to ensure that notation-separated-by-newlines works in both // directions -- since in practice, a given file might be read by // either language. - ensure_equals("Failed to read LLSD::Integer from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); + ensure("Failed to read LLSD::Integer from Python", + parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); ensure_equals(item.asInteger(), 17); - ensure_equals("Failed to read LLSD::Real from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); + ensure("Failed to read LLSD::Real from Python", + parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); ensure_approximately_equals("Bad LLSD::Real value from Python", item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure_equals("Failed to read LLSD::String from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); + ensure("Failed to read LLSD::String from Python", + parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); ensure_equals(item.asString(), "This string\n" "has several\n" "lines."); + } + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("from Python XML using LLSDSerialize::deserialize()"); + fromPythonUsing("format_xml"); } + + template<> template<> + void TestPythonCompatibleObject::test<9>() + { + set_test_name("from Python notation using LLSDSerialize::deserialize()"); + fromPythonUsing("format_notation"); + } + + template<> template<> + void TestPythonCompatibleObject::test<10>() + { + set_test_name("from Python binary using LLSDSerialize::deserialize()"); + fromPythonUsing("format_binary"); + } + + template<> template<> + void TestPythonCompatibleObject::test<11>() + { + set_test_name("from Python XML using fromXML()"); + // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors + fromPythonUsing("format_xml", + [](std::istream& istr, LLSD& data, size_t) + { return LLSDSerialize::fromXML(data, istr) > 0; }); + } + + template<> template<> + void TestPythonCompatibleObject::test<12>() + { + set_test_name("from Python notation using fromNotation()"); + fromPythonUsing("format_notation", + [](std::istream& istr, LLSD& data, size_t max_bytes) + { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<13>() + { + set_test_name("from Python binary using fromBinary()"); + // We don't expect this to work because format_binary() emits a + // header, but fromBinary() won't recognize a header. + fromPythonUsing("format_binary", + [](std::istream& istr, LLSD& data, size_t max_bytes) + { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); + } +|*==========================================================================*/ } -- cgit v1.2.3 From 2f557cd7faceec36acace1eee4ee38904ff06130 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 2 Dec 2022 15:17:56 -0500 Subject: SL-18330: Fix new C++ <-> Python LLSD compatibility tests. When sending multiple LEAP packets in the same file (for testing convenience), use a length prefix instead of delimiting with '\n'. Now that we allow a serialization format that includes an LLSD format header (e.g. ""), '\n' is part of the packet content. But in fact, testing binary LLSD means we can't pick any delimiter guaranteed not to appear in the packet content. Using a length prefix also lets us pass a specific max_bytes to the subject C++ LLSD parser. Make llleap_test.cpp use new freestanding Python llsd package when available. Update Python-side LEAP protocol code to work directly with encoded bytes stream, avoiding bytes<->str encoding and decoding, which breaks binary LLSD. Make LLSDSerialize::deserialize() recognize LLSD format header case- insensitively. Python emits and checks for "llsd/binary", while LLSDSerialize emits and checks for "LLSD/Binary". Once any of the headers is recognized, pass corrected max_bytes to the specific parser. Make deserialize() more careful about the no-header case: preserve '\n' in content. Introduce debugging code (disabled) because it's a little tricky to recreate. Revert LLLeap child process stdout parser from LLSDSerialize::deserialize() to the specific LLSDNotationParser(), as at present: the generic parser fails one of LLLeap's integration tests for reasons that remain mysterious. --- indra/llcommon/tests/llsdserialize_test.cpp | 118 +++++++++++++++++++--------- 1 file changed, 83 insertions(+), 35 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 601f2c580d..39b93cf335 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -51,10 +51,11 @@ typedef U32 uint32_t; #include "boost/phoenix/core/argument.hpp" using namespace boost::phoenix; -#include "../llsd.h" -#include "../llsdserialize.h" +#include "llsd.h" +#include "llsdserialize.h" #include "llsdutil.h" -#include "../llformat.h" +#include "llformat.h" +#include "llmemorystream.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" @@ -1898,14 +1899,22 @@ namespace tut static void writeLLSDArray(const FormatterFunction& serialize, std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item : llsd::inArray(array)) { - serialize(item, out); - // It's important to separate with newlines because Python's llsd - // module doesn't support parsing from a file stream, only from a - // string, so we have to know how much of the file to read into a - // string. - out << '\n'; + // It's important to delimit the entries in this file somehow + // because, although Python's llsd.parse() can accept a file + // stream, the XML parser expects EOF after a single outer element + // -- it doesn't just stop. So we must extract a sequence of bytes + // strings from the file. But since one of the serialization + // formats we want to test is binary, we can't pick any single + // byte value as a delimiter! Use a binary integer length prefix + // instead. + std::ostringstream buffer; + serialize(item, buffer); + auto buffstr{ buffer.str() }; + int bufflen{ static_cast(buffstr.length()) }; + out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); + out.write(buffstr.c_str(), buffstr.length()); } } @@ -1932,7 +1941,7 @@ namespace tut " except StopIteration:\n" " pass\n" " else:\n" - " assert False, 'Too many data items'\n"; + " raise AssertionError('Too many data items')\n"; // Create an llsdXXXXXX file containing 'data' serialized to // notation. @@ -1948,10 +1957,23 @@ namespace tut python("read C++ " + desc, placeholders::arg1 << import_llsd << - "def parse_each(iterable):\n" - " for item in iterable:\n" - " yield llsd.parse(item)\n" << - pydata << + "from functools import partial\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " try:\n" + " yield llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } @@ -2008,6 +2030,26 @@ namespace tut } |*==========================================================================*/ + // helper for test<8> - test<12> + bool itemFromStream(std::istream& istr, LLSD& item, const ParserFunction& parse) + { + // reset the output value for debugging clarity + item.clear(); + // We use an int length prefix as a foolproof delimiter even for + // binary serialized streams. + int length{ 0 }; + istr.read(reinterpret_cast(&length), sizeof(length)); +// return parse(istr, item, length); + // Sadly, as of 2022-12-01 it seems we can't really trust our LLSD + // parsers to honor max_bytes: this test works better when we read + // each item into its own distinct LLMemoryStream, instead of passing + // the original istr with a max_bytes constraint. + std::vector buffer(length); + istr.read(reinterpret_cast(buffer.data()), length); + LLMemoryStream stream(buffer.data(), length); + return parse(stream, item, length); + } + // helper for test<8> - test<12> void fromPythonUsing(const std::string& pyformatter, const ParserFunction& parse= @@ -2022,6 +2064,8 @@ namespace tut python("Python " + pyformatter, placeholders::arg1 << import_llsd << + "import struct\n" + "lenformat = struct.Struct('i')\n" "DATA = [\n" " 17,\n" " 3.14,\n" @@ -2034,29 +2078,33 @@ namespace tut // N.B. Using 'print' implicitly adds newlines. "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd." << pyformatter << "(item), file=f)\n"); + " serialized = llsd." << pyformatter << "(item)\n" + " f.write(lenformat.pack(len(serialized)))\n" + " f.write(serialized)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; - // Notice that we're not doing anything special to parse out the - // newlines: LLSDSerialize::fromNotation ignores them. While it would - // seem they're not strictly necessary, going in this direction, we - // want to ensure that notation-separated-by-newlines works in both - // directions -- since in practice, a given file might be read by - // either language. - ensure("Failed to read LLSD::Integer from Python", - parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); - ensure_equals(item.asInteger(), 17); - ensure("Failed to read LLSD::Real from Python", - parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); - ensure_approximately_equals("Bad LLSD::Real value from Python", - item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure("Failed to read LLSD::String from Python", - parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); - ensure_equals(item.asString(), - "This string\n" - "has several\n" - "lines."); + try + { + ensure("Failed to read LLSD::Integer from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asInteger(), 17); + ensure("Failed to read LLSD::Real from Python", + itemFromStream(inf, item, parse)); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure("Failed to read LLSD::String from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } + catch (const tut::failure& err) + { + std::cout << "for " << err.what() << ", item = " << item << std::endl; + throw; + } } template<> template<> -- cgit v1.2.3 From 761d8337fb70cea9cfb326f06a8b1b35438ed96b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 2 Dec 2022 17:14:31 -0500 Subject: SL-18330: Test Python llsd.parse() both from bytes and from stream. --- indra/llcommon/tests/llsdserialize_test.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 39b93cf335..6e5d8a26e1 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1958,6 +1958,7 @@ namespace tut placeholders::arg1 << import_llsd << "from functools import partial\n" + "import io\n" "import struct\n" "lenformat = struct.Struct('i')\n" "def parse_each(inf):\n" @@ -1968,11 +1969,16 @@ namespace tut // into a distinct bytes object and parse that. " data = inf.read(len)\n" " try:\n" - " yield llsd.parse(data)\n" + " frombytes = llsd.parse(data)\n" " except llsd.LLSDParseError as err:\n" " print(f'*** {err}')\n" " print(f'Bad content:\\n{data!r}')\n" " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); -- cgit v1.2.3 From 590e158cf36b6e6fca07f54837db084fe8a163c6 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Dec 2022 16:30:35 -0500 Subject: SL-18330: Adapt LLSDSerialize and tests to llssize max_bytes params. --- indra/llcommon/tests/llsdserialize_test.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 6e5d8a26e1..29e3007aff 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -63,7 +63,7 @@ using namespace boost::phoenix; #include typedef std::function FormatterFunction; -typedef std::function ParserFunction; +typedef std::function ParserFunction; std::vector string_to_vector(const std::string& str) { @@ -263,7 +263,7 @@ namespace tut }; // this lambda must be mutable since otherwise the bound 'parser' // is assumed to point to a const LLSDParser - mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) mutable + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) mutable { // reset() call is needed since test code re-uses parser object parser->reset(); @@ -271,10 +271,10 @@ namespace tut }; } - void setParser(bool (*parser)(LLSD&, std::istream&, size_t)) + void setParser(bool (*parser)(LLSD&, std::istream&, llssize)) { // why does LLSDSerialize::deserialize() reverse the parse() params?? - mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) { return (parser(data, istr, max_bytes) > 0); }; @@ -2059,7 +2059,7 @@ namespace tut // helper for test<8> - test<12> void fromPythonUsing(const std::string& pyformatter, const ParserFunction& parse= - [](std::istream& istr, LLSD& data, size_t max_bytes) + [](std::istream& istr, LLSD& data, llssize max_bytes) { return LLSDSerialize::deserialize(data, istr, max_bytes); }) { // Create an empty data file. This is just a placeholder for our @@ -2140,7 +2140,7 @@ namespace tut set_test_name("from Python XML using fromXML()"); // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors fromPythonUsing("format_xml", - [](std::istream& istr, LLSD& data, size_t) + [](std::istream& istr, LLSD& data, llssize) { return LLSDSerialize::fromXML(data, istr) > 0; }); } @@ -2149,7 +2149,7 @@ namespace tut { set_test_name("from Python notation using fromNotation()"); fromPythonUsing("format_notation", - [](std::istream& istr, LLSD& data, size_t max_bytes) + [](std::istream& istr, LLSD& data, llssize max_bytes) { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); } @@ -2161,7 +2161,7 @@ namespace tut // We don't expect this to work because format_binary() emits a // header, but fromBinary() won't recognize a header. fromPythonUsing("format_binary", - [](std::istream& istr, LLSD& data, size_t max_bytes) + [](std::istream& istr, LLSD& data, llssize max_bytes) { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); } |*==========================================================================*/ -- cgit v1.2.3 From 6cb6385bc71417c1017dc5ccabe53a678e371684 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 1 Mar 2023 16:37:55 -0500 Subject: SL-18330: Tweaks for Visual Studio builds --- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 29e3007aff..618f33cc13 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -276,7 +276,7 @@ namespace tut // why does LLSDSerialize::deserialize() reverse the parse() params?? mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) { - return (parser(data, istr, max_bytes) > 0); + return parser(data, istr, max_bytes); }; } -- cgit v1.2.3 From 971e851157da688b480d6ada188daec532e4b682 Mon Sep 17 00:00:00 2001 From: Andrey Lihatskiy Date: Thu, 4 May 2023 16:44:23 +0300 Subject: SL-19647 OSX buildfix --- indra/llcommon/tests/llsdserialize_test.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'indra/llcommon/tests/llsdserialize_test.cpp') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 85791e2979..acb2953b5b 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1922,8 +1922,6 @@ namespace tut static void toPythonUsing(const std::string& desc, const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - LLSD cdata(llsd::array(17, 3.14, "This string\n" "has several\n" -- cgit v1.2.3