From b180e4de23cb54ec385e2d999fc5fdd4ea804ba4 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 29 Nov 2022 14:03:02 -0500 Subject: SL-18330: WIP: Send LLLeap to child as binary LLSD; generic parser. Since parsing binary LLSD is faster than parsing notation LLSD, send data from the viewer to the LEAP plugin child process's stdin in binary instead of notation. Similarly, instead of parsing the child process's stdout using specifically a notation parser, use the generic LLSDSerialize::deserialize() LLSD parser. Add more LLSDSerialize Python compatibility tests. --- indra/llcommon/llleap.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) (limited to 'indra/llcommon/llleap.cpp') diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index 2704f8b6de..f71eaf92c4 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -204,30 +204,35 @@ public: LLSD packet(LLSDMap("pump", pump)("data", data)); std::ostringstream buffer; - buffer << LLSDNotationStreamer(packet); + // SL-18330: for large data blocks, it's much faster to parse binary + // LLSD than notation LLSD. Use serialize(LLSD_BINARY) rather than + // directly calling LLSDBinaryFormatter because, unlike the latter, + // serialize() prepends the relevant header, needed by a general- + // purpose LLSD parser to distinguish binary from notation. + LLSDSerialize::serialize(packet, buffer, LLSDSerialize::LLSD_BINARY, + LLSDFormatter::OPTIONS_NONE); /*==========================================================================*| // DEBUGGING ONLY: don't copy str() if we can avoid it. std::string strdata(buffer.str()); if (std::size_t(buffer.tellp()) != strdata.length()) { - LL_ERRS("LLLeap") << "tellp() -> " << buffer.tellp() << " != " + LL_ERRS("LLLeap") << "tellp() -> " << static_cast(buffer.tellp()) << " != " << "str().length() -> " << strdata.length() << LL_ENDL; } // DEBUGGING ONLY: reading back is terribly inefficient. std::istringstream readback(strdata); LLSD echo; - LLPointer parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(readback, echo, strdata.length())); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(echo, readback, strdata.length())); + if (! parse_status) { - LL_ERRS("LLLeap") << "LLSDNotationParser() cannot parse output of " - << "LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() cannot parse output of " + << "LLSDSerialize::serialize(LLSD_BINARY)" << LL_ENDL; } if (! llsd_equals(echo, packet)) { - LL_ERRS("LLLeap") << "LLSDNotationParser() produced different LLSD " - << "than passed to LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() returned different LLSD " + << "than passed to LLSDSerialize::serialize()" << LL_ENDL; } |*==========================================================================*/ @@ -312,9 +317,8 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; - LLPointer parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); + if (! parse_status) { bad_protocol("unparseable LLSD data"); } -- cgit v1.2.3 From 2f557cd7faceec36acace1eee4ee38904ff06130 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 2 Dec 2022 15:17:56 -0500 Subject: SL-18330: Fix new C++ <-> Python LLSD compatibility tests. When sending multiple LEAP packets in the same file (for testing convenience), use a length prefix instead of delimiting with '\n'. Now that we allow a serialization format that includes an LLSD format header (e.g. ""), '\n' is part of the packet content. But in fact, testing binary LLSD means we can't pick any delimiter guaranteed not to appear in the packet content. Using a length prefix also lets us pass a specific max_bytes to the subject C++ LLSD parser. Make llleap_test.cpp use new freestanding Python llsd package when available. Update Python-side LEAP protocol code to work directly with encoded bytes stream, avoiding bytes<->str encoding and decoding, which breaks binary LLSD. Make LLSDSerialize::deserialize() recognize LLSD format header case- insensitively. Python emits and checks for "llsd/binary", while LLSDSerialize emits and checks for "LLSD/Binary". Once any of the headers is recognized, pass corrected max_bytes to the specific parser. Make deserialize() more careful about the no-header case: preserve '\n' in content. Introduce debugging code (disabled) because it's a little tricky to recreate. Revert LLLeap child process stdout parser from LLSDSerialize::deserialize() to the specific LLSDNotationParser(), as at present: the generic parser fails one of LLLeap's integration tests for reasons that remain mysterious. --- indra/llcommon/llleap.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'indra/llcommon/llleap.cpp') diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index f71eaf92c4..e8ccc4300d 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -317,8 +317,17 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; +#if 1 + // specifically require notation LLSD from child + LLPointer parser(new LLSDNotationParser()); + S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); + if (parse_status == LLSDParser::PARSE_FAILURE) +#else + // SL-18330: accept any valid LLSD serialization format from child + // Unfortunately this runs into trouble we have not yet debugged. bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); if (! parse_status) +#endif { bad_protocol("unparseable LLSD data"); } -- cgit v1.2.3