From abe62c23d74d5319691a49881719b03cc9b5b090 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 22 Nov 2022 16:21:44 -0500 Subject: SL-18330: Make LLSDSerialize::deserialize() default to notation. LLSDSerialize::serialize() emits a header string, e.g. "" for notation format. Until now, LLSDSerialize::deserialize() has required that header to properly decode the input stream. But none of LLSDBinaryFormatter, LLSDXMLFormatter or LLSDNotationFormatter emit that header themselves. Nor do any of the Python llsd.format_binary(), format_xml() or format_notation() functions. Until now, you could not use LLSD::deserialize() to parse an arbitrary-format LLSD stream serialized by anything but LLSDSerialize::serialize(). Change LLSDSerialize::deserialize() so that if no header is recognized, instead of failing, it attempts to parse as notation. Add tests to exercise this case. The tricky part about this processing is that deserialize() necessarily reads some number of bytes from the input stream first, to try to recognize the header. If it fails to do so, it must prepend the bytes it has already read to the rest of the input stream since they're probably the beginning of the serialized data. To support this use case, introduce cat_streambuf, a std::streambuf subclass that (virtually) concatenates other std::streambuf instances. When read by a std::istream, the sequence of underlying std::streambufs appears to the consumer as a single continuous stream. --- indra/llcommon/llsdserialize.cpp | 87 ++++++++----- indra/llcommon/llstreamtools.cpp | 26 ++++ indra/llcommon/llstreamtools.h | 27 +++- indra/llcommon/tests/llsdserialize_test.cpp | 192 +++++++++++++++++++--------- 4 files changed, 233 insertions(+), 99 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 8b4a0ee6d8..9ff4676b95 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -45,6 +45,7 @@ #endif #include "lldate.h" +#include "llmemorystream.h" #include "llsd.h" #include "llstring.h" #include "lluri.h" @@ -61,6 +62,23 @@ const std::string LLSD_NOTATION_HEADER("llsd/notation"); #define windowBits 15 #define ENABLE_ZLIB_GZIP 32 +// If we published this in llsdserialize.h, we could use it in the +// implementation of LLSDOStreamer's operator<<(). +template +void format_using(const LLSD& data, std::ostream& ostr, + LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY) +{ + LLPointer f{ new Formatter }; + f->format(data, ostr, options); +} + +template +S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1) +{ + LLPointer p{ new Parser }; + return p->parse(istr, data, max_bytes, max_depth); +} + /** * LLSDSerialize */ @@ -83,10 +101,10 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize f = new LLSDXMLFormatter; break; - case LLSD_NOTATION: - str << "\n"; - f = new LLSDNotationFormatter; - break; + case LLSD_NOTATION: + str << "\n"; + f = new LLSDNotationFormatter; + break; default: LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL; @@ -101,10 +119,7 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize // static bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) { - LLPointer p = NULL; char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ - int i; - int inbuf = 0; bool legacy_no_header = false; bool fail_if_not_legacy = false; std::string header; @@ -112,7 +127,10 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) /* * Get the first line before anything. */ - str.get(hdr_buf, MAX_HDR_LEN, '\n'); + // Remember str's original input position: if there's no header, we'll + // want to back up and retry. + str.get(hdr_buf, sizeof(hdr_buf), '\n'); + auto inbuf = str.gcount(); if (str.fail()) { str.clear(); @@ -122,16 +140,18 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ { legacy_no_header = true; - inbuf = (int)str.gcount(); } else { if (fail_if_not_legacy) - goto fail; + { + LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; + return false; + } /* * Remove the newline chars */ - for (i = 0; i < MAX_HDR_LEN; i++) + for (size_t i = 0; i < sizeof(hdr_buf); i++) { if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' || hdr_buf[i] == '\n') @@ -149,50 +169,47 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) { end = header.find_first_of(" ?", start); } - if ((start == std::string::npos) || (end == std::string::npos)) - goto fail; - - header = header.substr(start, end - start); - ws(str); + if (! (start == std::string::npos) || (end == std::string::npos)) + { + header = header.substr(start, end - start); + ws(str); + } } /* * Create the parser as appropriate */ if (legacy_no_header) { // Create a LLSD XML parser, and parse the first chunk read above - LLSDXMLParser* x = new LLSDXMLParser(); - x->parsePart(hdr_buf, inbuf); // Parse the first part that was already read - x->parseLines(str, sd); // Parse the rest of it - delete x; - return true; + LLSDXMLParser x; + x.parsePart(hdr_buf, inbuf); // Parse the first part that was already read + auto parsed = x.parseLines(str, sd); // Parse the rest of it + // Formally we should probably check (parsed != PARSE_FAILURE && + // parsed > 0), but since PARSE_FAILURE is -1, this suffices. + return (parsed > 0); } if (header == LLSD_BINARY_HEADER) { - p = new LLSDBinaryParser; + return (parse_using(str, sd, max_bytes) > 0); } else if (header == LLSD_XML_HEADER) { - p = new LLSDXMLParser; + return (parse_using(str, sd, max_bytes) > 0); } else if (header == LLSD_NOTATION_HEADER) { - p = new LLSDNotationParser; + return (parse_using(str, sd, max_bytes) > 0); } else { - LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL; + LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL; + // Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that + // data to whatever remains in 'str'. + LLMemoryStreamBuf already(reinterpret_cast(hdr_buf), inbuf); + cat_streambuf prebuff(&already, str.rdbuf()); + std::istream prepend(&prebuff); + return (parse_using(prepend, sd, max_bytes) > 0); } - - if (p.notNull()) - { - p->parse(str, sd, max_bytes); - return true; - } - -fail: - LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; - return false; } /** diff --git a/indra/llcommon/llstreamtools.cpp b/indra/llcommon/llstreamtools.cpp index d7a6f47932..1674f6edc2 100644 --- a/indra/llcommon/llstreamtools.cpp +++ b/indra/llcommon/llstreamtools.cpp @@ -523,3 +523,29 @@ std::istream& operator>>(std::istream& str, const char *tocheck) } return str; } + +int cat_streambuf::underflow() +{ + if (gptr() == egptr()) + { + // here because our buffer is empty + std::streamsize size; + // Until we've run out of mInputs, try reading the first of them + // into mBuffer. If that fetches some characters, break the loop. + while (! mInputs.empty() + && ! (size = mInputs.front()->sgetn(mBuffer.data(), mBuffer.size()))) + { + // We tried to read mInputs.front() but got zero characters. + // Discard the first streambuf and try the next one. + mInputs.pop_front(); + } + // Either we ran out of mInputs or we succeeded in reading some + // characters, that is, size != 0. Tell base class what we have. + setg(mBuffer.data(), mBuffer.data(), mBuffer.data() + size); + } + // If we fell out of the above loop with mBuffer still empty, return + // eof(), otherwise return the next character. + return (gptr() == egptr()) + ? std::char_traits::eof() + : std::char_traits::to_int_type(*gptr()); +} diff --git a/indra/llcommon/llstreamtools.h b/indra/llcommon/llstreamtools.h index 1b04bf91d7..997f738840 100644 --- a/indra/llcommon/llstreamtools.h +++ b/indra/llcommon/llstreamtools.h @@ -27,8 +27,10 @@ #ifndef LL_STREAM_TOOLS_H #define LL_STREAM_TOOLS_H +#include #include #include +#include // unless specifed otherwise these all return input_stream.good() @@ -113,6 +115,27 @@ LL_COMMON_API std::streamsize fullread( LL_COMMON_API std::istream& operator>>(std::istream& str, const char *tocheck); -#endif - +/** + * cat_streambuf is a std::streambuf subclass that accepts a variadic number + * of std::streambuf* (e.g. some_istream.rdbuf()) and virtually concatenates + * their contents. + */ +// derived from https://stackoverflow.com/a/49441066/5533635 +class cat_streambuf: public std::streambuf +{ +private: + std::deque mInputs; + std::vector mBuffer; + +public: + // only valid for std::streambuf* arguments + template + cat_streambuf(Inputs... inputs): + mInputs{inputs...}, + mBuffer(1024) + {} + + int underflow(); +}; +#endif diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..0334699c7f 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -60,6 +60,7 @@ using namespace boost::phoenix; #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include std::vector string_to_vector(const std::string& str) { @@ -112,7 +113,7 @@ namespace tut mSD = LLUUID::null; expected = "\n"; xml_test("null uuid", expected); - + mSD = LLUUID("c96f9b1e-f589-4100-9774-d98643ce0bed"); expected = "c96f9b1e-f589-4100-9774-d98643ce0bed\n"; xml_test("uuid", expected); @@ -136,7 +137,7 @@ namespace tut expected = "aGVsbG8=\n"; xml_test("binary", expected); } - + template<> template<> void sd_xml_object::test<2>() { @@ -225,7 +226,7 @@ namespace tut expected = "bazfoobar\n"; xml_test("2 element map", expected); } - + template<> template<> void sd_xml_object::test<6>() { @@ -241,7 +242,7 @@ namespace tut expected = "Nnw2fGFzZGZoYXBweWJveHw2MGU0NGVjNS0zMDVjLTQzYzItOWExOS1iNGI4OWIxYWUyYTZ8NjBlNDRlYzUtMzA1Yy00M2MyLTlhMTktYjRiODliMWFlMmE2fDYwZTQ0ZWM1LTMwNWMtNDNjMi05YTE5LWI0Yjg5YjFhZTJhNnwwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDB8N2ZmZmZmZmZ8N2ZmZmZmZmZ8MHwwfDgyMDAwfDQ1MGZlMzk0LTI5MDQtYzlhZC0yMTRjLWEwN2ViN2ZlZWMyOXwoTm8gRGVzY3JpcHRpb24pfDB8MTB8MA==\n"; xml_test("binary", expected); } - + class TestLLSDSerializeData { public: @@ -250,9 +251,34 @@ namespace tut void doRoundTripTests(const std::string&); void checkRoundTrip(const std::string&, const LLSD& v); - - LLPointer mFormatter; - LLPointer mParser; + + void setFormatterParser(LLPointer formatter, LLPointer parser) + { + mFormatter = [formatter](const LLSD& data, std::ostream& str) + { + formatter->format(data, str); + }; + // this lambda must be mutable since otherwise the bound 'parser' + // is assumed to point to a const LLSDParser + mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) mutable + { + // reset() call is needed since test code re-uses parser object + parser->reset(); + return (parser->parse(istr, data, max_bytes) > 0); + }; + } + + void setParser(bool (*parser)(LLSD&, std::istream&, size_t)) + { + // why does LLSDSerialize::deserialize() reverse the parse() params?? + mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) + { + return (parser(data, istr, max_bytes) > 0); + }; + } + + std::function mFormatter; + std::function mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -265,12 +291,11 @@ namespace tut void TestLLSDSerializeData::checkRoundTrip(const std::string& msg, const LLSD& v) { - std::stringstream stream; - mFormatter->format(v, stream); + std::stringstream stream; + mFormatter(v, stream); //LL_INFOS() << "checkRoundTrip: length " << stream.str().length() << LL_ENDL; LLSD w; - mParser->reset(); // reset() call is needed since test code re-uses mParser - mParser->parse(stream, w, stream.str().size()); + mParser(stream, w, stream.str().size()); try { @@ -299,52 +324,52 @@ namespace tut fillmap(root[key], width, depth - 1); } } - + void TestLLSDSerializeData::doRoundTripTests(const std::string& msg) { LLSD v; checkRoundTrip(msg + " undefined", v); - + v = true; checkRoundTrip(msg + " true bool", v); - + v = false; checkRoundTrip(msg + " false bool", v); - + v = 1; checkRoundTrip(msg + " positive int", v); - + v = 0; checkRoundTrip(msg + " zero int", v); - + v = -1; checkRoundTrip(msg + " negative int", v); - + v = 1234.5f; checkRoundTrip(msg + " positive float", v); - + v = 0.0f; checkRoundTrip(msg + " zero float", v); - + v = -1234.5f; checkRoundTrip(msg + " negative float", v); - + // FIXME: need a NaN test - + v = LLUUID::null; checkRoundTrip(msg + " null uuid", v); - + LLUUID newUUID; newUUID.generate(); v = newUUID; checkRoundTrip(msg + " new uuid", v); - + v = ""; checkRoundTrip(msg + " empty string", v); - + v = "some string"; checkRoundTrip(msg + " non-empty string", v); - + v = "Second Life is a 3-D virtual world entirely built and owned by its residents. " "Since opening to the public in 2003, it has grown explosively and today is " @@ -372,7 +397,7 @@ namespace tut for (U32 block = 0x000000; block <= 0x10ffff; block += block_size) { std::ostringstream out; - + for (U32 c = block; c < block + block_size; ++c) { if (c <= 0x000001f @@ -386,7 +411,7 @@ namespace tut if (0x00fdd0 <= c && c <= 0x00fdef) { continue; } if ((c & 0x00fffe) == 0x00fffe) { continue; } // see Unicode standard, section 15.8 - + if (c <= 0x00007f) { out << (char)(c & 0x7f); @@ -410,55 +435,55 @@ namespace tut out << (char)(0x80 | ((c >> 0) & 0x3f)); } } - + v = out.str(); std::ostringstream blockmsg; blockmsg << msg << " unicode string block 0x" << std::hex << block; checkRoundTrip(blockmsg.str(), v); } - + LLDate epoch; v = epoch; checkRoundTrip(msg + " epoch date", v); - + LLDate aDay("2002-12-07T05:07:15.00Z"); v = aDay; checkRoundTrip(msg + " date", v); - + LLURI path("http://slurl.com/secondlife/Ambleside/57/104/26/"); v = path; checkRoundTrip(msg + " url", v); - + const char source[] = "it must be a blue moon again"; std::vector data; // note, includes terminating '\0' copy(&source[0], &source[sizeof(source)], back_inserter(data)); - + v = data; checkRoundTrip(msg + " binary", v); - + v = LLSD::emptyMap(); checkRoundTrip(msg + " empty map", v); - + v = LLSD::emptyMap(); v["name"] = "luke"; //v.insert("name", "luke"); v["age"] = 3; //v.insert("age", 3); checkRoundTrip(msg + " map", v); - + v.clear(); v["a"]["1"] = true; v["b"]["0"] = false; checkRoundTrip(msg + " nested maps", v); - + v = LLSD::emptyArray(); checkRoundTrip(msg + " empty array", v); - + v = LLSD::emptyArray(); v.append("ali"); v.append(28); checkRoundTrip(msg + " array", v); - + v.clear(); v[0][0] = true; v[1][0] = false; @@ -468,7 +493,7 @@ namespace tut fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } - + typedef tut::test_group TestLLSDSerializeGroup; typedef TestLLSDSerializeGroup::object TestLLSDSerializeObject; TestLLSDSerializeGroup gTestLLSDSerializeGroup("llsd serialization"); @@ -476,35 +501,78 @@ namespace tut template<> template<> void TestLLSDSerializeObject::test<1>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY), + new LLSDNotationParser()); doRoundTripTests("pretty binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<2>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); doRoundTripTests("raw binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<3>() { - mFormatter = new LLSDXMLFormatter(); - mParser = new LLSDXMLParser(); + setFormatterParser(new LLSDXMLFormatter(), new LLSDXMLParser()); doRoundTripTests("xml serialization"); } template<> template<> void TestLLSDSerializeObject::test<4>() { - mFormatter = new LLSDBinaryFormatter(); - mParser = new LLSDBinaryParser(); + setFormatterParser(new LLSDBinaryFormatter(), new LLSDBinaryParser()); doRoundTripTests("binary serialization"); } + template<> template<> + void TestLLSDSerializeObject::test<5>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_BINARY); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_BINARY)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<6>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_XML); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_XML)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<7>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_NOTATION); + }; + setParser(LLSDSerialize::deserialize); + // In this test, serialize(LLSD_NOTATION) emits a header recognized by + // deserialize(). + doRoundTripTests("serialize(LLSD_NOTATION)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<8>() + { + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDNotationFormatter does not + // emit an llsd/notation header. + doRoundTripTests("LLSDNotationFormatter -> deserialize"); + }; /** * @class TestLLSDParsing @@ -555,7 +623,7 @@ namespace tut public: TestLLSDXMLParsing() {} }; - + typedef tut::test_group TestLLSDXMLParsingGroup; typedef TestLLSDXMLParsingGroup::object TestLLSDXMLParsingObject; TestLLSDXMLParsingGroup gTestLLSDXMLParsingGroup("llsd XML parsing"); @@ -586,8 +654,8 @@ namespace tut LLSD(), LLSDParser::PARSE_FAILURE); } - - + + template<> template<> void TestLLSDXMLParsingObject::test<2>() { @@ -596,7 +664,7 @@ namespace tut v["amy"] = 23; v["bob"] = LLSD(); v["cam"] = 1.23; - + ensureParse( "unknown data type", "" @@ -607,16 +675,16 @@ namespace tut v, v.size() + 1); } - + template<> template<> void TestLLSDXMLParsingObject::test<3>() { // test handling of nested bad data - + LLSD v; v["amy"] = 23; v["cam"] = 1.23; - + ensureParse( "map with html", "" @@ -626,7 +694,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["cam"] = 1.23; @@ -639,7 +707,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["bob"] = LLSD::emptyMap(); @@ -661,7 +729,7 @@ namespace tut v[0] = 23; v[1] = LLSD(); v[2] = 1.23; - + ensureParse( "array value of html", "" @@ -671,7 +739,7 @@ namespace tut "", v, v.size() + 1); - + v.clear(); v[0] = 23; v[1] = LLSD::emptyMap(); @@ -1225,7 +1293,7 @@ namespace tut vec[0] = 'a'; vec[1] = 'b'; vec[2] = 'c'; vec[3] = '3'; vec[4] = '2'; vec[5] = '1'; LLSD value = vec; - + vec.resize(11); vec[0] = 'b'; // for binary vec[5] = 'a'; vec[6] = 'b'; vec[7] = 'c'; @@ -1909,6 +1977,6 @@ namespace tut "This string\n" "has several\n" "lines."); - + } } -- cgit v1.2.3 From 42e0787446d35bf967ff088148f8ad5f902e929b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 23 Nov 2022 17:15:15 -0500 Subject: SL-18330: LLSDSerialize::deserialize() w/o hdr uses XML or notation Absent a header from LLSDSerialize::serialize(), make deserialize() distinguish between XML or notation by recognizing an initial '<'. --- indra/llcommon/llsdserialize.cpp | 87 ++++++++++++++--------------- indra/llcommon/tests/llsdserialize_test.cpp | 28 ++++++++++ 2 files changed, 70 insertions(+), 45 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 9ff4676b95..dc8f8f5737 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -120,9 +120,7 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) { char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ - bool legacy_no_header = false; bool fail_if_not_legacy = false; - std::string header; /* * Get the first line before anything. @@ -131,6 +129,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) // want to back up and retry. str.get(hdr_buf, sizeof(hdr_buf), '\n'); auto inbuf = str.gcount(); + std::string header{ hdr_buf, inbuf }; if (str.fail()) { str.clear(); @@ -138,38 +137,36 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) } if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ + { // Create a LLSD XML parser, and parse the first chunk read above. + LLSDXMLParser x; + x.parsePart(hdr_buf, inbuf); // Parse the first part that was already read + auto parsed = x.parse(str, sd, max_bytes - inbuf); // Parse the rest of it + // Formally we should probably check (parsed != PARSE_FAILURE && + // parsed > 0), but since PARSE_FAILURE is -1, this suffices. + return (parsed > 0); + } + + if (fail_if_not_legacy) { - legacy_no_header = true; + LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; + return false; } - else + + /* + * Remove the newline chars + */ + auto lastchar = header.find_last_not_of("\r\n"); + if (lastchar != std::string::npos) { - if (fail_if_not_legacy) - { - LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; - return false; - } - /* - * Remove the newline chars - */ - for (size_t i = 0; i < sizeof(hdr_buf); i++) - { - if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' || - hdr_buf[i] == '\n') - { - hdr_buf[i] = 0; - break; - } - } - header = hdr_buf; + header.erase(lastchar+1); + } - std::string::size_type start = std::string::npos; - std::string::size_type end = std::string::npos; - start = header.find_first_not_of(" header syntax + auto start = header.find_first_not_of(" 0), but since PARSE_FAILURE is -1, this suffices. - return (parsed > 0); - } - if (header == LLSD_BINARY_HEADER) { return (parse_using(str, sd, max_bytes) > 0); @@ -200,15 +187,27 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) { return (parse_using(str, sd, max_bytes) > 0); } - else + else // no header we recognize { - LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL; + LLPointer p; + if (inbuf && hdr_buf[0] == '<') + { + // looks like XML + LL_DEBUGS() << "deserialize request with no header, assuming XML" << LL_ENDL; + p = new LLSDXMLParser; + } + else + { + // assume notation + LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL; + p = new LLSDNotationParser; + } // Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that // data to whatever remains in 'str'. LLMemoryStreamBuf already(reinterpret_cast(hdr_buf), inbuf); cat_streambuf prebuff(&already, str.rdbuf()); std::istream prepend(&prebuff); - return (parse_using(prepend, sd, max_bytes) > 0); + return (p->parse(prepend, sd, max_bytes) > 0); } } @@ -2411,5 +2410,3 @@ U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 return result; } - - diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 0334699c7f..bbbc32214b 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -574,6 +574,34 @@ namespace tut doRoundTripTests("LLSDNotationFormatter -> deserialize"); }; + template<> template<> + void TestLLSDSerializeObject::test<9>() + { + setFormatterParser(new LLSDXMLFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDXMLParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDXMLFormatter does not + // emit an LLSD/XML header. + doRoundTripTests("LLSDXMLFormatter -> deserialize"); + }; + +/*==========================================================================*| + // We do not expect this test to succeed. Without a header, neither + // notation LLSD nor binary LLSD reliably start with a distinct character, + // the way XML LLSD starts with '<'. By convention, we default to notation + // rather than binary. + template<> template<> + void TestLLSDSerializeObject::test<10>() + { + setFormatterParser(new LLSDBinaryFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDBinaryParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDBinaryFormatter does not + // emit an LLSD/Binary header. + doRoundTripTests("LLSDBinaryFormatter -> deserialize"); + }; +|*==========================================================================*/ + /** * @class TestLLSDParsing * @brief Base class for of a parse tester. -- cgit v1.2.3 From b180e4de23cb54ec385e2d999fc5fdd4ea804ba4 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 29 Nov 2022 14:03:02 -0500 Subject: SL-18330: WIP: Send LLLeap to child as binary LLSD; generic parser. Since parsing binary LLSD is faster than parsing notation LLSD, send data from the viewer to the LEAP plugin child process's stdin in binary instead of notation. Similarly, instead of parsing the child process's stdout using specifically a notation parser, use the generic LLSDSerialize::deserialize() LLSD parser. Add more LLSDSerialize Python compatibility tests. --- indra/llcommon/llleap.cpp | 28 +-- indra/llcommon/tests/llsdserialize_test.cpp | 286 +++++++++++++++++++--------- 2 files changed, 211 insertions(+), 103 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index 2704f8b6de..f71eaf92c4 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -204,30 +204,35 @@ public: LLSD packet(LLSDMap("pump", pump)("data", data)); std::ostringstream buffer; - buffer << LLSDNotationStreamer(packet); + // SL-18330: for large data blocks, it's much faster to parse binary + // LLSD than notation LLSD. Use serialize(LLSD_BINARY) rather than + // directly calling LLSDBinaryFormatter because, unlike the latter, + // serialize() prepends the relevant header, needed by a general- + // purpose LLSD parser to distinguish binary from notation. + LLSDSerialize::serialize(packet, buffer, LLSDSerialize::LLSD_BINARY, + LLSDFormatter::OPTIONS_NONE); /*==========================================================================*| // DEBUGGING ONLY: don't copy str() if we can avoid it. std::string strdata(buffer.str()); if (std::size_t(buffer.tellp()) != strdata.length()) { - LL_ERRS("LLLeap") << "tellp() -> " << buffer.tellp() << " != " + LL_ERRS("LLLeap") << "tellp() -> " << static_cast(buffer.tellp()) << " != " << "str().length() -> " << strdata.length() << LL_ENDL; } // DEBUGGING ONLY: reading back is terribly inefficient. std::istringstream readback(strdata); LLSD echo; - LLPointer parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(readback, echo, strdata.length())); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(echo, readback, strdata.length())); + if (! parse_status) { - LL_ERRS("LLLeap") << "LLSDNotationParser() cannot parse output of " - << "LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() cannot parse output of " + << "LLSDSerialize::serialize(LLSD_BINARY)" << LL_ENDL; } if (! llsd_equals(echo, packet)) { - LL_ERRS("LLLeap") << "LLSDNotationParser() produced different LLSD " - << "than passed to LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() returned different LLSD " + << "than passed to LLSDSerialize::serialize()" << LL_ENDL; } |*==========================================================================*/ @@ -312,9 +317,8 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; - LLPointer parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); + if (! parse_status) { bad_protocol("unparseable LLSD data"); } diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index bbbc32214b..601f2c580d 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -46,7 +46,6 @@ typedef U32 uint32_t; #include "boost/range.hpp" #include "boost/foreach.hpp" -#include "boost/function.hpp" #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" @@ -62,6 +61,9 @@ using namespace boost::phoenix; #include "stringize.h" #include +typedef std::function FormatterFunction; +typedef std::function ParserFunction; + std::vector string_to_vector(const std::string& str) { return std::vector(str.begin(), str.end()); @@ -277,8 +279,8 @@ namespace tut }; } - std::function mFormatter; - std::function mParser; + FormatterFunction mFormatter; + ParserFunction mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -1790,85 +1792,83 @@ namespace tut ensureBinaryAndXML("map", test); } - struct TestPythonCompatible + // helper for TestPythonCompatible + static std::string import_llsd("import os.path\n" + "import sys\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n"); + + // helper for TestPythonCompatible + template + void python(const std::string& desc, const CONTENT& script, int expect=0) { - TestPythonCompatible(): - // Note the peculiar insertion of __FILE__ into this string. Since - // this script is being written into a platform-dependent temp - // directory, we can't locate indra/lib/python relative to - // Python's __file__. Use __FILE__ instead, navigating relative - // to this C++ source file. Use Python raw-string syntax so - // Windows pathname backslashes won't mislead Python's string - // scanner. - import_llsd("import os.path\n" - "import sys\n" - "from llbase import llsd\n") - {} - ~TestPythonCompatible() {} + auto PYTHON(LLStringUtil::getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - std::string import_llsd; + NamedTempFile scriptfile("py", script); - template - void python(const std::string& desc, const CONTENT& script, int expect=0) +#if LL_WINDOWS + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) { - auto PYTHON(LLStringUtil::getenv("PYTHON")); - ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - - NamedTempFile scriptfile("py", script); + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } -#if LL_WINDOWS - std::string q("\""); - std::string qPYTHON(q + PYTHON + q); - std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); - if (rc == -1) - { - char buffer[256]; - strerror_s(buffer, errno); // C++ can infer the buffer size! :-O - ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); - } - else +#else // LL_DARWIN, LL_LINUX + LLProcess::Params params; + params.executable = PYTHON; + params.args.add(scriptfile.getName()); + LLProcessPtr py(LLProcess::create(params)); + ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py->getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) { - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); } - -#else // LL_DARWIN, LL_LINUX - LLProcess::Params params; - params.executable = PYTHON; - params.args.add(scriptfile.getName()); - LLProcessPtr py(LLProcess::create(params)); - ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); - // Implementing timeout would mean messing with alarm() and - // catching SIGALRM... later maybe... - int status(0); - if (waitpid(py->getProcessID(), &status, 0) == -1) + else if (WIFSIGNALED(status)) { - int waitpid_errno(errno); - ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " - "waitpid() errno " << waitpid_errno), - waitpid_errno, ECHILD); + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); } else { - if (WIFEXITED(status)) - { - int rc(WEXITSTATUS(status)); - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), - rc, expect); - } - else if (WIFSIGNALED(status)) - { - ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), - false); - } - else - { - ensure(STRINGIZE(desc << " script produced impossible status " << status), - false); - } + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); } -#endif } +#endif + } + + struct TestPythonCompatible + { + TestPythonCompatible() {} + ~TestPythonCompatible() {} }; typedef tut::test_group TestPythonCompatibleGroup; @@ -1894,12 +1894,13 @@ namespace tut "print('Running on', sys.platform)\n"); } - // helper for test<3> - static void writeLLSDArray(std::ostream& out, const LLSD& array) + // helper for test<3> - test<7> + static void writeLLSDArray(const FormatterFunction& serialize, + std::ostream& out, const LLSD& array) { BOOST_FOREACH(LLSD item, llsd::inArray(array)) { - LLSDSerialize::toNotation(item, out); + serialize(item, out); // It's important to separate with newlines because Python's llsd // module doesn't support parsing from a file stream, only from a // string, so we have to know how much of the file to read into a @@ -1908,11 +1909,10 @@ namespace tut } } - template<> template<> - void TestPythonCompatibleObject::test<3>() + // helper for test<3> - test<7> + static void toPythonUsing(const std::string& desc, + const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - LLSD cdata(LLSDArray(17)(3.14) ("This string\n" "has several\n" @@ -1941,9 +1941,11 @@ namespace tut // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [serialize, cdata] + (std::ostream& out) + { writeLLSDArray(serialize, out, cdata); }); - python("read C++ notation", + python("read C++ " + desc, placeholders::arg1 << import_llsd << "def parse_each(iterable):\n" @@ -1954,17 +1956,70 @@ namespace tut "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } + template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_XML)"); + toPythonUsing("LLSD_XML", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_XML); }); + } + template<> template<> void TestPythonCompatibleObject::test<4>() { - set_test_name("verify sequence from Python"); + set_test_name("to Python using LLSDSerialize::serialize(LLSD_NOTATION)"); + toPythonUsing("LLSD_NOTATION", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_NOTATION); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<5>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_BINARY)"); + toPythonUsing("LLSD_BINARY", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_BINARY); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<6>() + { + set_test_name("to Python using LLSDSerialize::toXML()"); + toPythonUsing("toXML()", LLSDSerialize::toXML); + } + + template<> template<> + void TestPythonCompatibleObject::test<7>() + { + set_test_name("to Python using LLSDSerialize::toNotation()"); + toPythonUsing("toNotation()", LLSDSerialize::toNotation); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("to Python using LLSDSerialize::toBinary()"); + // We don't expect this to work because, without a header, + // llsd.parse() will assume notation rather than binary. + toPythonUsing("toBinary()", LLSDSerialize::toBinary); + } +|*==========================================================================*/ + // helper for test<8> - test<12> + void fromPythonUsing(const std::string& pyformatter, + const ParserFunction& parse= + [](std::istream& istr, LLSD& data, size_t max_bytes) + { return LLSDSerialize::deserialize(data, istr, max_bytes); }) + { // Create an empty data file. This is just a placeholder for our // script to write into. Create it to establish a unique name that // we know. NamedTempFile file("llsd", ""); - python("write Python notation", + python("Python " + pyformatter, placeholders::arg1 << import_llsd << "DATA = [\n" @@ -1977,9 +2032,9 @@ namespace tut "]\n" // Don't forget raw-string syntax for Windows pathnames. // N.B. Using 'print' implicitly adds newlines. - "with open(r'" << file.getName() << "', 'w') as f:\n" + "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " print(llsd." << pyformatter << "(item), file=f)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; @@ -1989,22 +2044,71 @@ namespace tut // want to ensure that notation-separated-by-newlines works in both // directions -- since in practice, a given file might be read by // either language. - ensure_equals("Failed to read LLSD::Integer from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); + ensure("Failed to read LLSD::Integer from Python", + parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); ensure_equals(item.asInteger(), 17); - ensure_equals("Failed to read LLSD::Real from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); + ensure("Failed to read LLSD::Real from Python", + parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); ensure_approximately_equals("Bad LLSD::Real value from Python", item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure_equals("Failed to read LLSD::String from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); + ensure("Failed to read LLSD::String from Python", + parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); ensure_equals(item.asString(), "This string\n" "has several\n" "lines."); + } + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("from Python XML using LLSDSerialize::deserialize()"); + fromPythonUsing("format_xml"); } + + template<> template<> + void TestPythonCompatibleObject::test<9>() + { + set_test_name("from Python notation using LLSDSerialize::deserialize()"); + fromPythonUsing("format_notation"); + } + + template<> template<> + void TestPythonCompatibleObject::test<10>() + { + set_test_name("from Python binary using LLSDSerialize::deserialize()"); + fromPythonUsing("format_binary"); + } + + template<> template<> + void TestPythonCompatibleObject::test<11>() + { + set_test_name("from Python XML using fromXML()"); + // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors + fromPythonUsing("format_xml", + [](std::istream& istr, LLSD& data, size_t) + { return LLSDSerialize::fromXML(data, istr) > 0; }); + } + + template<> template<> + void TestPythonCompatibleObject::test<12>() + { + set_test_name("from Python notation using fromNotation()"); + fromPythonUsing("format_notation", + [](std::istream& istr, LLSD& data, size_t max_bytes) + { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<13>() + { + set_test_name("from Python binary using fromBinary()"); + // We don't expect this to work because format_binary() emits a + // header, but fromBinary() won't recognize a header. + fromPythonUsing("format_binary", + [](std::istream& istr, LLSD& data, size_t max_bytes) + { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); + } +|*==========================================================================*/ } -- cgit v1.2.3 From 2f557cd7faceec36acace1eee4ee38904ff06130 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 2 Dec 2022 15:17:56 -0500 Subject: SL-18330: Fix new C++ <-> Python LLSD compatibility tests. When sending multiple LEAP packets in the same file (for testing convenience), use a length prefix instead of delimiting with '\n'. Now that we allow a serialization format that includes an LLSD format header (e.g. ""), '\n' is part of the packet content. But in fact, testing binary LLSD means we can't pick any delimiter guaranteed not to appear in the packet content. Using a length prefix also lets us pass a specific max_bytes to the subject C++ LLSD parser. Make llleap_test.cpp use new freestanding Python llsd package when available. Update Python-side LEAP protocol code to work directly with encoded bytes stream, avoiding bytes<->str encoding and decoding, which breaks binary LLSD. Make LLSDSerialize::deserialize() recognize LLSD format header case- insensitively. Python emits and checks for "llsd/binary", while LLSDSerialize emits and checks for "LLSD/Binary". Once any of the headers is recognized, pass corrected max_bytes to the specific parser. Make deserialize() more careful about the no-header case: preserve '\n' in content. Introduce debugging code (disabled) because it's a little tricky to recreate. Revert LLLeap child process stdout parser from LLSDSerialize::deserialize() to the specific LLSDNotationParser(), as at present: the generic parser fails one of LLLeap's integration tests for reasons that remain mysterious. --- indra/llcommon/llleap.cpp | 9 +++ indra/llcommon/llsdserialize.cpp | 49 +++++++++--- indra/llcommon/tests/llleap_test.cpp | 32 ++++---- indra/llcommon/tests/llsdserialize_test.cpp | 118 +++++++++++++++++++--------- 4 files changed, 147 insertions(+), 61 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index f71eaf92c4..e8ccc4300d 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -317,8 +317,17 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; +#if 1 + // specifically require notation LLSD from child + LLPointer parser(new LLSDNotationParser()); + S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); + if (parse_status == LLSDParser::PARSE_FAILURE) +#else + // SL-18330: accept any valid LLSD serialization format from child + // Unfortunately this runs into trouble we have not yet debugged. bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); if (! parse_status) +#endif { bad_protocol("unparseable LLSD data"); } diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index dc8f8f5737..97bf51eeaa 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -123,12 +123,26 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) bool fail_if_not_legacy = false; /* - * Get the first line before anything. + * Get the first line before anything. Don't read more than max_bytes: + * this get() overload reads no more than (count-1) bytes into the + * specified buffer. In the usual case when max_bytes exceeds + * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. */ - // Remember str's original input position: if there's no header, we'll - // want to back up and retry. - str.get(hdr_buf, sizeof(hdr_buf), '\n'); + str.get(hdr_buf, std::min(max_bytes+1, sizeof(hdr_buf)-1), '\n'); auto inbuf = str.gcount(); + // https://en.cppreference.com/w/cpp/io/basic_istream/get + // When the get() above sees the specified delimiter '\n', it stops there + // without pulling it from the stream. If it turns out that the stream + // does NOT contain a header, and the content includes meaningful '\n', + // it's important to pull that into hdr_buf too. + if (inbuf < max_bytes && str.get(hdr_buf[inbuf])) + { + // got the delimiting '\n' + ++inbuf; + // None of the following requires that hdr_buf contain a final '\0' + // byte. We could store one if needed, since even the incremented + // inbuf won't exceed sizeof(hdr_buf)-1, but there's no need. + } std::string header{ hdr_buf, inbuf }; if (str.fail()) { @@ -175,17 +189,17 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) /* * Create the parser as appropriate */ - if (header == LLSD_BINARY_HEADER) + if (0 == LLStringUtil::compareInsensitive(header, LLSD_BINARY_HEADER)) { - return (parse_using(str, sd, max_bytes) > 0); + return (parse_using(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_XML_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_XML_HEADER)) { - return (parse_using(str, sd, max_bytes) > 0); + return (parse_using(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_NOTATION_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_NOTATION_HEADER)) { - return (parse_using(str, sd, max_bytes) > 0); + return (parse_using(str, sd, max_bytes-inbuf) > 0); } else // no header we recognize { @@ -207,7 +221,22 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) LLMemoryStreamBuf already(reinterpret_cast(hdr_buf), inbuf); cat_streambuf prebuff(&already, str.rdbuf()); std::istream prepend(&prebuff); +#if 1 return (p->parse(prepend, sd, max_bytes) > 0); +#else + // debugging the reconstituted 'prepend' stream + // allocate a buffer that we hope is big enough for the whole thing + std::vector wholemsg((max_bytes == size_t(SIZE_UNLIMITED))? 1024 : max_bytes); + prepend.read(wholemsg.data(), std::min(max_bytes, wholemsg.size())); + LLMemoryStream replay(reinterpret_cast(wholemsg.data()), prepend.gcount()); + auto success{ p->parse(replay, sd, prepend.gcount()) > 0 }; + { + LL_DEBUGS() << (success? "parsed: $$" : "failed: '") + << std::string(wholemsg.data(), llmin(prepend.gcount(), 100)) << "$$" + << LL_ENDL; + } + return success; +#endif } } diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 9754353ab0..6c799b7993 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -110,12 +110,12 @@ namespace tut "import os\n" "import sys\n" "\n" - // Don't forget that this Python script is written to some - // temp directory somewhere! Its __file__ is useless in - // finding indra/lib/python. Use our __FILE__, with - // raw-string syntax to deal with Windows pathnames. - "mydir = os.path.dirname(r'" << __FILE__ << "')\n" - "from llbase import llsd\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" @@ -126,26 +126,26 @@ namespace tut " pass\n" "\n" "def get():\n" - " hdr = ''\n" - " while ':' not in hdr and len(hdr) < 20:\n" - " hdr += sys.stdin.read(1)\n" + " hdr = []\n" + " while b':' not in hdr and len(hdr) < 20:\n" + " hdr.append(sys.stdin.buffer.read(1))\n" " if not hdr:\n" " sys.exit(0)\n" - " if not hdr.endswith(':'):\n" + " if not hdr[-1] == b':':\n" " raise ProtocolError('Expected len:data, got %r' % hdr, hdr)\n" " try:\n" - " length = int(hdr[:-1])\n" + " length = int(b''.join(hdr[:-1]))\n" " except ValueError:\n" " raise ProtocolError('Non-numeric len %r' % hdr[:-1], hdr[:-1])\n" " parts = []\n" " received = 0\n" " while received < length:\n" - " parts.append(sys.stdin.read(length - received))\n" + " parts.append(sys.stdin.buffer.read(length - received))\n" " received += len(parts[-1])\n" - " data = ''.join(parts)\n" + " data = b''.join(parts)\n" " assert len(data) == length\n" " try:\n" - " return llsd.parse(data.encode())\n" + " return llsd.parse(data)\n" // Seems the old indra.base.llsd module didn't properly // convert IndexError (from running off end of string) to // LLSDParseError. @@ -185,11 +185,11 @@ namespace tut " return _reply\n" "\n" "def put(req):\n" - " sys.stdout.write(':'.join((str(len(req)), req)))\n" + " sys.stdout.buffer.write(b'%d:%b' % (len(req), req))\n" " sys.stdout.flush()\n" "\n" "def send(pump, data):\n" - " put(llsd.format_notation(dict(pump=pump, data=data)).decode())\n" + " put(llsd.format_notation(dict(pump=pump, data=data)))\n" "\n" "def request(pump, data):\n" " # we expect 'data' is a dict\n" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 601f2c580d..39b93cf335 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -51,10 +51,11 @@ typedef U32 uint32_t; #include "boost/phoenix/core/argument.hpp" using namespace boost::phoenix; -#include "../llsd.h" -#include "../llsdserialize.h" +#include "llsd.h" +#include "llsdserialize.h" #include "llsdutil.h" -#include "../llformat.h" +#include "llformat.h" +#include "llmemorystream.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" @@ -1898,14 +1899,22 @@ namespace tut static void writeLLSDArray(const FormatterFunction& serialize, std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item : llsd::inArray(array)) { - serialize(item, out); - // It's important to separate with newlines because Python's llsd - // module doesn't support parsing from a file stream, only from a - // string, so we have to know how much of the file to read into a - // string. - out << '\n'; + // It's important to delimit the entries in this file somehow + // because, although Python's llsd.parse() can accept a file + // stream, the XML parser expects EOF after a single outer element + // -- it doesn't just stop. So we must extract a sequence of bytes + // strings from the file. But since one of the serialization + // formats we want to test is binary, we can't pick any single + // byte value as a delimiter! Use a binary integer length prefix + // instead. + std::ostringstream buffer; + serialize(item, buffer); + auto buffstr{ buffer.str() }; + int bufflen{ static_cast(buffstr.length()) }; + out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); + out.write(buffstr.c_str(), buffstr.length()); } } @@ -1932,7 +1941,7 @@ namespace tut " except StopIteration:\n" " pass\n" " else:\n" - " assert False, 'Too many data items'\n"; + " raise AssertionError('Too many data items')\n"; // Create an llsdXXXXXX file containing 'data' serialized to // notation. @@ -1948,10 +1957,23 @@ namespace tut python("read C++ " + desc, placeholders::arg1 << import_llsd << - "def parse_each(iterable):\n" - " for item in iterable:\n" - " yield llsd.parse(item)\n" << - pydata << + "from functools import partial\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " try:\n" + " yield llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } @@ -2008,6 +2030,26 @@ namespace tut } |*==========================================================================*/ + // helper for test<8> - test<12> + bool itemFromStream(std::istream& istr, LLSD& item, const ParserFunction& parse) + { + // reset the output value for debugging clarity + item.clear(); + // We use an int length prefix as a foolproof delimiter even for + // binary serialized streams. + int length{ 0 }; + istr.read(reinterpret_cast(&length), sizeof(length)); +// return parse(istr, item, length); + // Sadly, as of 2022-12-01 it seems we can't really trust our LLSD + // parsers to honor max_bytes: this test works better when we read + // each item into its own distinct LLMemoryStream, instead of passing + // the original istr with a max_bytes constraint. + std::vector buffer(length); + istr.read(reinterpret_cast(buffer.data()), length); + LLMemoryStream stream(buffer.data(), length); + return parse(stream, item, length); + } + // helper for test<8> - test<12> void fromPythonUsing(const std::string& pyformatter, const ParserFunction& parse= @@ -2022,6 +2064,8 @@ namespace tut python("Python " + pyformatter, placeholders::arg1 << import_llsd << + "import struct\n" + "lenformat = struct.Struct('i')\n" "DATA = [\n" " 17,\n" " 3.14,\n" @@ -2034,29 +2078,33 @@ namespace tut // N.B. Using 'print' implicitly adds newlines. "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd." << pyformatter << "(item), file=f)\n"); + " serialized = llsd." << pyformatter << "(item)\n" + " f.write(lenformat.pack(len(serialized)))\n" + " f.write(serialized)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; - // Notice that we're not doing anything special to parse out the - // newlines: LLSDSerialize::fromNotation ignores them. While it would - // seem they're not strictly necessary, going in this direction, we - // want to ensure that notation-separated-by-newlines works in both - // directions -- since in practice, a given file might be read by - // either language. - ensure("Failed to read LLSD::Integer from Python", - parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); - ensure_equals(item.asInteger(), 17); - ensure("Failed to read LLSD::Real from Python", - parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); - ensure_approximately_equals("Bad LLSD::Real value from Python", - item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure("Failed to read LLSD::String from Python", - parse(inf, item, LLSDSerialize::SIZE_UNLIMITED)); - ensure_equals(item.asString(), - "This string\n" - "has several\n" - "lines."); + try + { + ensure("Failed to read LLSD::Integer from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asInteger(), 17); + ensure("Failed to read LLSD::Real from Python", + itemFromStream(inf, item, parse)); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure("Failed to read LLSD::String from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } + catch (const tut::failure& err) + { + std::cout << "for " << err.what() << ", item = " << item << std::endl; + throw; + } } template<> template<> -- cgit v1.2.3 From 761d8337fb70cea9cfb326f06a8b1b35438ed96b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 2 Dec 2022 17:14:31 -0500 Subject: SL-18330: Test Python llsd.parse() both from bytes and from stream. --- indra/llcommon/tests/llsdserialize_test.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 39b93cf335..6e5d8a26e1 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1958,6 +1958,7 @@ namespace tut placeholders::arg1 << import_llsd << "from functools import partial\n" + "import io\n" "import struct\n" "lenformat = struct.Struct('i')\n" "def parse_each(inf):\n" @@ -1968,11 +1969,16 @@ namespace tut // into a distinct bytes object and parse that. " data = inf.read(len)\n" " try:\n" - " yield llsd.parse(data)\n" + " frombytes = llsd.parse(data)\n" " except llsd.LLSDParseError as err:\n" " print(f'*** {err}')\n" " print(f'Bad content:\\n{data!r}')\n" " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); -- cgit v1.2.3 From 590e158cf36b6e6fca07f54837db084fe8a163c6 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Dec 2022 16:30:35 -0500 Subject: SL-18330: Adapt LLSDSerialize and tests to llssize max_bytes params. --- indra/llcommon/llsdserialize.cpp | 2 +- indra/llcommon/tests/llsdserialize_test.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 97bf51eeaa..c1b24d8138 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -128,7 +128,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) * specified buffer. In the usual case when max_bytes exceeds * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. */ - str.get(hdr_buf, std::min(max_bytes+1, sizeof(hdr_buf)-1), '\n'); + str.get(hdr_buf, llmin(max_bytes+1, sizeof(hdr_buf)-1), '\n'); auto inbuf = str.gcount(); // https://en.cppreference.com/w/cpp/io/basic_istream/get // When the get() above sees the specified delimiter '\n', it stops there diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 6e5d8a26e1..29e3007aff 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -63,7 +63,7 @@ using namespace boost::phoenix; #include typedef std::function FormatterFunction; -typedef std::function ParserFunction; +typedef std::function ParserFunction; std::vector string_to_vector(const std::string& str) { @@ -263,7 +263,7 @@ namespace tut }; // this lambda must be mutable since otherwise the bound 'parser' // is assumed to point to a const LLSDParser - mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) mutable + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) mutable { // reset() call is needed since test code re-uses parser object parser->reset(); @@ -271,10 +271,10 @@ namespace tut }; } - void setParser(bool (*parser)(LLSD&, std::istream&, size_t)) + void setParser(bool (*parser)(LLSD&, std::istream&, llssize)) { // why does LLSDSerialize::deserialize() reverse the parse() params?? - mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) { return (parser(data, istr, max_bytes) > 0); }; @@ -2059,7 +2059,7 @@ namespace tut // helper for test<8> - test<12> void fromPythonUsing(const std::string& pyformatter, const ParserFunction& parse= - [](std::istream& istr, LLSD& data, size_t max_bytes) + [](std::istream& istr, LLSD& data, llssize max_bytes) { return LLSDSerialize::deserialize(data, istr, max_bytes); }) { // Create an empty data file. This is just a placeholder for our @@ -2140,7 +2140,7 @@ namespace tut set_test_name("from Python XML using fromXML()"); // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors fromPythonUsing("format_xml", - [](std::istream& istr, LLSD& data, size_t) + [](std::istream& istr, LLSD& data, llssize) { return LLSDSerialize::fromXML(data, istr) > 0; }); } @@ -2149,7 +2149,7 @@ namespace tut { set_test_name("from Python notation using fromNotation()"); fromPythonUsing("format_notation", - [](std::istream& istr, LLSD& data, size_t max_bytes) + [](std::istream& istr, LLSD& data, llssize max_bytes) { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); } @@ -2161,7 +2161,7 @@ namespace tut // We don't expect this to work because format_binary() emits a // header, but fromBinary() won't recognize a header. fromPythonUsing("format_binary", - [](std::istream& istr, LLSD& data, size_t max_bytes) + [](std::istream& istr, LLSD& data, llssize max_bytes) { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); } |*==========================================================================*/ -- cgit v1.2.3 From 61b93e778bf58f6db4d2f3d8dd6ec67de25598ea Mon Sep 17 00:00:00 2001 From: Henri Beauchamp Date: Tue, 7 Feb 2023 00:17:22 +0100 Subject: SL-19110 Make HBXXH* classes no-copy. (#72) These classes are not trivially copyable because of the mState pointer on an internal XXH3 state that would have to be explicitely copied. While it would be possible to add custom copy constructor and operator for them, it does not really make sense to allow copying an instance of these classes, since all we care about storing and copying is the digest (which is either an U64 or an LLUUID). --- indra/llcommon/hbxxh.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/hbxxh.h b/indra/llcommon/hbxxh.h index 236716722a..9c0e9cf172 100644 --- a/indra/llcommon/hbxxh.h +++ b/indra/llcommon/hbxxh.h @@ -96,6 +96,15 @@ public: } } + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH64(const HBXXH64&) noexcept = delete; + HBXXH64& operator=(const HBXXH64&) noexcept = delete; + ~HBXXH64(); void update(const void* buffer, size_t len); @@ -199,6 +208,15 @@ public: } } + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH128(const HBXXH128&) noexcept = delete; + HBXXH128& operator=(const HBXXH128&) noexcept = delete; + ~HBXXH128(); void update(const void* buffer, size_t len); -- cgit v1.2.3 From 1eb2b23f4b254052924b198d446c052df9ca3124 Mon Sep 17 00:00:00 2001 From: Henri Beauchamp Date: Tue, 7 Feb 2023 13:51:53 +0100 Subject: SL-19159 Faster LLUUID and LLMaterialID hashing for std and boost containers keys (#70) LLUUID and LLMaterialID already have an excellent entropy and value dispersion; there is therefore strictly no need to further (slowly) hash their value for use with std and boost libraries containers. This commit adds a trivial getDigest64() method to both LLUUID and LLMaterialID (which simply returns the XOR of the two 64 bits long words their value is made of), and uses it in std::hash and hash_value() specializations for use with containers. --- indra/llcommon/lluuid.cpp | 2 +- indra/llcommon/lluuid.h | 50 ++++++++++++++++++++--------------------------- 2 files changed, 22 insertions(+), 30 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index 8ff6c45760..aa8045e6ab 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -878,7 +878,7 @@ U32 LLUUID::getRandomSeed() seed[7]=(unsigned char)(pid); getSystemTime((uuid_time_t *)(&seed[8])); - U64 seed64 = HBXXH64((const void*)seed, 16).digest(); + U64 seed64 = HBXXH64::digest((const void*)seed, 16); return U32(seed64) ^ U32(seed64 >> 32); } diff --git a/indra/llcommon/lluuid.h b/indra/llcommon/lluuid.h index 86a396ab06..2c36a1d222 100644 --- a/indra/llcommon/lluuid.h +++ b/indra/llcommon/lluuid.h @@ -119,6 +119,14 @@ public: U16 getCRC16() const; U32 getCRC32() const; + // Returns a 64 bits digest of the UUID, by XORing its two 64 bits long + // words. HB + inline U64 getDigest64() const + { + U64* tmp = (U64*)mData; + return tmp[0] ^ tmp[1]; + } + static BOOL validate(const std::string& in_string); // Validate that the UUID string is legal. static const LLUUID null; @@ -165,36 +173,20 @@ public: LLAssetID makeAssetID(const LLUUID& session) const; }; -// Generate a hash of an LLUUID object using the boost hash templates. -template <> -struct boost::hash -{ - typedef LLUUID argument_type; - typedef std::size_t result_type; - result_type operator()(argument_type const& s) const - { - result_type seed(0); - - for (S32 i = 0; i < UUID_BYTES; ++i) - { - boost::hash_combine(seed, s.mData[i]); - } - - return seed; - } -}; - -// Adapt boost hash to std hash +// std::hash implementation for LLUUID namespace std { - template<> struct hash - { - std::size_t operator()(LLUUID const& s) const noexcept - { - return boost::hash()(s); - } - }; + template<> struct hash + { + inline size_t operator()(const LLUUID& id) const noexcept + { + return (size_t)id.getDigest64(); + } + }; } -#endif - +// For use with boost containers. +inline size_t hash_value(const LLUUID& id) noexcept +{ + return (size_t)id.getDigest64(); +} -- cgit v1.2.3 From e2836313a8d2b5e2acd4fd20bde50521613ed0ef Mon Sep 17 00:00:00 2001 From: Henri Beauchamp Date: Mon, 13 Feb 2023 18:34:00 +0100 Subject: SL-19110 revert LLUUID::combine() to old algorithm to match server code. (#75) As it happens, the change in the LLUUID::combine() algorithm introduced by one of my previous commits is causing invalid assets creation (seen with some clothing items, such as Shape and Universal types); obviously, the server is using the old algorithm for UUID validation purpose of these assets. This commit reverts LLUUID::combine() code to use LLMD5. --- indra/llcommon/lluuid.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index aa8045e6ab..adf31b2d28 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -44,6 +44,7 @@ #include "lltimer.h" #include "llthread.h" #include "llmutex.h" +#include "llmd5.h" #include "hbxxh.h" const LLUUID LLUUID::null; @@ -400,11 +401,16 @@ LLUUID LLUUID::operator^(const LLUUID& rhs) const return id; } +// WARNING: this algorithm SHALL NOT be changed. It is also used by the server +// and plays a role in some assets validation (e.g. clothing items). Changing +// it would cause invalid assets. void LLUUID::combine(const LLUUID& other, LLUUID& result) const { - HBXXH128 hash((const void*)mData, 16, false); // false = do not finalize - hash.update((const void*)other.mData, 16); - hash.digest(result); + LLMD5 md5_uuid; + md5_uuid.update((unsigned char*)mData, 16); + md5_uuid.update((unsigned char*)other.mData, 16); + md5_uuid.finalize(); + md5_uuid.raw_digest(result.mData); } LLUUID LLUUID::combine(const LLUUID &other) const -- cgit v1.2.3 From a92f08b2bc0ee5a63f678c55f9a9d0ec0c4ff574 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Mon, 13 Feb 2023 15:08:51 -0800 Subject: Fix up a couple of tiny merge conflicts after a merge with main --- indra/llcommon/llsdserialize.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index cf6c1ab73a..2f12c6d1ff 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -870,7 +870,7 @@ public: LL_COMMON_API std::string zip_llsd(LLSD& data); -LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize,std::istream& is, S32 size); +LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, size_t& outsize,std::istream& is, S32 size); // returns a pointer to the array or past the array if the deprecated header exists LL_COMMON_API char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size = nullptr); -- cgit v1.2.3 From e2496eff18315a82a82e50784c5959dfb038d02f Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Wed, 15 Feb 2023 23:42:58 +0200 Subject: SL-19159 Build fixes --- indra/llcommon/lluuid.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lluuid.h b/indra/llcommon/lluuid.h index 2c36a1d222..cc36797bc1 100644 --- a/indra/llcommon/lluuid.h +++ b/indra/llcommon/lluuid.h @@ -190,3 +190,5 @@ inline size_t hash_value(const LLUUID& id) noexcept { return (size_t)id.getDigest64(); } + +#endif -- cgit v1.2.3 From 152f32918d15e9343658b728adaf1d5c8c3c3e10 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 15 Feb 2023 16:50:18 -0500 Subject: SL-18330: Fix egregious existing build errors in contribute branch. --- indra/llcommon/lluuid.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lluuid.h b/indra/llcommon/lluuid.h index cc36797bc1..0d4896ee97 100644 --- a/indra/llcommon/lluuid.h +++ b/indra/llcommon/lluuid.h @@ -191,4 +191,4 @@ inline size_t hash_value(const LLUUID& id) noexcept return (size_t)id.getDigest64(); } -#endif +#endif // LL_LLUUID_H -- cgit v1.2.3 From 292bb3991b589d39d61cf721b82fe7bdae460785 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 15 Feb 2023 17:29:48 -0500 Subject: SL-18330: Fix per PR review comments. --- indra/llcommon/llsdserialize.cpp | 6 +++++- indra/llcommon/llstreamtools.cpp | 2 +- indra/llcommon/llstreamtools.h | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 0a5e3652c7..a14a6b5b1b 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -169,9 +169,13 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) /* * Remove the newline chars */ - auto lastchar = header.find_last_not_of("\r\n"); + std::string::size_type lastchar = header.find_last_not_of("\r\n"); if (lastchar != std::string::npos) { + // It's important that find_last_not_of() returns size_type, which is + // why lastchar explicitly declares the type above. erase(size_type) + // erases from that offset to the end of the string, whereas + // erase(iterator) erases only a single character. header.erase(lastchar+1); } diff --git a/indra/llcommon/llstreamtools.cpp b/indra/llcommon/llstreamtools.cpp index 979e96b848..bc32b6fd9e 100644 --- a/indra/llcommon/llstreamtools.cpp +++ b/indra/llcommon/llstreamtools.cpp @@ -519,7 +519,7 @@ int cat_streambuf::underflow() if (gptr() == egptr()) { // here because our buffer is empty - std::streamsize size; + std::streamsize size = 0; // Until we've run out of mInputs, try reading the first of them // into mBuffer. If that fetches some characters, break the loop. while (! mInputs.empty() diff --git a/indra/llcommon/llstreamtools.h b/indra/llcommon/llstreamtools.h index 997f738840..bb7bc20327 100644 --- a/indra/llcommon/llstreamtools.h +++ b/indra/llcommon/llstreamtools.h @@ -135,7 +135,7 @@ public: mBuffer(1024) {} - int underflow(); + int underflow() override; }; #endif -- cgit v1.2.3 From ba47aa4afff0cdcf08617b48660ef8505b9d43c0 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 1 Mar 2023 14:11:27 -0500 Subject: SL-18330: Use C++ standard types for llmd5, not local typedefs. Not only do the local typedefs make the code less readable, they also rely on assumptions about the implementation. The standard types are guaranteed by the C++ library implementation. --- indra/llcommon/llmd5.cpp | 32 ++++++++++++++++---------------- indra/llcommon/llmd5.h | 23 ++++++++++------------- 2 files changed, 26 insertions(+), 29 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llmd5.cpp b/indra/llcommon/llmd5.cpp index 9b2a2bab60..0abe817f1d 100644 --- a/indra/llcommon/llmd5.cpp +++ b/indra/llcommon/llmd5.cpp @@ -96,7 +96,7 @@ LLMD5::LLMD5() // operation, processing another message block, and updating the // context. -void LLMD5::update (const uint1 *input, const size_t input_length) { +void LLMD5::update (const uint8_t *input, const size_t input_length) { size_t input_index, buffer_index; size_t buffer_space; // how much space is left in buffer @@ -189,7 +189,7 @@ void LLMD5::finalize (){ unsigned char bits[8]; /* Flawfinder: ignore */ size_t index, padLen; - static uint1 PADDING[64]={ + static uint8_t PADDING[64]={ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 @@ -201,8 +201,8 @@ void LLMD5::finalize (){ } // Save number of bits. - // Treat count, a uint64_t, as uint4[2]. - encode (bits, reinterpret_cast(&count), 8); + // Treat count, a uint64_t, as uint32_t[2]. + encode (bits, reinterpret_cast(&count), 8); // Pad out to 56 mod 64. index = size_t((count >> 3) & 0x3f); @@ -412,7 +412,7 @@ Rotation is separate from addition to prevent recomputation. // LLMD5 basic transformation. Transforms state based on block. void LLMD5::transform (const U8 block[64]){ - uint4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; + uint32_t a = state[0], b = state[1], c = state[2], d = state[3], x[16]; decode (x, block, 64); @@ -496,38 +496,38 @@ void LLMD5::transform (const U8 block[64]){ state[3] += d; // Zeroize sensitive information. - memset ( (uint1 *) x, 0, sizeof(x)); + memset ( (uint8_t *) x, 0, sizeof(x)); } -// Encodes input (UINT4) into output (unsigned char). Assumes len is +// Encodes input (uint32_t) into output (unsigned char). Assumes len is // a multiple of 4. -void LLMD5::encode (uint1 *output, const uint4 *input, const size_t len) { +void LLMD5::encode (uint8_t *output, const uint32_t *input, const size_t len) { size_t i, j; for (i = 0, j = 0; j < len; i++, j += 4) { - output[j] = (uint1) (input[i] & 0xff); - output[j+1] = (uint1) ((input[i] >> 8) & 0xff); - output[j+2] = (uint1) ((input[i] >> 16) & 0xff); - output[j+3] = (uint1) ((input[i] >> 24) & 0xff); + output[j] = (uint8_t) (input[i] & 0xff); + output[j+1] = (uint8_t) ((input[i] >> 8) & 0xff); + output[j+2] = (uint8_t) ((input[i] >> 16) & 0xff); + output[j+3] = (uint8_t) ((input[i] >> 24) & 0xff); } } -// Decodes input (unsigned char) into output (UINT4). Assumes len is +// Decodes input (unsigned char) into output (uint32_t). Assumes len is // a multiple of 4. -void LLMD5::decode (uint4 *output, const uint1 *input, const size_t len){ +void LLMD5::decode (uint32_t *output, const uint8_t *input, const size_t len){ size_t i, j; for (i = 0, j = 0; j < len; i++, j += 4) - output[i] = ((uint4)input[j]) | (((uint4)input[j+1]) << 8) | - (((uint4)input[j+2]) << 16) | (((uint4)input[j+3]) << 24); + output[i] = ((uint32_t)input[j]) | (((uint32_t)input[j+1]) << 8) | + (((uint32_t)input[j+2]) << 16) | (((uint32_t)input[j+3]) << 24); } diff --git a/indra/llcommon/llmd5.h b/indra/llcommon/llmd5.h index 8530dc0389..7d6373c20c 100644 --- a/indra/llcommon/llmd5.h +++ b/indra/llcommon/llmd5.h @@ -67,6 +67,8 @@ documentation and/or software. */ +#include // uint32_t et al. + // use for the raw digest output const int MD5RAW_BYTES = 16; @@ -75,18 +77,13 @@ const int MD5HEX_STR_SIZE = 33; // char hex[MD5HEX_STR_SIZE]; with null const int MD5HEX_STR_BYTES = 32; // message system fixed size class LL_COMMON_API LLMD5 { -// first, some types: - typedef unsigned int uint4; // assumes integer is 4 words long - typedef unsigned short int uint2; // assumes short integer is 2 words long - typedef unsigned char uint1; // assumes char is 1 word long - // how many bytes to grab at a time when checking files static const int BLOCK_LEN; public: // methods for controlled operation: LLMD5 (); // simple initializer - void update (const uint1 *input, const size_t input_length); + void update (const uint8_t *input, const size_t input_length); void update (std::istream& stream); void update (FILE *file); void update (const std::string& str); @@ -109,19 +106,19 @@ private: // next, the private data: - uint4 state[4]; + uint32_t state[4]; uint64_t count; // number of *bits*, mod 2^64 - uint1 buffer[64]; // input buffer - uint1 digest[16]; - uint1 finalized; + uint8_t buffer[64]; // input buffer + uint8_t digest[16]; + uint8_t finalized; // last, the private methods, mostly static: void init (); // called by all constructors - void transform (const uint1 *buffer); // does the real update work. Note + void transform (const uint8_t *buffer); // does the real update work. Note // that length is implied to be 64. - static void encode (uint1 *dest, const uint4 *src, const size_t length); - static void decode (uint4 *dest, const uint1 *src, const size_t length); + static void encode (uint8_t *dest, const uint32_t *src, const size_t length); + static void decode (uint32_t *dest, const uint8_t *src, const size_t length); }; -- cgit v1.2.3 From 1bba8fd31e2dabf27fe2b4b9177bf156799d7299 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 1 Mar 2023 14:20:33 -0500 Subject: SL-18330: Use size_t max for LLTrace::PeriodicRecording methods. The num_periods arguments have all been changed to size_t, but the default argument values were still coded as S32_MAX. Change to std::numeric_limits::max(). --- indra/llcommon/lltracerecording.cpp | 26 ++++++------ indra/llcommon/lltracerecording.h | 81 +++++++++++++++++++------------------ 2 files changed, 54 insertions(+), 53 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lltracerecording.cpp b/indra/llcommon/lltracerecording.cpp index a8dcc5226a..5f16334c84 100644 --- a/indra/llcommon/lltracerecording.cpp +++ b/indra/llcommon/lltracerecording.cpp @@ -789,7 +789,7 @@ void PeriodicRecording::handleSplitTo(PeriodicRecording& other) getCurRecording().splitTo(other.getCurRecording()); } -F64 PeriodicRecording::getPeriodMin( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMin( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -811,7 +811,7 @@ F64 PeriodicRecording::getPeriodMin( const StatType& stat, siz : NaN; } -F64 PeriodicRecording::getPeriodMax( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMax( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -834,7 +834,7 @@ F64 PeriodicRecording::getPeriodMax( const StatType& stat, siz } // calculates means using aggregates per period -F64 PeriodicRecording::getPeriodMean( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMean( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -857,7 +857,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType& stat, si : NaN; } -F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -882,7 +882,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMin( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -904,7 +904,7 @@ F64 PeriodicRecording::getPeriodMin( const StatType& stat, si : NaN; } -F64 PeriodicRecording::getPeriodMax(const StatType& stat, size_t num_periods /*= S32_MAX*/) +F64 PeriodicRecording::getPeriodMax(const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -927,7 +927,7 @@ F64 PeriodicRecording::getPeriodMax(const StatType& stat, siz } -F64 PeriodicRecording::getPeriodMean( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMean( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -950,7 +950,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType& stat, s : NaN; } -F64 PeriodicRecording::getPeriodMedian( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMedian( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -976,7 +976,7 @@ F64 PeriodicRecording::getPeriodMedian( const StatType& stat, return F64((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); } -F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -1002,7 +1002,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodMin( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -1022,7 +1022,7 @@ F64Kilobytes PeriodicRecording::getPeriodMin(const MemStatHandle& stat, size_t n return getPeriodMin(static_cast&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodMax(const StatType& stat, size_t num_periods /*= S32_MAX*/) +F64Kilobytes PeriodicRecording::getPeriodMax(const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -1042,7 +1042,7 @@ F64Kilobytes PeriodicRecording::getPeriodMax(const MemStatHandle& stat, size_t n return getPeriodMax(static_cast&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodMean( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodMean( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -1063,7 +1063,7 @@ F64Kilobytes PeriodicRecording::getPeriodMean(const MemStatHandle& stat, size_t return getPeriodMean(static_cast&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, size_t num_periods /*= std::numeric_limits::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); diff --git a/indra/llcommon/lltracerecording.h b/indra/llcommon/lltracerecording.h index 8b56721f42..66cee59764 100644 --- a/indra/llcommon/lltracerecording.h +++ b/indra/llcommon/lltracerecording.h @@ -33,6 +33,7 @@ #include "lltimer.h" #include "lltraceaccumulators.h" #include "llpointer.h" +#include class LLStopWatchControlsMixinCommon { @@ -353,7 +354,7 @@ namespace LLTrace Recording snapshotCurRecording() const; template - auto getSampleCount(const StatType& stat, size_t num_periods = S32_MAX) + auto getSampleCount(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -373,7 +374,7 @@ namespace LLTrace // catch all for stats that have a defined sum template - typename T::value_t getPeriodMin(const StatType& stat, size_t num_periods = S32_MAX) + typename T::value_t getPeriodMin(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -396,33 +397,33 @@ namespace LLTrace } template - T getPeriodMin(const CountStatHandle& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast&>(stat), num_periods)); } - F64 getPeriodMin(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMin(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - T getPeriodMin(const SampleStatHandle& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const SampleStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast&>(stat), num_periods)); } - F64 getPeriodMin(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMin(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - T getPeriodMin(const EventStatHandle& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const EventStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast&>(stat), num_periods)); } - F64Kilobytes getPeriodMin(const StatType& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMin(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMin(const StatType& stat, size_t num_periods = std::numeric_limits::max()); + F64Kilobytes getPeriodMin(const MemStatHandle& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodMinPerSec(const StatType& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMinPerSec(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -437,7 +438,7 @@ namespace LLTrace } template - typename RelatedTypes::fractional_t getPeriodMinPerSec(const CountStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMinPerSec(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMinPerSec(static_cast&>(stat), num_periods)); @@ -449,7 +450,7 @@ namespace LLTrace // catch all for stats that have a defined sum template - typename T::value_t getPeriodMax(const StatType& stat, size_t num_periods = S32_MAX) + typename T::value_t getPeriodMax(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -472,33 +473,33 @@ namespace LLTrace } template - T getPeriodMax(const CountStatHandle& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast&>(stat), num_periods)); } - F64 getPeriodMax(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMax(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - T getPeriodMax(const SampleStatHandle& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const SampleStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast&>(stat), num_periods)); } - F64 getPeriodMax(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMax(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - T getPeriodMax(const EventStatHandle& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const EventStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast&>(stat), num_periods)); } - F64Kilobytes getPeriodMax(const StatType& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMax(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMax(const StatType& stat, size_t num_periods = std::numeric_limits::max()); + F64Kilobytes getPeriodMax(const MemStatHandle& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodMaxPerSec(const StatType& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMaxPerSec(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -513,7 +514,7 @@ namespace LLTrace } template - typename RelatedTypes::fractional_t getPeriodMaxPerSec(const CountStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMaxPerSec(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMaxPerSec(static_cast&>(stat), num_periods)); @@ -525,7 +526,7 @@ namespace LLTrace // catch all for stats that have a defined sum template - typename RelatedTypes::fractional_t getPeriodMean(const StatType& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMean(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -546,32 +547,32 @@ namespace LLTrace } template - typename RelatedTypes::fractional_t getPeriodMean(const CountStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMean(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMean(static_cast&>(stat), num_periods)); } - F64 getPeriodMean(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMean(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodMean(const SampleStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMean(const SampleStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMean(static_cast&>(stat), num_periods)); } - F64 getPeriodMean(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMean(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodMean(const EventStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMean(const EventStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMean(static_cast&>(stat), num_periods)); } - F64Kilobytes getPeriodMean(const StatType& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMean(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMean(const StatType& stat, size_t num_periods = std::numeric_limits::max()); + F64Kilobytes getPeriodMean(const MemStatHandle& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodMeanPerSec(const StatType& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMeanPerSec(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -593,16 +594,16 @@ namespace LLTrace } template - typename RelatedTypes::fractional_t getPeriodMeanPerSec(const CountStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMeanPerSec(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMeanPerSec(static_cast&>(stat), num_periods)); } - F64 getPeriodMedian( const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodMedian( const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodMedianPerSec(const StatType& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMedianPerSec(const StatType& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -622,7 +623,7 @@ namespace LLTrace } template - typename RelatedTypes::fractional_t getPeriodMedianPerSec(const CountStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodMedianPerSec(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodMedianPerSec(static_cast&>(stat), num_periods)); @@ -632,25 +633,25 @@ namespace LLTrace // PERIODIC STANDARD DEVIATION // - F64 getPeriodStandardDeviation(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodStandardDeviation(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodStandardDeviation(const SampleStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodStandardDeviation(const SampleStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodStandardDeviation(static_cast&>(stat), num_periods)); } - F64 getPeriodStandardDeviation(const StatType& stat, size_t num_periods = S32_MAX); + F64 getPeriodStandardDeviation(const StatType& stat, size_t num_periods = std::numeric_limits::max()); template - typename RelatedTypes::fractional_t getPeriodStandardDeviation(const EventStatHandle& stat, size_t num_periods = S32_MAX) + typename RelatedTypes::fractional_t getPeriodStandardDeviation(const EventStatHandle& stat, size_t num_periods = std::numeric_limits::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes::fractional_t(getPeriodStandardDeviation(static_cast&>(stat), num_periods)); } - F64Kilobytes getPeriodStandardDeviation(const StatType& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodStandardDeviation(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodStandardDeviation(const StatType& stat, size_t num_periods = std::numeric_limits::max()); + F64Kilobytes getPeriodStandardDeviation(const MemStatHandle& stat, size_t num_periods = std::numeric_limits::max()); private: // implementation for LLStopWatchControlsMixin -- cgit v1.2.3 From 18ac7ab027217e05876a066c86ab3fbaa2a7328f Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 1 Mar 2023 14:33:35 -0500 Subject: SL-18330: Use recursive variadic llmax(), llmin() instead of having specific binary, ternary and quaternary overloads. --- indra/llcommon/lldefs.h | 48 +++++++++++++++++------------------------------- 1 file changed, 17 insertions(+), 31 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lldefs.h b/indra/llcommon/lldefs.h index 5c46f6a796..4e25001fff 100644 --- a/indra/llcommon/lldefs.h +++ b/indra/llcommon/lldefs.h @@ -167,48 +167,34 @@ const U32 MAXADDRSTR = 17; // 123.567.901.345 = 15 chars + \0 + 1 for good luc // // defined for U16, U32, U64, S16, S32, S64, : // llclampb(a) // clamps a to [0 .. 255] -// - -template -inline auto llmax(T1 d1, T2 d2) -{ - return (d1 > d2) ? d1 : d2; -} - -template -inline auto llmax(T1 d1, T2 d2, T3 d3) -{ - auto r = llmax(d1,d2); - return llmax(r, d3); -} +// -template -inline auto llmax(T1 d1, T2 d2, T3 d3, T4 d4) +// recursion tail +template +inline auto llmax(T data) { - auto r1 = llmax(d1,d2); - auto r2 = llmax(d3,d4); - return llmax(r1, r2); + return data; } -template -inline auto llmin(T1 d1, T2 d2) +template +inline auto llmax(T0 d0, T1 d1, Ts... rest) { - return (d1 < d2) ? d1 : d2; + auto maxrest = llmax(d1, rest...); + return (d0 > maxrest)? d0 : maxrest; } -template -inline auto llmin(T1 d1, T2 d2, T3 d3) +// recursion tail +template +inline auto llmin(T data) { - auto r = llmin(d1,d2); - return (r < d3 ? r : d3); + return data; } -template -inline auto llmin(T1 d1, T2 d2, T3 d3, T4 d4) +template +inline auto llmin(T0 d0, T1 d1, Ts... rest) { - auto r1 = llmin(d1,d2); - auto r2 = llmin(d3,d4); - return llmin(r1, r2); + auto minrest = llmin(d1, rest...); + return (d0 < minrest) ? d0 : minrest; } template -- cgit v1.2.3 From c18f0a61abfe8b9527f9a91b0e52e909cf7debfc Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 1 Mar 2023 14:35:04 -0500 Subject: SL-18330: Fix a narrowing conversion in LLSDSerialize::deserialize(). --- indra/llcommon/llsdserialize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index a14a6b5b1b..99d0c39694 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -143,7 +143,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) // byte. We could store one if needed, since even the incremented // inbuf won't exceed sizeof(hdr_buf)-1, but there's no need. } - std::string header{ hdr_buf, inbuf }; + std::string header{ hdr_buf, narrow(inbuf) }; if (str.fail()) { str.clear(); -- cgit v1.2.3 From 6cb6385bc71417c1017dc5ccabe53a678e371684 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 1 Mar 2023 16:37:55 -0500 Subject: SL-18330: Tweaks for Visual Studio builds --- indra/llcommon/llsdserialize.cpp | 2 +- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 99d0c39694..046523dbb1 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -143,7 +143,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) // byte. We could store one if needed, since even the incremented // inbuf won't exceed sizeof(hdr_buf)-1, but there's no need. } - std::string header{ hdr_buf, narrow(inbuf) }; + std::string header{ hdr_buf, static_cast(inbuf) }; if (str.fail()) { str.clear(); diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 29e3007aff..618f33cc13 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -276,7 +276,7 @@ namespace tut // why does LLSDSerialize::deserialize() reverse the parse() params?? mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) { - return (parser(data, istr, max_bytes) > 0); + return parser(data, istr, max_bytes); }; } -- cgit v1.2.3 From 42b1fd21814e679a1557e0d5a55943c5b0e63723 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 2 Mar 2023 15:00:50 -0500 Subject: SL-18330: Review and tweak LLTrace::PeriodicRecording indexing per Leviathan code review. --- indra/llcommon/lltracerecording.cpp | 98 ++++++++++++++++--------------------- indra/llcommon/lltracerecording.h | 97 +++++++++++++++++++++++------------- 2 files changed, 105 insertions(+), 90 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lltracerecording.cpp b/indra/llcommon/lltracerecording.cpp index 5f16334c84..bb3d667a42 100644 --- a/indra/llcommon/lltracerecording.cpp +++ b/indra/llcommon/lltracerecording.cpp @@ -577,10 +577,12 @@ S32 Recording::getSampleCount( const StatType& stat ) // PeriodicRecording /////////////////////////////////////////////////////////////////////// -PeriodicRecording::PeriodicRecording( S32 num_periods, EPlayState state) +PeriodicRecording::PeriodicRecording( size_t num_periods, EPlayState state) : mAutoResize(num_periods == 0), mCurPeriod(0), mNumRecordedPeriods(0), + // This guarantee that mRecordingPeriods cannot be empty is essential for + // code in several methods. mRecordingPeriods(num_periods ? num_periods : 1) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; @@ -596,18 +598,19 @@ PeriodicRecording::~PeriodicRecording() void PeriodicRecording::nextPeriod() { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; if (mAutoResize) { mRecordingPeriods.push_back(Recording()); } Recording& old_recording = getCurRecording(); - mCurPeriod = (mCurPeriod + 1) % mRecordingPeriods.size(); + inci(mCurPeriod); old_recording.splitTo(getCurRecording()); - mNumRecordedPeriods = mRecordingPeriods.empty()? 0 : - llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + 1); + // Since mRecordingPeriods always has at least one entry, we can always + // safely subtract 1 from its size(). + mNumRecordedPeriods = llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + 1); } void PeriodicRecording::appendRecording(Recording& recording) @@ -620,31 +623,29 @@ void PeriodicRecording::appendRecording(Recording& recording) void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; if (other.mRecordingPeriods.empty()) return; getCurRecording().update(); other.getCurRecording().update(); - - const auto other_recording_slots = other.mRecordingPeriods.size(); + const auto other_num_recordings = other.getNumRecordedPeriods(); const auto other_current_recording_index = other.mCurPeriod; - const auto other_oldest_recording_index = (other_current_recording_index + other_recording_slots - other_num_recordings) % other_recording_slots; + const auto other_oldest_recording_index = other.previ(other_current_recording_index, other_num_recordings); // append first recording into our current slot getCurRecording().appendRecording(other.mRecordingPeriods[other_oldest_recording_index]); // from now on, add new recordings for everything after the first - auto other_index = (other_oldest_recording_index + 1) % other_recording_slots; + auto other_index = other.nexti(other_oldest_recording_index); if (mAutoResize) { // push back recordings for everything in the middle - auto other_index = (other_oldest_recording_index + 1) % other_recording_slots; while (other_index != other_current_recording_index) { mRecordingPeriods.push_back(other.mRecordingPeriods[other_index]); - other_index = (other_index + 1) % other_recording_slots; + other.inci(other_index); } // add final recording, if it wasn't already added as the first @@ -653,36 +654,25 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) mRecordingPeriods.push_back(other.mRecordingPeriods[other_current_recording_index]); } - mCurPeriod = mRecordingPeriods.empty()? 0 : mRecordingPeriods.size() - 1; + // mRecordingPeriods is never empty() + mCurPeriod = mRecordingPeriods.size() - 1; mNumRecordedPeriods = mCurPeriod; } else { - S32 num_to_copy = llmin((S32)mRecordingPeriods.size(), (S32)other_num_recordings); - - std::vector::iterator src_it = other.mRecordingPeriods.begin() + other_index ; - std::vector::iterator dest_it = mRecordingPeriods.begin() + mCurPeriod; - + auto num_to_copy = llmin(mRecordingPeriods.size(), other_num_recordings); // already consumed the first recording from other, so start counting at 1 - for(S32 i = 1; i < num_to_copy; i++) + for (size_t n = 1, srci = other_index, dsti = mCurPeriod; + n < num_to_copy; + ++n, other.inci(srci), inci(dsti)) { - *dest_it = *src_it; - - if (++src_it == other.mRecordingPeriods.end()) - { - src_it = other.mRecordingPeriods.begin(); - } - - if (++dest_it == mRecordingPeriods.end()) - { - dest_it = mRecordingPeriods.begin(); - } + mRecordingPeriods[dsti] = other.mRecordingPeriods[srci]; } - + // want argument to % to be positive, otherwise result could be negative and thus out of bounds llassert(num_to_copy >= 1); // advance to last recording period copied, and make that our current period - mCurPeriod = (mCurPeriod + num_to_copy - 1) % mRecordingPeriods.size(); + inci(mCurPeriod, num_to_copy - 1); mNumRecordedPeriods = llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + num_to_copy - 1); } @@ -694,13 +684,11 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) F64Seconds PeriodicRecording::getDuration() const { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; F64Seconds duration; - auto num_periods = mRecordingPeriods.size(); - for (size_t i = 1; i <= num_periods; i++) + for (size_t n = 0; n < mRecordingPeriods.size(); ++n) { - auto index = (mCurPeriod + num_periods - i) % num_periods; - duration += mRecordingPeriods[index].getDuration(); + duration += mRecordingPeriods[nexti(mCurPeriod, n)].getDuration(); } return duration; } @@ -737,16 +725,14 @@ const Recording& PeriodicRecording::getCurRecording() const Recording& PeriodicRecording::getPrevRecording( size_t offset ) { - auto num_periods = mRecordingPeriods.size(); - offset = llclamp(offset, 0, num_periods - 1); - return mRecordingPeriods[(mCurPeriod + num_periods - offset) % num_periods]; + // reuse const implementation, but return non-const reference + return const_cast( + const_cast(this)->getPrevRecording(offset)); } const Recording& PeriodicRecording::getPrevRecording( size_t offset ) const { - auto num_periods = mRecordingPeriods.size(); - offset = llclamp(offset, 0, num_periods - 1); - return mRecordingPeriods[(mCurPeriod + num_periods - offset) % num_periods]; + return mRecordingPeriods[previ(mCurPeriod, offset)]; } void PeriodicRecording::handleStart() @@ -796,7 +782,7 @@ F64 PeriodicRecording::getPeriodMin( const StatType& stat, siz bool has_value = false; F64 min_val = std::numeric_limits::max(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -818,7 +804,7 @@ F64 PeriodicRecording::getPeriodMax( const StatType& stat, siz bool has_value = false; F64 max_val = std::numeric_limits::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -842,7 +828,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType& stat, si F64 mean = 0; S32 valid_period_count = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -866,7 +852,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& stat, si bool has_value = false; F64 min_val = std::numeric_limits::max(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -911,7 +897,7 @@ F64 PeriodicRecording::getPeriodMax(const StatType& stat, siz bool has_value = false; F64 max_val = std::numeric_limits::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -935,7 +921,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType& stat, s S32 valid_period_count = 0; F64 mean = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -956,7 +942,7 @@ F64 PeriodicRecording::getPeriodMedian( const StatType& stat, num_periods = llmin(num_periods, getNumRecordedPeriods()); std::vector buf; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -985,7 +971,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType& st num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes min_val(std::numeric_limits::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); min_val = llmin(min_val, recording.getMin(stat)); @@ -1028,7 +1014,7 @@ F64Kilobytes PeriodicRecording::getPeriodMax(const StatType& sta num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes max_val(0.0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); max_val = llmax(max_val, recording.getMax(stat)); @@ -1049,7 +1035,7 @@ F64Kilobytes PeriodicRecording::getPeriodMean( const StatType& s F64Kilobytes mean(0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); mean += recording.getMean(stat); @@ -1072,7 +1058,7 @@ F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType { public: - PeriodicRecording(S32 num_periods, EPlayState state = STOPPED); + PeriodicRecording(size_t num_periods, EPlayState state = STOPPED); ~PeriodicRecording(); void nextPeriod(); @@ -381,7 +381,7 @@ namespace LLTrace bool has_value = false; typename T::value_t min_val(std::numeric_limits::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -429,7 +429,7 @@ namespace LLTrace num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes::fractional_t min_val(std::numeric_limits::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); min_val = llmin(min_val, recording.getPerSec(stat)); @@ -457,7 +457,7 @@ namespace LLTrace bool has_value = false; typename T::value_t max_val(std::numeric_limits::min()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -505,7 +505,7 @@ namespace LLTrace num_periods = llmin(num_periods, getNumRecordedPeriods()); F64 max_val = std::numeric_limits::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); max_val = llmax(max_val, recording.getPerSec(stat)); @@ -533,7 +533,7 @@ namespace LLTrace typename RelatedTypes::fractional_t mean(0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -579,7 +579,7 @@ namespace LLTrace typename RelatedTypes::fractional_t mean = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -600,34 +600,34 @@ namespace LLTrace return typename RelatedTypes::fractional_t(getPeriodMeanPerSec(static_cast&>(stat), num_periods)); } - F64 getPeriodMedian( const StatType& stat, size_t num_periods = std::numeric_limits::max()); + F64 getPeriodMedian( const StatType& stat, size_t num_periods = std::numeric_limits::max()); - template - typename RelatedTypes::fractional_t getPeriodMedianPerSec(const StatType& stat, size_t num_periods = std::numeric_limits::max()) - { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; - num_periods = llmin(num_periods, getNumRecordedPeriods()); - - std::vector ::fractional_t> buf; - for (S32 i = 1; i <= num_periods; i++) - { - Recording& recording = getPrevRecording(i); - if (recording.getDuration() > (F32Seconds)0.f) - { - buf.push_back(recording.getPerSec(stat)); - } - } - std::sort(buf.begin(), buf.end()); - - return typename RelatedTypes::fractional_t((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); - } - - template - typename RelatedTypes::fractional_t getPeriodMedianPerSec(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) - { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; - return typename RelatedTypes::fractional_t(getPeriodMedianPerSec(static_cast&>(stat), num_periods)); - } + template + typename RelatedTypes::fractional_t getPeriodMedianPerSec(const StatType& stat, size_t num_periods = std::numeric_limits::max()) + { + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + num_periods = llmin(num_periods, getNumRecordedPeriods()); + + std::vector ::fractional_t> buf; + for (size_t i = 1; i <= num_periods; i++) + { + Recording& recording = getPrevRecording(i); + if (recording.getDuration() > (F32Seconds)0.f) + { + buf.push_back(recording.getPerSec(stat)); + } + } + std::sort(buf.begin(), buf.end()); + + return typename RelatedTypes::fractional_t((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); + } + + template + typename RelatedTypes::fractional_t getPeriodMedianPerSec(const CountStatHandle& stat, size_t num_periods = std::numeric_limits::max()) + { + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + return typename RelatedTypes::fractional_t(getPeriodMedianPerSec(static_cast&>(stat), num_periods)); + } // // PERIODIC STANDARD DEVIATION @@ -660,6 +660,35 @@ namespace LLTrace /*virtual*/ void handleReset(); /*virtual*/ void handleSplitTo(PeriodicRecording& other); + // helper methods for wraparound ring-buffer arithmetic + inline + size_t wrapi(size_t i) const + { + return i % mRecordingPeriods.size(); + } + + inline + size_t nexti(size_t i, size_t offset=1) const + { + return wrapi(i + offset); + } + + inline + size_t previ(size_t i, size_t offset=1) const + { + auto num_periods = mRecordingPeriods.size(); + // constrain offset + offset = llclamp(offset, 0, num_periods - 1); + // add size() so expression can't go (unsigned) "negative" + return wrapi(i + num_periods - offset); + } + + inline + void inci(size_t& i, size_t offset=1) const + { + i = nexti(i, offset); + } + private: std::vector mRecordingPeriods; const bool mAutoResize; -- cgit v1.2.3 From 230c96b20d8658236e83742169a7744b18a04ed2 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 3 Apr 2023 13:53:18 -0400 Subject: SL-18330: Remove some set-but-unused vars flagged by Xcode 14.3. --- indra/llcommon/llerrorthread.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llerrorthread.cpp b/indra/llcommon/llerrorthread.cpp index f6bc68b5c1..4f8f0a88ad 100644 --- a/indra/llcommon/llerrorthread.cpp +++ b/indra/llcommon/llerrorthread.cpp @@ -108,11 +108,9 @@ void LLErrorThread::run() // application state as APP_STATUS_ERROR. LL_INFOS() << "thread_error - Waiting for an error" << LL_ENDL; - S32 counter = 0; while (! (LLApp::isError() || LLApp::isStopped())) { ms_sleep(10); - counter++; } if (LLApp::isError()) { -- cgit v1.2.3 From ff0edab69a1dc4884b7636c2546327901ce8fdde Mon Sep 17 00:00:00 2001 From: Andrey Lihatskiy Date: Fri, 7 Apr 2023 01:07:59 +0300 Subject: post-merge buildfix --- indra/llcommon/llsdserialize.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index cf6c1ab73a..2f12c6d1ff 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -870,7 +870,7 @@ public: LL_COMMON_API std::string zip_llsd(LLSD& data); -LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize,std::istream& is, S32 size); +LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, size_t& outsize,std::istream& is, S32 size); // returns a pointer to the array or past the array if the deprecated header exists LL_COMMON_API char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size = nullptr); -- cgit v1.2.3 From 22a07b1121bb429e6bfe10b5a5e859bcea1262b6 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 3 May 2023 12:48:22 -0400 Subject: SL-18330: fix strip_deprecated_header(cur_size) param to llssize. --- indra/llcommon/llsdserialize.cpp | 2 +- indra/llcommon/llsdserialize.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index a510b73096..32d0847141 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -2388,7 +2388,7 @@ U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 return result; } -char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size) +char* strip_deprecated_header(char* in, llssize& cur_size, U32* header_size) { const char* deprecated_header = ""; constexpr size_t deprecated_header_size = 17; diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index d33d2b6f34..4f596856de 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -873,5 +873,5 @@ LL_COMMON_API std::string zip_llsd(LLSD& data); LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize,std::istream& is, S32 size); // returns a pointer to the array or past the array if the deprecated header exists -LL_COMMON_API char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size = nullptr); +LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, U32* header_size = nullptr); #endif // LL_LLSDSERIALIZE_H -- cgit v1.2.3 From 971e851157da688b480d6ada188daec532e4b682 Mon Sep 17 00:00:00 2001 From: Andrey Lihatskiy Date: Thu, 4 May 2023 16:44:23 +0300 Subject: SL-19647 OSX buildfix --- indra/llcommon/tests/llsdserialize_test.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 85791e2979..acb2953b5b 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1922,8 +1922,6 @@ namespace tut static void toPythonUsing(const std::string& desc, const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - LLSD cdata(llsd::array(17, 3.14, "This string\n" "has several\n" -- cgit v1.2.3 From 626bbe5032cab1fd2641ad02443e421567e7cce5 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 22 May 2023 19:14:48 -0400 Subject: SL-18837: Fix call to strip_deprecated_header(). Pass llssize instead of S32. --- indra/llcommon/llsdserialize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index ad3f61ce71..c8dbcb2404 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -2193,9 +2193,9 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, const U8* in, S32 size) { U8* result = NULL; - U32 cur_size = 0; + llssize cur_size = 0; z_stream strm; - + constexpr U32 CHUNK = 1024 * 512; static thread_local std::unique_ptr out; -- cgit v1.2.3 From 55460b49585ceea5d6388204c7f3bd74ed4f4827 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 12 May 2023 17:29:41 +0100 Subject: SL-19711 - viewer can't parse environments. Fixed max parsing bug (cherry picked from commit eb0516b9940f200b32349d611f38f1ccee48005d) --- indra/llcommon/llsdserialize.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index e7482b601d..56cc141f0d 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -125,14 +125,20 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ bool fail_if_not_legacy = false; - /* - * Get the first line before anything. Don't read more than max_bytes: - * this get() overload reads no more than (count-1) bytes into the - * specified buffer. In the usual case when max_bytes exceeds - * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. - */ - str.get(hdr_buf, llmin(max_bytes+1, sizeof(hdr_buf)-1), '\n'); + /* + * Get the first line before anything. Don't read more than max_bytes: + * this get() overload reads no more than (count-1) bytes into the + * specified buffer. In the usual case when max_bytes exceeds + * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. + */ + llssize max_hdr_read = MAX_HDR_LEN; + if (max_bytes != LLSDSerialize::SIZE_UNLIMITED) + { + max_hdr_read = llmin(max_bytes + 1, max_hdr_read); + } + str.get(hdr_buf, max_hdr_read, '\n'); auto inbuf = str.gcount(); + // https://en.cppreference.com/w/cpp/io/basic_istream/get // When the get() above sees the specified delimiter '\n', it stops there // without pulling it from the stream. If it turns out that the stream -- cgit v1.2.3 From 3b13c0ca15e2bc413731b0be07db6446bb32df1b Mon Sep 17 00:00:00 2001 From: Andrey Lihatskiy Date: Fri, 26 May 2023 19:06:07 +0300 Subject: DRTVWR-582 xcode-14.3 merge follow up --- indra/llcommon/llsdserialize.cpp | 2 +- indra/llcommon/llsdserialize.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index a904ce700e..3db456ddb3 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -2443,7 +2443,7 @@ U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size return result; } -char* strip_deprecated_header(char* in, llssize& cur_size, U32* header_size) +char* strip_deprecated_header(char* in, llssize& cur_size, llssize* header_size) { const char* deprecated_header = ""; constexpr size_t deprecated_header_size = 17; diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 5ddf0ff552..676b7bfd6a 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -873,5 +873,5 @@ LL_COMMON_API std::string zip_llsd(LLSD& data); LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, size_t& outsize,std::istream& is, S32 size); // returns a pointer to the array or past the array if the deprecated header exists -LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, U32* header_size = nullptr); +LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, llssize* header_size = nullptr); #endif // LL_LLSDSERIALIZE_H -- cgit v1.2.3 From b11b53b72cebe85b0fd2a6c8e18c0aca3cf16430 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 31 May 2023 16:33:01 -0400 Subject: SL-18330: Change to strip_deprecated_header(size_t* header_size) from U32*. --- indra/llcommon/llsdserialize.cpp | 2 +- indra/llcommon/llsdserialize.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index c8dbcb2404..9181600f23 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -2388,7 +2388,7 @@ U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size return result; } -char* strip_deprecated_header(char* in, llssize& cur_size, U32* header_size) +char* strip_deprecated_header(char* in, llssize& cur_size, size_t* header_size) { const char* deprecated_header = ""; constexpr size_t deprecated_header_size = 17; diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 5ddf0ff552..0957ba9714 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -873,5 +873,5 @@ LL_COMMON_API std::string zip_llsd(LLSD& data); LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, size_t& outsize,std::istream& is, S32 size); // returns a pointer to the array or past the array if the deprecated header exists -LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, U32* header_size = nullptr); +LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, size_t* header_size = nullptr); #endif // LL_LLSDSERIALIZE_H -- cgit v1.2.3 From ed92f1d8034dcaa84f808b3267c772202e7ce113 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 31 May 2023 16:34:04 -0400 Subject: SL-18330: Consistently use new standalone Python llsd package. --- indra/llcommon/tests/llleap_test.cpp | 2 +- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 7ee36a9ea6..60005fc6a9 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -109,7 +109,7 @@ namespace tut "import os\n" "import sys\n" "\n" - "from llbase import llsd\n" + "import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..be7ec12094 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1706,7 +1706,7 @@ namespace tut // scanner. import_llsd("import os.path\n" "import sys\n" - "from llbase import llsd\n") + "import llsd\n") {} ~TestPythonCompatible() {} -- cgit v1.2.3 From 364516e8bc19f6435fd42893809080d4f64a3232 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 1 Jun 2023 11:34:48 -0400 Subject: DRTVWR-575: GCC prefers std::make_signed for llssize. --- indra/llcommon/stdtypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/stdtypes.h b/indra/llcommon/stdtypes.h index da8512169c..0b43d7ad4b 100644 --- a/indra/llcommon/stdtypes.h +++ b/indra/llcommon/stdtypes.h @@ -41,7 +41,7 @@ typedef unsigned int U32; // to express an index that might go negative // (ssize_t is provided by SOME compilers, don't collide) -typedef typename std::make_signed::type llssize; +typedef typename std::make_signed::type llssize; #if LL_WINDOWS // https://docs.microsoft.com/en-us/cpp/build/reference/zc-wchar-t-wchar-t-is-native-type -- cgit v1.2.3 From 834a8eeddeda035551ed4071273c26e806946147 Mon Sep 17 00:00:00 2001 From: Alexander Gavriliuk Date: Tue, 11 Jul 2023 21:25:10 +0200 Subject: SL-19728 Objects that cannot be clicked or cammed unless in edit mode --- indra/llcommon/indra_constants.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/indra_constants.h b/indra/llcommon/indra_constants.h index 10b98f49aa..679f79039b 100644 --- a/indra/llcommon/indra_constants.h +++ b/indra/llcommon/indra_constants.h @@ -345,6 +345,7 @@ const U8 CLICK_ACTION_PLAY = 5; const U8 CLICK_ACTION_OPEN_MEDIA = 6; const U8 CLICK_ACTION_ZOOM = 7; const U8 CLICK_ACTION_DISABLED = 8; +const U8 CLICK_ACTION_IGNORE = 9; // DO NOT CHANGE THE SEQUENCE OF THIS LIST!! -- cgit v1.2.3