diff options
author | Andrey Kleshchev <andreykproductengine@lindenlab.com> | 2023-08-23 22:40:17 +0300 |
---|---|---|
committer | Andrey Kleshchev <andreykproductengine@lindenlab.com> | 2023-08-23 22:40:17 +0300 |
commit | 3d73a10da3e051b232b4ddc89c1463a8b097ca7f (patch) | |
tree | 8b32d955d0628ca1138301177e1413cc419229fd /indra/llcommon | |
parent | 4b5b048cb6280cbd9fd303acd96fce73461d1fb4 (diff) | |
parent | d454512050e636a19e4b7545515dea4f4b1bbf0d (diff) |
Merge branch main (DRTVWR-582) into DRTVWR-567
# Conflicts:
# indra/newview/app_settings/settings.xml
# indra/newview/llinventoryfunctions.cpp
# indra/newview/llinventoryfunctions.h
# indra/newview/llinventorymodel.cpp
# indra/newview/llinventoryobserver.cpp
# indra/newview/llinventoryobserver.h
# indra/newview/skins/default/xui/ja/floater_inventory_item_properties.xml
Diffstat (limited to 'indra/llcommon')
-rw-r--r-- | indra/llcommon/hbxxh.h | 18 | ||||
-rw-r--r-- | indra/llcommon/indra_constants.h | 1 | ||||
-rw-r--r-- | indra/llcommon/lldefs.h | 48 | ||||
-rw-r--r-- | indra/llcommon/llleap.cpp | 31 | ||||
-rw-r--r-- | indra/llcommon/llmd5.cpp | 32 | ||||
-rw-r--r-- | indra/llcommon/llmd5.h | 23 | ||||
-rw-r--r-- | indra/llcommon/llsdserialize.cpp | 197 | ||||
-rw-r--r-- | indra/llcommon/llsdserialize.h | 2 | ||||
-rw-r--r-- | indra/llcommon/llstreamtools.cpp | 26 | ||||
-rw-r--r-- | indra/llcommon/llstreamtools.h | 27 | ||||
-rw-r--r-- | indra/llcommon/lltracerecording.cpp | 124 | ||||
-rw-r--r-- | indra/llcommon/lltracerecording.h | 172 | ||||
-rw-r--r-- | indra/llcommon/lluuid.cpp | 2 | ||||
-rw-r--r-- | indra/llcommon/lluuid.h | 50 | ||||
-rw-r--r-- | indra/llcommon/stdtypes.h | 2 | ||||
-rw-r--r-- | indra/llcommon/tests/llleap_test.cpp | 27 | ||||
-rw-r--r-- | indra/llcommon/tests/llsdserialize_test.cpp | 610 |
17 files changed, 890 insertions, 502 deletions
diff --git a/indra/llcommon/hbxxh.h b/indra/llcommon/hbxxh.h index 236716722a..9c0e9cf172 100644 --- a/indra/llcommon/hbxxh.h +++ b/indra/llcommon/hbxxh.h @@ -96,6 +96,15 @@ public: } } + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH64(const HBXXH64&) noexcept = delete; + HBXXH64& operator=(const HBXXH64&) noexcept = delete; + ~HBXXH64(); void update(const void* buffer, size_t len); @@ -199,6 +208,15 @@ public: } } + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH128(const HBXXH128&) noexcept = delete; + HBXXH128& operator=(const HBXXH128&) noexcept = delete; + ~HBXXH128(); void update(const void* buffer, size_t len); diff --git a/indra/llcommon/indra_constants.h b/indra/llcommon/indra_constants.h index 10b98f49aa..679f79039b 100644 --- a/indra/llcommon/indra_constants.h +++ b/indra/llcommon/indra_constants.h @@ -345,6 +345,7 @@ const U8 CLICK_ACTION_PLAY = 5; const U8 CLICK_ACTION_OPEN_MEDIA = 6; const U8 CLICK_ACTION_ZOOM = 7; const U8 CLICK_ACTION_DISABLED = 8; +const U8 CLICK_ACTION_IGNORE = 9; // DO NOT CHANGE THE SEQUENCE OF THIS LIST!! diff --git a/indra/llcommon/lldefs.h b/indra/llcommon/lldefs.h index 5c46f6a796..4e25001fff 100644 --- a/indra/llcommon/lldefs.h +++ b/indra/llcommon/lldefs.h @@ -167,48 +167,34 @@ const U32 MAXADDRSTR = 17; // 123.567.901.345 = 15 chars + \0 + 1 for good luc // // defined for U16, U32, U64, S16, S32, S64, : // llclampb(a) // clamps a to [0 .. 255] -// - -template <typename T1, typename T2> -inline auto llmax(T1 d1, T2 d2) -{ - return (d1 > d2) ? d1 : d2; -} - -template <typename T1, typename T2, typename T3> -inline auto llmax(T1 d1, T2 d2, T3 d3) -{ - auto r = llmax(d1,d2); - return llmax(r, d3); -} +// -template <typename T1, typename T2, typename T3, typename T4> -inline auto llmax(T1 d1, T2 d2, T3 d3, T4 d4) +// recursion tail +template <typename T> +inline auto llmax(T data) { - auto r1 = llmax(d1,d2); - auto r2 = llmax(d3,d4); - return llmax(r1, r2); + return data; } -template <typename T1, typename T2> -inline auto llmin(T1 d1, T2 d2) +template <typename T0, typename T1, typename... Ts> +inline auto llmax(T0 d0, T1 d1, Ts... rest) { - return (d1 < d2) ? d1 : d2; + auto maxrest = llmax(d1, rest...); + return (d0 > maxrest)? d0 : maxrest; } -template <typename T1, typename T2, typename T3> -inline auto llmin(T1 d1, T2 d2, T3 d3) +// recursion tail +template <typename T> +inline auto llmin(T data) { - auto r = llmin(d1,d2); - return (r < d3 ? r : d3); + return data; } -template <typename T1, typename T2, typename T3, typename T4> -inline auto llmin(T1 d1, T2 d2, T3 d3, T4 d4) +template <typename T0, typename T1, typename... Ts> +inline auto llmin(T0 d0, T1 d1, Ts... rest) { - auto r1 = llmin(d1,d2); - auto r2 = llmin(d3,d4); - return llmin(r1, r2); + auto minrest = llmin(d1, rest...); + return (d0 < minrest) ? d0 : minrest; } template <typename A, typename MIN, typename MAX> diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index c87c0758fe..259f5bc505 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -204,30 +204,35 @@ public: LLSD packet(LLSDMap("pump", pump)("data", data)); std::ostringstream buffer; - buffer << LLSDNotationStreamer(packet); + // SL-18330: for large data blocks, it's much faster to parse binary + // LLSD than notation LLSD. Use serialize(LLSD_BINARY) rather than + // directly calling LLSDBinaryFormatter because, unlike the latter, + // serialize() prepends the relevant header, needed by a general- + // purpose LLSD parser to distinguish binary from notation. + LLSDSerialize::serialize(packet, buffer, LLSDSerialize::LLSD_BINARY, + LLSDFormatter::OPTIONS_NONE); /*==========================================================================*| // DEBUGGING ONLY: don't copy str() if we can avoid it. std::string strdata(buffer.str()); if (std::size_t(buffer.tellp()) != strdata.length()) { - LL_ERRS("LLLeap") << "tellp() -> " << buffer.tellp() << " != " + LL_ERRS("LLLeap") << "tellp() -> " << static_cast<U64>(buffer.tellp()) << " != " << "str().length() -> " << strdata.length() << LL_ENDL; } // DEBUGGING ONLY: reading back is terribly inefficient. std::istringstream readback(strdata); LLSD echo; - LLPointer<LLSDParser> parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(readback, echo, strdata.length())); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(echo, readback, strdata.length())); + if (! parse_status) { - LL_ERRS("LLLeap") << "LLSDNotationParser() cannot parse output of " - << "LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() cannot parse output of " + << "LLSDSerialize::serialize(LLSD_BINARY)" << LL_ENDL; } if (! llsd_equals(echo, packet)) { - LL_ERRS("LLLeap") << "LLSDNotationParser() produced different LLSD " - << "than passed to LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() returned different LLSD " + << "than passed to LLSDSerialize::serialize()" << LL_ENDL; } |*==========================================================================*/ @@ -314,9 +319,17 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; +#if 1 + // specifically require notation LLSD from child LLPointer<LLSDParser> parser(new LLSDNotationParser()); S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); if (parse_status == LLSDParser::PARSE_FAILURE) +#else + // SL-18330: accept any valid LLSD serialization format from child + // Unfortunately this runs into trouble we have not yet debugged. + bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); + if (! parse_status) +#endif { bad_protocol("unparseable LLSD data"); } diff --git a/indra/llcommon/llmd5.cpp b/indra/llcommon/llmd5.cpp index 9b2a2bab60..0abe817f1d 100644 --- a/indra/llcommon/llmd5.cpp +++ b/indra/llcommon/llmd5.cpp @@ -96,7 +96,7 @@ LLMD5::LLMD5() // operation, processing another message block, and updating the // context. -void LLMD5::update (const uint1 *input, const size_t input_length) { +void LLMD5::update (const uint8_t *input, const size_t input_length) { size_t input_index, buffer_index; size_t buffer_space; // how much space is left in buffer @@ -189,7 +189,7 @@ void LLMD5::finalize (){ unsigned char bits[8]; /* Flawfinder: ignore */ size_t index, padLen; - static uint1 PADDING[64]={ + static uint8_t PADDING[64]={ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 @@ -201,8 +201,8 @@ void LLMD5::finalize (){ } // Save number of bits. - // Treat count, a uint64_t, as uint4[2]. - encode (bits, reinterpret_cast<uint4*>(&count), 8); + // Treat count, a uint64_t, as uint32_t[2]. + encode (bits, reinterpret_cast<uint32_t*>(&count), 8); // Pad out to 56 mod 64. index = size_t((count >> 3) & 0x3f); @@ -412,7 +412,7 @@ Rotation is separate from addition to prevent recomputation. // LLMD5 basic transformation. Transforms state based on block. void LLMD5::transform (const U8 block[64]){ - uint4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; + uint32_t a = state[0], b = state[1], c = state[2], d = state[3], x[16]; decode (x, block, 64); @@ -496,38 +496,38 @@ void LLMD5::transform (const U8 block[64]){ state[3] += d; // Zeroize sensitive information. - memset ( (uint1 *) x, 0, sizeof(x)); + memset ( (uint8_t *) x, 0, sizeof(x)); } -// Encodes input (UINT4) into output (unsigned char). Assumes len is +// Encodes input (uint32_t) into output (unsigned char). Assumes len is // a multiple of 4. -void LLMD5::encode (uint1 *output, const uint4 *input, const size_t len) { +void LLMD5::encode (uint8_t *output, const uint32_t *input, const size_t len) { size_t i, j; for (i = 0, j = 0; j < len; i++, j += 4) { - output[j] = (uint1) (input[i] & 0xff); - output[j+1] = (uint1) ((input[i] >> 8) & 0xff); - output[j+2] = (uint1) ((input[i] >> 16) & 0xff); - output[j+3] = (uint1) ((input[i] >> 24) & 0xff); + output[j] = (uint8_t) (input[i] & 0xff); + output[j+1] = (uint8_t) ((input[i] >> 8) & 0xff); + output[j+2] = (uint8_t) ((input[i] >> 16) & 0xff); + output[j+3] = (uint8_t) ((input[i] >> 24) & 0xff); } } -// Decodes input (unsigned char) into output (UINT4). Assumes len is +// Decodes input (unsigned char) into output (uint32_t). Assumes len is // a multiple of 4. -void LLMD5::decode (uint4 *output, const uint1 *input, const size_t len){ +void LLMD5::decode (uint32_t *output, const uint8_t *input, const size_t len){ size_t i, j; for (i = 0, j = 0; j < len; i++, j += 4) - output[i] = ((uint4)input[j]) | (((uint4)input[j+1]) << 8) | - (((uint4)input[j+2]) << 16) | (((uint4)input[j+3]) << 24); + output[i] = ((uint32_t)input[j]) | (((uint32_t)input[j+1]) << 8) | + (((uint32_t)input[j+2]) << 16) | (((uint32_t)input[j+3]) << 24); } diff --git a/indra/llcommon/llmd5.h b/indra/llcommon/llmd5.h index 8530dc0389..7d6373c20c 100644 --- a/indra/llcommon/llmd5.h +++ b/indra/llcommon/llmd5.h @@ -67,6 +67,8 @@ documentation and/or software. */ +#include <cstdint> // uint32_t et al. + // use for the raw digest output const int MD5RAW_BYTES = 16; @@ -75,18 +77,13 @@ const int MD5HEX_STR_SIZE = 33; // char hex[MD5HEX_STR_SIZE]; with null const int MD5HEX_STR_BYTES = 32; // message system fixed size class LL_COMMON_API LLMD5 { -// first, some types: - typedef unsigned int uint4; // assumes integer is 4 words long - typedef unsigned short int uint2; // assumes short integer is 2 words long - typedef unsigned char uint1; // assumes char is 1 word long - // how many bytes to grab at a time when checking files static const int BLOCK_LEN; public: // methods for controlled operation: LLMD5 (); // simple initializer - void update (const uint1 *input, const size_t input_length); + void update (const uint8_t *input, const size_t input_length); void update (std::istream& stream); void update (FILE *file); void update (const std::string& str); @@ -109,19 +106,19 @@ private: // next, the private data: - uint4 state[4]; + uint32_t state[4]; uint64_t count; // number of *bits*, mod 2^64 - uint1 buffer[64]; // input buffer - uint1 digest[16]; - uint1 finalized; + uint8_t buffer[64]; // input buffer + uint8_t digest[16]; + uint8_t finalized; // last, the private methods, mostly static: void init (); // called by all constructors - void transform (const uint1 *buffer); // does the real update work. Note + void transform (const uint8_t *buffer); // does the real update work. Note // that length is implied to be 64. - static void encode (uint1 *dest, const uint4 *src, const size_t length); - static void decode (uint4 *dest, const uint1 *src, const size_t length); + static void encode (uint8_t *dest, const uint32_t *src, const size_t length); + static void decode (uint32_t *dest, const uint8_t *src, const size_t length); }; diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index b7e316da10..3db456ddb3 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -48,6 +48,7 @@ #endif #include "lldate.h" +#include "llmemorystream.h" #include "llsd.h" #include "llstring.h" #include "lluri.h" @@ -64,6 +65,23 @@ const std::string LLSD_NOTATION_HEADER("llsd/notation"); #define windowBits 15 #define ENABLE_ZLIB_GZIP 32 +// If we published this in llsdserialize.h, we could use it in the +// implementation of LLSDOStreamer's operator<<(). +template <class Formatter> +void format_using(const LLSD& data, std::ostream& ostr, + LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY) +{ + LLPointer<Formatter> f{ new Formatter }; + f->format(data, ostr, options); +} + +template <class Parser> +S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1) +{ + LLPointer<Parser> p{ new Parser }; + return p->parse(istr, data, max_bytes, max_depth); +} + /** * LLSDSerialize */ @@ -86,10 +104,10 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize f = new LLSDXMLFormatter; break; - case LLSD_NOTATION: - str << "<? " << LLSD_NOTATION_HEADER << " ?>\n"; - f = new LLSDNotationFormatter; - break; + case LLSD_NOTATION: + str << "<? " << LLSD_NOTATION_HEADER << " ?>\n"; + f = new LLSDNotationFormatter; + break; default: LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL; @@ -104,18 +122,37 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize // static bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) { - LLPointer<LLSDParser> p = NULL; char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ - int i; - int inbuf = 0; - bool legacy_no_header = false; bool fail_if_not_legacy = false; - std::string header; - /* - * Get the first line before anything. - */ - str.get(hdr_buf, MAX_HDR_LEN, '\n'); + /* + * Get the first line before anything. Don't read more than max_bytes: + * this get() overload reads no more than (count-1) bytes into the + * specified buffer. In the usual case when max_bytes exceeds + * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. + */ + llssize max_hdr_read = MAX_HDR_LEN; + if (max_bytes != LLSDSerialize::SIZE_UNLIMITED) + { + max_hdr_read = llmin(max_bytes + 1, max_hdr_read); + } + str.get(hdr_buf, max_hdr_read, '\n'); + auto inbuf = str.gcount(); + + // https://en.cppreference.com/w/cpp/io/basic_istream/get + // When the get() above sees the specified delimiter '\n', it stops there + // without pulling it from the stream. If it turns out that the stream + // does NOT contain a header, and the content includes meaningful '\n', + // it's important to pull that into hdr_buf too. + if (inbuf < max_bytes && str.get(hdr_buf[inbuf])) + { + // got the delimiting '\n' + ++inbuf; + // None of the following requires that hdr_buf contain a final '\0' + // byte. We could store one if needed, since even the incremented + // inbuf won't exceed sizeof(hdr_buf)-1, but there's no need. + } + std::string header{ hdr_buf, static_cast<std::string::size_type>(inbuf) }; if (str.fail()) { str.clear(); @@ -123,79 +160,97 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) } if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ + { // Create a LLSD XML parser, and parse the first chunk read above. + LLSDXMLParser x; + x.parsePart(hdr_buf, inbuf); // Parse the first part that was already read + auto parsed = x.parse(str, sd, max_bytes - inbuf); // Parse the rest of it + // Formally we should probably check (parsed != PARSE_FAILURE && + // parsed > 0), but since PARSE_FAILURE is -1, this suffices. + return (parsed > 0); + } + + if (fail_if_not_legacy) { - legacy_no_header = true; - inbuf = (int)str.gcount(); + LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; + return false; } - else + + /* + * Remove the newline chars + */ + std::string::size_type lastchar = header.find_last_not_of("\r\n"); + if (lastchar != std::string::npos) { - if (fail_if_not_legacy) - goto fail; - /* - * Remove the newline chars - */ - for (i = 0; i < MAX_HDR_LEN; i++) - { - if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' || - hdr_buf[i] == '\n') - { - hdr_buf[i] = 0; - break; - } - } - header = hdr_buf; + // It's important that find_last_not_of() returns size_type, which is + // why lastchar explicitly declares the type above. erase(size_type) + // erases from that offset to the end of the string, whereas + // erase(iterator) erases only a single character. + header.erase(lastchar+1); + } - std::string::size_type start = std::string::npos; - std::string::size_type end = std::string::npos; - start = header.find_first_not_of("<? "); - if (start != std::string::npos) + // trim off the <? ... ?> header syntax + auto start = header.find_first_not_of("<? "); + if (start != std::string::npos) + { + auto end = header.find_first_of(" ?", start); + if (end != std::string::npos) { - end = header.find_first_of(" ?", start); + header = header.substr(start, end - start); + ws(str); } - if ((start == std::string::npos) || (end == std::string::npos)) - goto fail; - - header = header.substr(start, end - start); - ws(str); } /* * Create the parser as appropriate */ - if (legacy_no_header) - { // Create a LLSD XML parser, and parse the first chunk read above - LLSDXMLParser* x = new LLSDXMLParser(); - x->parsePart(hdr_buf, inbuf); // Parse the first part that was already read - x->parseLines(str, sd); // Parse the rest of it - delete x; - return true; - } - - if (header == LLSD_BINARY_HEADER) + if (0 == LLStringUtil::compareInsensitive(header, LLSD_BINARY_HEADER)) { - p = new LLSDBinaryParser; + return (parse_using<LLSDBinaryParser>(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_XML_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_XML_HEADER)) { - p = new LLSDXMLParser; + return (parse_using<LLSDXMLParser>(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_NOTATION_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_NOTATION_HEADER)) { - p = new LLSDNotationParser; + return (parse_using<LLSDNotationParser>(str, sd, max_bytes-inbuf) > 0); } - else + else // no header we recognize { - LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL; - } - - if (p.notNull()) - { - p->parse(str, sd, max_bytes); - return true; + LLPointer<LLSDParser> p; + if (inbuf && hdr_buf[0] == '<') + { + // looks like XML + LL_DEBUGS() << "deserialize request with no header, assuming XML" << LL_ENDL; + p = new LLSDXMLParser; + } + else + { + // assume notation + LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL; + p = new LLSDNotationParser; + } + // Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that + // data to whatever remains in 'str'. + LLMemoryStreamBuf already(reinterpret_cast<const U8*>(hdr_buf), inbuf); + cat_streambuf prebuff(&already, str.rdbuf()); + std::istream prepend(&prebuff); +#if 1 + return (p->parse(prepend, sd, max_bytes) > 0); +#else + // debugging the reconstituted 'prepend' stream + // allocate a buffer that we hope is big enough for the whole thing + std::vector<char> wholemsg((max_bytes == size_t(SIZE_UNLIMITED))? 1024 : max_bytes); + prepend.read(wholemsg.data(), std::min(max_bytes, wholemsg.size())); + LLMemoryStream replay(reinterpret_cast<const U8*>(wholemsg.data()), prepend.gcount()); + auto success{ p->parse(replay, sd, prepend.gcount()) > 0 }; + { + LL_DEBUGS() << (success? "parsed: $$" : "failed: '") + << std::string(wholemsg.data(), llmin(prepend.gcount(), 100)) << "$$" + << LL_ENDL; + } + return success; +#endif } - -fail: - LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; - return false; } /** @@ -2193,9 +2248,9 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, const U8* in, S32 size) { U8* result = NULL; - U32 cur_size = 0; + llssize cur_size = 0; z_stream strm; - + constexpr U32 CHUNK = 1024 * 512; static thread_local std::unique_ptr<U8[]> out; @@ -2388,7 +2443,7 @@ U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size return result; } -char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size) +char* strip_deprecated_header(char* in, llssize& cur_size, llssize* header_size) { const char* deprecated_header = "<? LLSD/Binary ?>"; constexpr size_t deprecated_header_size = 17; diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 2f12c6d1ff..676b7bfd6a 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -873,5 +873,5 @@ LL_COMMON_API std::string zip_llsd(LLSD& data); LL_COMMON_API U8* unzip_llsdNavMesh( bool& valid, size_t& outsize,std::istream& is, S32 size); // returns a pointer to the array or past the array if the deprecated header exists -LL_COMMON_API char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size = nullptr); +LL_COMMON_API char* strip_deprecated_header(char* in, llssize& cur_size, llssize* header_size = nullptr); #endif // LL_LLSDSERIALIZE_H diff --git a/indra/llcommon/llstreamtools.cpp b/indra/llcommon/llstreamtools.cpp index 1ff15fcf89..bc32b6fd9e 100644 --- a/indra/llcommon/llstreamtools.cpp +++ b/indra/llcommon/llstreamtools.cpp @@ -513,3 +513,29 @@ std::istream& operator>>(std::istream& str, const char *tocheck) } return str; } + +int cat_streambuf::underflow() +{ + if (gptr() == egptr()) + { + // here because our buffer is empty + std::streamsize size = 0; + // Until we've run out of mInputs, try reading the first of them + // into mBuffer. If that fetches some characters, break the loop. + while (! mInputs.empty() + && ! (size = mInputs.front()->sgetn(mBuffer.data(), mBuffer.size()))) + { + // We tried to read mInputs.front() but got zero characters. + // Discard the first streambuf and try the next one. + mInputs.pop_front(); + } + // Either we ran out of mInputs or we succeeded in reading some + // characters, that is, size != 0. Tell base class what we have. + setg(mBuffer.data(), mBuffer.data(), mBuffer.data() + size); + } + // If we fell out of the above loop with mBuffer still empty, return + // eof(), otherwise return the next character. + return (gptr() == egptr()) + ? std::char_traits<char>::eof() + : std::char_traits<char>::to_int_type(*gptr()); +} diff --git a/indra/llcommon/llstreamtools.h b/indra/llcommon/llstreamtools.h index 1b04bf91d7..bb7bc20327 100644 --- a/indra/llcommon/llstreamtools.h +++ b/indra/llcommon/llstreamtools.h @@ -27,8 +27,10 @@ #ifndef LL_STREAM_TOOLS_H #define LL_STREAM_TOOLS_H +#include <deque> #include <iostream> #include <string> +#include <vector> // unless specifed otherwise these all return input_stream.good() @@ -113,6 +115,27 @@ LL_COMMON_API std::streamsize fullread( LL_COMMON_API std::istream& operator>>(std::istream& str, const char *tocheck); -#endif - +/** + * cat_streambuf is a std::streambuf subclass that accepts a variadic number + * of std::streambuf* (e.g. some_istream.rdbuf()) and virtually concatenates + * their contents. + */ +// derived from https://stackoverflow.com/a/49441066/5533635 +class cat_streambuf: public std::streambuf +{ +private: + std::deque<std::streambuf*> mInputs; + std::vector<char> mBuffer; + +public: + // only valid for std::streambuf* arguments + template <typename... Inputs> + cat_streambuf(Inputs... inputs): + mInputs{inputs...}, + mBuffer(1024) + {} + + int underflow() override; +}; +#endif diff --git a/indra/llcommon/lltracerecording.cpp b/indra/llcommon/lltracerecording.cpp index a8dcc5226a..bb3d667a42 100644 --- a/indra/llcommon/lltracerecording.cpp +++ b/indra/llcommon/lltracerecording.cpp @@ -577,10 +577,12 @@ S32 Recording::getSampleCount( const StatType<EventAccumulator>& stat ) // PeriodicRecording /////////////////////////////////////////////////////////////////////// -PeriodicRecording::PeriodicRecording( S32 num_periods, EPlayState state) +PeriodicRecording::PeriodicRecording( size_t num_periods, EPlayState state) : mAutoResize(num_periods == 0), mCurPeriod(0), mNumRecordedPeriods(0), + // This guarantee that mRecordingPeriods cannot be empty is essential for + // code in several methods. mRecordingPeriods(num_periods ? num_periods : 1) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; @@ -596,18 +598,19 @@ PeriodicRecording::~PeriodicRecording() void PeriodicRecording::nextPeriod() { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; if (mAutoResize) { mRecordingPeriods.push_back(Recording()); } Recording& old_recording = getCurRecording(); - mCurPeriod = (mCurPeriod + 1) % mRecordingPeriods.size(); + inci(mCurPeriod); old_recording.splitTo(getCurRecording()); - mNumRecordedPeriods = mRecordingPeriods.empty()? 0 : - llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + 1); + // Since mRecordingPeriods always has at least one entry, we can always + // safely subtract 1 from its size(). + mNumRecordedPeriods = llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + 1); } void PeriodicRecording::appendRecording(Recording& recording) @@ -620,31 +623,29 @@ void PeriodicRecording::appendRecording(Recording& recording) void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; if (other.mRecordingPeriods.empty()) return; getCurRecording().update(); other.getCurRecording().update(); - - const auto other_recording_slots = other.mRecordingPeriods.size(); + const auto other_num_recordings = other.getNumRecordedPeriods(); const auto other_current_recording_index = other.mCurPeriod; - const auto other_oldest_recording_index = (other_current_recording_index + other_recording_slots - other_num_recordings) % other_recording_slots; + const auto other_oldest_recording_index = other.previ(other_current_recording_index, other_num_recordings); // append first recording into our current slot getCurRecording().appendRecording(other.mRecordingPeriods[other_oldest_recording_index]); // from now on, add new recordings for everything after the first - auto other_index = (other_oldest_recording_index + 1) % other_recording_slots; + auto other_index = other.nexti(other_oldest_recording_index); if (mAutoResize) { // push back recordings for everything in the middle - auto other_index = (other_oldest_recording_index + 1) % other_recording_slots; while (other_index != other_current_recording_index) { mRecordingPeriods.push_back(other.mRecordingPeriods[other_index]); - other_index = (other_index + 1) % other_recording_slots; + other.inci(other_index); } // add final recording, if it wasn't already added as the first @@ -653,36 +654,25 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) mRecordingPeriods.push_back(other.mRecordingPeriods[other_current_recording_index]); } - mCurPeriod = mRecordingPeriods.empty()? 0 : mRecordingPeriods.size() - 1; + // mRecordingPeriods is never empty() + mCurPeriod = mRecordingPeriods.size() - 1; mNumRecordedPeriods = mCurPeriod; } else { - S32 num_to_copy = llmin((S32)mRecordingPeriods.size(), (S32)other_num_recordings); - - std::vector<Recording>::iterator src_it = other.mRecordingPeriods.begin() + other_index ; - std::vector<Recording>::iterator dest_it = mRecordingPeriods.begin() + mCurPeriod; - + auto num_to_copy = llmin(mRecordingPeriods.size(), other_num_recordings); // already consumed the first recording from other, so start counting at 1 - for(S32 i = 1; i < num_to_copy; i++) + for (size_t n = 1, srci = other_index, dsti = mCurPeriod; + n < num_to_copy; + ++n, other.inci(srci), inci(dsti)) { - *dest_it = *src_it; - - if (++src_it == other.mRecordingPeriods.end()) - { - src_it = other.mRecordingPeriods.begin(); - } - - if (++dest_it == mRecordingPeriods.end()) - { - dest_it = mRecordingPeriods.begin(); - } + mRecordingPeriods[dsti] = other.mRecordingPeriods[srci]; } - + // want argument to % to be positive, otherwise result could be negative and thus out of bounds llassert(num_to_copy >= 1); // advance to last recording period copied, and make that our current period - mCurPeriod = (mCurPeriod + num_to_copy - 1) % mRecordingPeriods.size(); + inci(mCurPeriod, num_to_copy - 1); mNumRecordedPeriods = llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + num_to_copy - 1); } @@ -694,13 +684,11 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) F64Seconds PeriodicRecording::getDuration() const { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; F64Seconds duration; - auto num_periods = mRecordingPeriods.size(); - for (size_t i = 1; i <= num_periods; i++) + for (size_t n = 0; n < mRecordingPeriods.size(); ++n) { - auto index = (mCurPeriod + num_periods - i) % num_periods; - duration += mRecordingPeriods[index].getDuration(); + duration += mRecordingPeriods[nexti(mCurPeriod, n)].getDuration(); } return duration; } @@ -737,16 +725,14 @@ const Recording& PeriodicRecording::getCurRecording() const Recording& PeriodicRecording::getPrevRecording( size_t offset ) { - auto num_periods = mRecordingPeriods.size(); - offset = llclamp(offset, 0, num_periods - 1); - return mRecordingPeriods[(mCurPeriod + num_periods - offset) % num_periods]; + // reuse const implementation, but return non-const reference + return const_cast<Recording&>( + const_cast<const PeriodicRecording*>(this)->getPrevRecording(offset)); } const Recording& PeriodicRecording::getPrevRecording( size_t offset ) const { - auto num_periods = mRecordingPeriods.size(); - offset = llclamp(offset, 0, num_periods - 1); - return mRecordingPeriods[(mCurPeriod + num_periods - offset) % num_periods]; + return mRecordingPeriods[previ(mCurPeriod, offset)]; } void PeriodicRecording::handleStart() @@ -789,14 +775,14 @@ void PeriodicRecording::handleSplitTo(PeriodicRecording& other) getCurRecording().splitTo(other.getCurRecording()); } -F64 PeriodicRecording::getPeriodMin( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMin( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 min_val = std::numeric_limits<F64>::max(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -811,14 +797,14 @@ F64 PeriodicRecording::getPeriodMin( const StatType<EventAccumulator>& stat, siz : NaN; } -F64 PeriodicRecording::getPeriodMax( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMax( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 max_val = std::numeric_limits<F64>::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -834,7 +820,7 @@ F64 PeriodicRecording::getPeriodMax( const StatType<EventAccumulator>& stat, siz } // calculates means using aggregates per period -F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -842,7 +828,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, si F64 mean = 0; S32 valid_period_count = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -857,7 +843,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, si : NaN; } -F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -866,7 +852,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulat F64 sum_of_squares = 0; S32 valid_period_count = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -882,14 +868,14 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulat : NaN; } -F64 PeriodicRecording::getPeriodMin( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMin( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 min_val = std::numeric_limits<F64>::max(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -904,14 +890,14 @@ F64 PeriodicRecording::getPeriodMin( const StatType<SampleAccumulator>& stat, si : NaN; } -F64 PeriodicRecording::getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/) +F64 PeriodicRecording::getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 max_val = std::numeric_limits<F64>::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -927,7 +913,7 @@ F64 PeriodicRecording::getPeriodMax(const StatType<SampleAccumulator>& stat, siz } -F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -935,7 +921,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, s S32 valid_period_count = 0; F64 mean = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -950,13 +936,13 @@ F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, s : NaN; } -F64 PeriodicRecording::getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); std::vector<F64> buf; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -976,7 +962,7 @@ F64 PeriodicRecording::getPeriodMedian( const StatType<SampleAccumulator>& stat, return F64((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); } -F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -985,7 +971,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumula S32 valid_period_count = 0; F64 sum_of_squares = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -1002,13 +988,13 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumula } -F64Kilobytes PeriodicRecording::getPeriodMin( const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodMin( const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes min_val(std::numeric_limits<F64>::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); min_val = llmin(min_val, recording.getMin(stat)); @@ -1022,13 +1008,13 @@ F64Kilobytes PeriodicRecording::getPeriodMin(const MemStatHandle& stat, size_t n return getPeriodMin(static_cast<const StatType<MemAccumulator>&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/) +F64Kilobytes PeriodicRecording::getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes max_val(0.0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); max_val = llmax(max_val, recording.getMax(stat)); @@ -1042,14 +1028,14 @@ F64Kilobytes PeriodicRecording::getPeriodMax(const MemStatHandle& stat, size_t n return getPeriodMax(static_cast<const StatType<MemAccumulator>&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodMean( const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodMean( const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes mean(0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); mean += recording.getMean(stat); @@ -1063,7 +1049,7 @@ F64Kilobytes PeriodicRecording::getPeriodMean(const MemStatHandle& stat, size_t return getPeriodMean(static_cast<const StatType<MemAccumulator>&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -1072,7 +1058,7 @@ F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType<MemAc S32 valid_period_count = 0; F64 sum_of_squares = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) diff --git a/indra/llcommon/lltracerecording.h b/indra/llcommon/lltracerecording.h index 8b56721f42..a6b1a67d02 100644 --- a/indra/llcommon/lltracerecording.h +++ b/indra/llcommon/lltracerecording.h @@ -33,6 +33,7 @@ #include "lltimer.h" #include "lltraceaccumulators.h" #include "llpointer.h" +#include <limits> class LLStopWatchControlsMixinCommon { @@ -330,7 +331,7 @@ namespace LLTrace : public LLStopWatchControlsMixin<PeriodicRecording> { public: - PeriodicRecording(S32 num_periods, EPlayState state = STOPPED); + PeriodicRecording(size_t num_periods, EPlayState state = STOPPED); ~PeriodicRecording(); void nextPeriod(); @@ -353,7 +354,7 @@ namespace LLTrace Recording snapshotCurRecording() const; template <typename T> - auto getSampleCount(const StatType<T>& stat, size_t num_periods = S32_MAX) + auto getSampleCount(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -373,14 +374,14 @@ namespace LLTrace // catch all for stats that have a defined sum template <typename T> - typename T::value_t getPeriodMin(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename T::value_t getPeriodMin(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; typename T::value_t min_val(std::numeric_limits<typename T::value_t>::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -396,39 +397,39 @@ namespace LLTrace } template<typename T> - T getPeriodMin(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMin(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMin(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMin(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodMin(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMin(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMin(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodMin(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMin(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMin(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodMin(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMinPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMinPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes<typename T::value_t>::fractional_t min_val(std::numeric_limits<F64>::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); min_val = llmin(min_val, recording.getPerSec(stat)); @@ -437,7 +438,7 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMinPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMinPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMinPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); @@ -449,14 +450,14 @@ namespace LLTrace // catch all for stats that have a defined sum template <typename T> - typename T::value_t getPeriodMax(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename T::value_t getPeriodMax(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; typename T::value_t max_val(std::numeric_limits<typename T::value_t>::min()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -472,39 +473,39 @@ namespace LLTrace } template<typename T> - T getPeriodMax(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMax(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodMax(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMax(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMax(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMax(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodMax(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMaxPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMaxPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64 max_val = std::numeric_limits<F64>::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); max_val = llmax(max_val, recording.getPerSec(stat)); @@ -513,7 +514,7 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMaxPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMaxPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMaxPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); @@ -525,14 +526,14 @@ namespace LLTrace // catch all for stats that have a defined sum template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMean(const StatType<T >& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMean(const StatType<T >& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes<typename T::value_t>::fractional_t mean(0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -546,39 +547,39 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMean(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMean(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMean(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMean(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMean(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMean(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMean(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMean(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodMean(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMean(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMean(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMean(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMean(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodMean(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMean(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMean(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodMean(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMeanPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMeanPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes<typename T::value_t>::fractional_t mean = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -593,64 +594,64 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMeanPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMeanPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMeanPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); - template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMedianPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) - { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; - num_periods = llmin(num_periods, getNumRecordedPeriods()); - - std::vector <typename RelatedTypes<typename T::value_t>::fractional_t> buf; - for (S32 i = 1; i <= num_periods; i++) - { - Recording& recording = getPrevRecording(i); - if (recording.getDuration() > (F32Seconds)0.f) - { - buf.push_back(recording.getPerSec(stat)); - } - } - std::sort(buf.begin(), buf.end()); - - return typename RelatedTypes<T>::fractional_t((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); - } - - template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMedianPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) - { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; - return typename RelatedTypes<T>::fractional_t(getPeriodMedianPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); - } + template <typename T> + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMedianPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) + { + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + num_periods = llmin(num_periods, getNumRecordedPeriods()); + + std::vector <typename RelatedTypes<typename T::value_t>::fractional_t> buf; + for (size_t i = 1; i <= num_periods; i++) + { + Recording& recording = getPrevRecording(i); + if (recording.getDuration() > (F32Seconds)0.f) + { + buf.push_back(recording.getPerSec(stat)); + } + } + std::sort(buf.begin(), buf.end()); + + return typename RelatedTypes<T>::fractional_t((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); + } + + template<typename T> + typename RelatedTypes<T>::fractional_t getPeriodMedianPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) + { + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + return typename RelatedTypes<T>::fractional_t(getPeriodMedianPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); + } // // PERIODIC STANDARD DEVIATION // - F64 getPeriodStandardDeviation(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodStandardDeviation(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodStandardDeviation(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodStandardDeviation(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodStandardDeviation(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodStandardDeviation(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodStandardDeviation(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodStandardDeviation(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodStandardDeviation(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodStandardDeviation(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); private: // implementation for LLStopWatchControlsMixin @@ -659,6 +660,35 @@ namespace LLTrace /*virtual*/ void handleReset(); /*virtual*/ void handleSplitTo(PeriodicRecording& other); + // helper methods for wraparound ring-buffer arithmetic + inline + size_t wrapi(size_t i) const + { + return i % mRecordingPeriods.size(); + } + + inline + size_t nexti(size_t i, size_t offset=1) const + { + return wrapi(i + offset); + } + + inline + size_t previ(size_t i, size_t offset=1) const + { + auto num_periods = mRecordingPeriods.size(); + // constrain offset + offset = llclamp(offset, 0, num_periods - 1); + // add size() so expression can't go (unsigned) "negative" + return wrapi(i + num_periods - offset); + } + + inline + void inci(size_t& i, size_t offset=1) const + { + i = nexti(i, offset); + } + private: std::vector<Recording> mRecordingPeriods; const bool mAutoResize; diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index fc04dca08d..5655e8e2f2 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -884,7 +884,7 @@ U32 LLUUID::getRandomSeed() seed[7]=(unsigned char)(pid); getSystemTime((uuid_time_t *)(&seed[8])); - U64 seed64 = HBXXH64((const void*)seed, 16).digest(); + U64 seed64 = HBXXH64::digest((const void*)seed, 16); return U32(seed64) ^ U32(seed64 >> 32); } diff --git a/indra/llcommon/lluuid.h b/indra/llcommon/lluuid.h index c139c4eb4e..80597fa186 100644 --- a/indra/llcommon/lluuid.h +++ b/indra/llcommon/lluuid.h @@ -116,6 +116,14 @@ public: U16 getCRC16() const; U32 getCRC32() const; + // Returns a 64 bits digest of the UUID, by XORing its two 64 bits long + // words. HB + inline U64 getDigest64() const + { + U64* tmp = (U64*)mData; + return tmp[0] ^ tmp[1]; + } + static BOOL validate(const std::string& in_string); // Validate that the UUID string is legal. static const LLUUID null; @@ -165,36 +173,22 @@ public: LLAssetID makeAssetID(const LLUUID& session) const; }; -// Generate a hash of an LLUUID object using the boost hash templates. -template <> -struct boost::hash<LLUUID> -{ - typedef LLUUID argument_type; - typedef std::size_t result_type; - result_type operator()(argument_type const& s) const - { - result_type seed(0); - - for (S32 i = 0; i < UUID_BYTES; ++i) - { - boost::hash_combine(seed, s.mData[i]); - } - - return seed; - } -}; - -// Adapt boost hash to std hash +// std::hash implementation for LLUUID namespace std { - template<> struct hash<LLUUID> - { - std::size_t operator()(LLUUID const& s) const noexcept - { - return boost::hash<LLUUID>()(s); - } - }; + template<> struct hash<LLUUID> + { + inline size_t operator()(const LLUUID& id) const noexcept + { + return (size_t)id.getDigest64(); + } + }; } -#endif +// For use with boost containers. +inline size_t hash_value(const LLUUID& id) noexcept +{ + return (size_t)id.getDigest64(); +} +#endif // LL_LLUUID_H diff --git a/indra/llcommon/stdtypes.h b/indra/llcommon/stdtypes.h index da8512169c..0b43d7ad4b 100644 --- a/indra/llcommon/stdtypes.h +++ b/indra/llcommon/stdtypes.h @@ -41,7 +41,7 @@ typedef unsigned int U32; // to express an index that might go negative // (ssize_t is provided by SOME compilers, don't collide) -typedef typename std::make_signed<size_t>::type llssize; +typedef typename std::make_signed<std::size_t>::type llssize; #if LL_WINDOWS // https://docs.microsoft.com/en-us/cpp/build/reference/zc-wchar-t-wchar-t-is-native-type diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 7ee36a9ea6..3ae48a2532 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -109,7 +109,12 @@ namespace tut "import os\n" "import sys\n" "\n" - "from llbase import llsd\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" @@ -120,26 +125,26 @@ namespace tut " pass\n" "\n" "def get():\n" - " hdr = ''\n" - " while ':' not in hdr and len(hdr) < 20:\n" - " hdr += sys.stdin.read(1)\n" + " hdr = []\n" + " while b':' not in hdr and len(hdr) < 20:\n" + " hdr.append(sys.stdin.buffer.read(1))\n" " if not hdr:\n" " sys.exit(0)\n" - " if not hdr.endswith(':'):\n" + " if not hdr[-1] == b':':\n" " raise ProtocolError('Expected len:data, got %r' % hdr, hdr)\n" " try:\n" - " length = int(hdr[:-1])\n" + " length = int(b''.join(hdr[:-1]))\n" " except ValueError:\n" " raise ProtocolError('Non-numeric len %r' % hdr[:-1], hdr[:-1])\n" " parts = []\n" " received = 0\n" " while received < length:\n" - " parts.append(sys.stdin.read(length - received))\n" + " parts.append(sys.stdin.buffer.read(length - received))\n" " received += len(parts[-1])\n" - " data = ''.join(parts)\n" + " data = b''.join(parts)\n" " assert len(data) == length\n" " try:\n" - " return llsd.parse(data.encode())\n" + " return llsd.parse(data)\n" // Seems the old indra.base.llsd module didn't properly // convert IndexError (from running off end of string) to // LLSDParseError. @@ -179,11 +184,11 @@ namespace tut " return _reply\n" "\n" "def put(req):\n" - " sys.stdout.write(':'.join((str(len(req)), req)))\n" + " sys.stdout.buffer.write(b'%d:%b' % (len(req), req))\n" " sys.stdout.flush()\n" "\n" "def send(pump, data):\n" - " put(llsd.format_notation(dict(pump=pump, data=data)).decode())\n" + " put(llsd.format_notation(dict(pump=pump, data=data)))\n" "\n" "def request(pump, data):\n" " # we expect 'data' is a dict\n" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 5dbcf4c9b8..acb2953b5b 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -46,20 +46,24 @@ typedef U32 uint32_t; #include "boost/range.hpp" #include "boost/foreach.hpp" -#include "boost/function.hpp" #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" using namespace boost::phoenix; -#include "../llsd.h" -#include "../llsdserialize.h" +#include "llsd.h" +#include "llsdserialize.h" #include "llsdutil.h" -#include "../llformat.h" +#include "llformat.h" +#include "llmemorystream.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include <functional> + +typedef std::function<void(const LLSD& data, std::ostream& str)> FormatterFunction; +typedef std::function<bool(std::istream& istr, LLSD& data, llssize max_bytes)> ParserFunction; std::vector<U8> string_to_vector(const std::string& str) { @@ -112,7 +116,7 @@ namespace tut mSD = LLUUID::null; expected = "<llsd><uuid /></llsd>\n"; xml_test("null uuid", expected); - + mSD = LLUUID("c96f9b1e-f589-4100-9774-d98643ce0bed"); expected = "<llsd><uuid>c96f9b1e-f589-4100-9774-d98643ce0bed</uuid></llsd>\n"; xml_test("uuid", expected); @@ -136,7 +140,7 @@ namespace tut expected = "<llsd><binary encoding=\"base64\">aGVsbG8=</binary></llsd>\n"; xml_test("binary", expected); } - + template<> template<> void sd_xml_object::test<2>() { @@ -225,7 +229,7 @@ namespace tut expected = "<llsd><map><key>baz</key><undef /><key>foo</key><string>bar</string></map></llsd>\n"; xml_test("2 element map", expected); } - + template<> template<> void sd_xml_object::test<6>() { @@ -241,7 +245,7 @@ namespace tut expected = "<llsd><binary encoding=\"base64\">Nnw2fGFzZGZoYXBweWJveHw2MGU0NGVjNS0zMDVjLTQzYzItOWExOS1iNGI4OWIxYWUyYTZ8NjBlNDRlYzUtMzA1Yy00M2MyLTlhMTktYjRiODliMWFlMmE2fDYwZTQ0ZWM1LTMwNWMtNDNjMi05YTE5LWI0Yjg5YjFhZTJhNnwwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDB8N2ZmZmZmZmZ8N2ZmZmZmZmZ8MHwwfDgyMDAwfDQ1MGZlMzk0LTI5MDQtYzlhZC0yMTRjLWEwN2ViN2ZlZWMyOXwoTm8gRGVzY3JpcHRpb24pfDB8MTB8MA==</binary></llsd>\n"; xml_test("binary", expected); } - + class TestLLSDSerializeData { public: @@ -250,9 +254,34 @@ namespace tut void doRoundTripTests(const std::string&); void checkRoundTrip(const std::string&, const LLSD& v); - - LLPointer<LLSDFormatter> mFormatter; - LLPointer<LLSDParser> mParser; + + void setFormatterParser(LLPointer<LLSDFormatter> formatter, LLPointer<LLSDParser> parser) + { + mFormatter = [formatter](const LLSD& data, std::ostream& str) + { + formatter->format(data, str); + }; + // this lambda must be mutable since otherwise the bound 'parser' + // is assumed to point to a const LLSDParser + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) mutable + { + // reset() call is needed since test code re-uses parser object + parser->reset(); + return (parser->parse(istr, data, max_bytes) > 0); + }; + } + + void setParser(bool (*parser)(LLSD&, std::istream&, llssize)) + { + // why does LLSDSerialize::deserialize() reverse the parse() params?? + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) + { + return parser(data, istr, max_bytes); + }; + } + + FormatterFunction mFormatter; + ParserFunction mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -265,12 +294,11 @@ namespace tut void TestLLSDSerializeData::checkRoundTrip(const std::string& msg, const LLSD& v) { - std::stringstream stream; - mFormatter->format(v, stream); + std::stringstream stream; + mFormatter(v, stream); //LL_INFOS() << "checkRoundTrip: length " << stream.str().length() << LL_ENDL; LLSD w; - mParser->reset(); // reset() call is needed since test code re-uses mParser - mParser->parse(stream, w, stream.str().size()); + mParser(stream, w, stream.str().size()); try { @@ -299,52 +327,52 @@ namespace tut fillmap(root[key], width, depth - 1); } } - + void TestLLSDSerializeData::doRoundTripTests(const std::string& msg) { LLSD v; checkRoundTrip(msg + " undefined", v); - + v = true; checkRoundTrip(msg + " true bool", v); - + v = false; checkRoundTrip(msg + " false bool", v); - + v = 1; checkRoundTrip(msg + " positive int", v); - + v = 0; checkRoundTrip(msg + " zero int", v); - + v = -1; checkRoundTrip(msg + " negative int", v); - + v = 1234.5f; checkRoundTrip(msg + " positive float", v); - + v = 0.0f; checkRoundTrip(msg + " zero float", v); - + v = -1234.5f; checkRoundTrip(msg + " negative float", v); - + // FIXME: need a NaN test - + v = LLUUID::null; checkRoundTrip(msg + " null uuid", v); - + LLUUID newUUID; newUUID.generate(); v = newUUID; checkRoundTrip(msg + " new uuid", v); - + v = ""; checkRoundTrip(msg + " empty string", v); - + v = "some string"; checkRoundTrip(msg + " non-empty string", v); - + v = "Second Life is a 3-D virtual world entirely built and owned by its residents. " "Since opening to the public in 2003, it has grown explosively and today is " @@ -372,7 +400,7 @@ namespace tut for (U32 block = 0x000000; block <= 0x10ffff; block += block_size) { std::ostringstream out; - + for (U32 c = block; c < block + block_size; ++c) { if (c <= 0x000001f @@ -386,7 +414,7 @@ namespace tut if (0x00fdd0 <= c && c <= 0x00fdef) { continue; } if ((c & 0x00fffe) == 0x00fffe) { continue; } // see Unicode standard, section 15.8 - + if (c <= 0x00007f) { out << (char)(c & 0x7f); @@ -410,55 +438,55 @@ namespace tut out << (char)(0x80 | ((c >> 0) & 0x3f)); } } - + v = out.str(); std::ostringstream blockmsg; blockmsg << msg << " unicode string block 0x" << std::hex << block; checkRoundTrip(blockmsg.str(), v); } - + LLDate epoch; v = epoch; checkRoundTrip(msg + " epoch date", v); - + LLDate aDay("2002-12-07T05:07:15.00Z"); v = aDay; checkRoundTrip(msg + " date", v); - + LLURI path("http://slurl.com/secondlife/Ambleside/57/104/26/"); v = path; checkRoundTrip(msg + " url", v); - + const char source[] = "it must be a blue moon again"; std::vector<U8> data; // note, includes terminating '\0' copy(&source[0], &source[sizeof(source)], back_inserter(data)); - + v = data; checkRoundTrip(msg + " binary", v); - + v = LLSD::emptyMap(); checkRoundTrip(msg + " empty map", v); - + v = LLSD::emptyMap(); v["name"] = "luke"; //v.insert("name", "luke"); v["age"] = 3; //v.insert("age", 3); checkRoundTrip(msg + " map", v); - + v.clear(); v["a"]["1"] = true; v["b"]["0"] = false; checkRoundTrip(msg + " nested maps", v); - + v = LLSD::emptyArray(); checkRoundTrip(msg + " empty array", v); - + v = LLSD::emptyArray(); v.append("ali"); v.append(28); checkRoundTrip(msg + " array", v); - + v.clear(); v[0][0] = true; v[1][0] = false; @@ -468,7 +496,7 @@ namespace tut fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } - + typedef tut::test_group<TestLLSDSerializeData> TestLLSDSerializeGroup; typedef TestLLSDSerializeGroup::object TestLLSDSerializeObject; TestLLSDSerializeGroup gTestLLSDSerializeGroup("llsd serialization"); @@ -476,35 +504,106 @@ namespace tut template<> template<> void TestLLSDSerializeObject::test<1>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY), + new LLSDNotationParser()); doRoundTripTests("pretty binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<2>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); doRoundTripTests("raw binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<3>() { - mFormatter = new LLSDXMLFormatter(); - mParser = new LLSDXMLParser(); + setFormatterParser(new LLSDXMLFormatter(), new LLSDXMLParser()); doRoundTripTests("xml serialization"); } template<> template<> void TestLLSDSerializeObject::test<4>() { - mFormatter = new LLSDBinaryFormatter(); - mParser = new LLSDBinaryParser(); + setFormatterParser(new LLSDBinaryFormatter(), new LLSDBinaryParser()); doRoundTripTests("binary serialization"); } + template<> template<> + void TestLLSDSerializeObject::test<5>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_BINARY); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_BINARY)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<6>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_XML); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_XML)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<7>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_NOTATION); + }; + setParser(LLSDSerialize::deserialize); + // In this test, serialize(LLSD_NOTATION) emits a header recognized by + // deserialize(). + doRoundTripTests("serialize(LLSD_NOTATION)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<8>() + { + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDNotationFormatter does not + // emit an llsd/notation header. + doRoundTripTests("LLSDNotationFormatter -> deserialize"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<9>() + { + setFormatterParser(new LLSDXMLFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDXMLParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDXMLFormatter does not + // emit an LLSD/XML header. + doRoundTripTests("LLSDXMLFormatter -> deserialize"); + }; + +/*==========================================================================*| + // We do not expect this test to succeed. Without a header, neither + // notation LLSD nor binary LLSD reliably start with a distinct character, + // the way XML LLSD starts with '<'. By convention, we default to notation + // rather than binary. + template<> template<> + void TestLLSDSerializeObject::test<10>() + { + setFormatterParser(new LLSDBinaryFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDBinaryParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDBinaryFormatter does not + // emit an LLSD/Binary header. + doRoundTripTests("LLSDBinaryFormatter -> deserialize"); + }; +|*==========================================================================*/ /** * @class TestLLSDParsing @@ -555,7 +654,7 @@ namespace tut public: TestLLSDXMLParsing() {} }; - + typedef tut::test_group<TestLLSDXMLParsing> TestLLSDXMLParsingGroup; typedef TestLLSDXMLParsingGroup::object TestLLSDXMLParsingObject; TestLLSDXMLParsingGroup gTestLLSDXMLParsingGroup("llsd XML parsing"); @@ -586,8 +685,8 @@ namespace tut LLSD(), LLSDParser::PARSE_FAILURE); } - - + + template<> template<> void TestLLSDXMLParsingObject::test<2>() { @@ -596,7 +695,7 @@ namespace tut v["amy"] = 23; v["bob"] = LLSD(); v["cam"] = 1.23; - + ensureParse( "unknown data type", "<llsd><map>" @@ -607,16 +706,16 @@ namespace tut v, v.size() + 1); } - + template<> template<> void TestLLSDXMLParsingObject::test<3>() { // test handling of nested bad data - + LLSD v; v["amy"] = 23; v["cam"] = 1.23; - + ensureParse( "map with html", "<llsd><map>" @@ -626,7 +725,7 @@ namespace tut "</map></llsd>", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["cam"] = 1.23; @@ -639,7 +738,7 @@ namespace tut "</map></llsd>", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["bob"] = LLSD::emptyMap(); @@ -661,7 +760,7 @@ namespace tut v[0] = 23; v[1] = LLSD(); v[2] = 1.23; - + ensureParse( "array value of html", "<llsd><array>" @@ -671,7 +770,7 @@ namespace tut "</array></llsd>", v, v.size() + 1); - + v.clear(); v[0] = 23; v[1] = LLSD::emptyMap(); @@ -1225,7 +1324,7 @@ namespace tut vec[0] = 'a'; vec[1] = 'b'; vec[2] = 'c'; vec[3] = '3'; vec[4] = '2'; vec[5] = '1'; LLSD value = vec; - + vec.resize(11); vec[0] = 'b'; // for binary vec[5] = 'a'; vec[6] = 'b'; vec[7] = 'c'; @@ -1694,85 +1793,83 @@ namespace tut ensureBinaryAndXML("map", test); } - struct TestPythonCompatible + // helper for TestPythonCompatible + static std::string import_llsd("import os.path\n" + "import sys\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n"); + + // helper for TestPythonCompatible + template <typename CONTENT> + void python(const std::string& desc, const CONTENT& script, int expect=0) { - TestPythonCompatible(): - // Note the peculiar insertion of __FILE__ into this string. Since - // this script is being written into a platform-dependent temp - // directory, we can't locate indra/lib/python relative to - // Python's __file__. Use __FILE__ instead, navigating relative - // to this C++ source file. Use Python raw-string syntax so - // Windows pathname backslashes won't mislead Python's string - // scanner. - import_llsd("import os.path\n" - "import sys\n" - "from llbase import llsd\n") - {} - ~TestPythonCompatible() {} + auto PYTHON(LLStringUtil::getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - std::string import_llsd; + NamedTempFile scriptfile("py", script); - template <typename CONTENT> - void python(const std::string& desc, const CONTENT& script, int expect=0) +#if LL_WINDOWS + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) { - auto PYTHON(LLStringUtil::getenv("PYTHON")); - ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - - NamedTempFile scriptfile("py", script); + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } -#if LL_WINDOWS - std::string q("\""); - std::string qPYTHON(q + PYTHON + q); - std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); - if (rc == -1) - { - char buffer[256]; - strerror_s(buffer, errno); // C++ can infer the buffer size! :-O - ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); - } - else +#else // LL_DARWIN, LL_LINUX + LLProcess::Params params; + params.executable = PYTHON; + params.args.add(scriptfile.getName()); + LLProcessPtr py(LLProcess::create(params)); + ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py->getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) { - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); } - -#else // LL_DARWIN, LL_LINUX - LLProcess::Params params; - params.executable = PYTHON; - params.args.add(scriptfile.getName()); - LLProcessPtr py(LLProcess::create(params)); - ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); - // Implementing timeout would mean messing with alarm() and - // catching SIGALRM... later maybe... - int status(0); - if (waitpid(py->getProcessID(), &status, 0) == -1) + else if (WIFSIGNALED(status)) { - int waitpid_errno(errno); - ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " - "waitpid() errno " << waitpid_errno), - waitpid_errno, ECHILD); + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); } else { - if (WIFEXITED(status)) - { - int rc(WEXITSTATUS(status)); - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), - rc, expect); - } - else if (WIFSIGNALED(status)) - { - ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), - false); - } - else - { - ensure(STRINGIZE(desc << " script produced impossible status " << status), - false); - } + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); } -#endif } +#endif + } + + struct TestPythonCompatible + { + TestPythonCompatible() {} + ~TestPythonCompatible() {} }; typedef tut::test_group<TestPythonCompatible> TestPythonCompatibleGroup; @@ -1798,25 +1895,33 @@ namespace tut "print('Running on', sys.platform)\n"); } - // helper for test<3> - static void writeLLSDArray(std::ostream& out, const LLSD& array) + // helper for test<3> - test<7> + static void writeLLSDArray(const FormatterFunction& serialize, + std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item : llsd::inArray(array)) { - LLSDSerialize::toNotation(item, out); - // It's important to separate with newlines because Python's llsd - // module doesn't support parsing from a file stream, only from a - // string, so we have to know how much of the file to read into a - // string. - out << '\n'; + // It's important to delimit the entries in this file somehow + // because, although Python's llsd.parse() can accept a file + // stream, the XML parser expects EOF after a single outer element + // -- it doesn't just stop. So we must extract a sequence of bytes + // strings from the file. But since one of the serialization + // formats we want to test is binary, we can't pick any single + // byte value as a delimiter! Use a binary integer length prefix + // instead. + std::ostringstream buffer; + serialize(item, buffer); + auto buffstr{ buffer.str() }; + int bufflen{ static_cast<int>(buffstr.length()) }; + out.write(reinterpret_cast<const char*>(&bufflen), sizeof(bufflen)); + out.write(buffstr.c_str(), buffstr.length()); } } - template<> template<> - void TestPythonCompatibleObject::test<3>() + // helper for test<3> - test<7> + static void toPythonUsing(const std::string& desc, + const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - LLSD cdata(llsd::array(17, 3.14, "This string\n" "has several\n" @@ -1836,7 +1941,7 @@ namespace tut " except StopIteration:\n" " pass\n" " else:\n" - " assert False, 'Too many data items'\n"; + " raise AssertionError('Too many data items')\n"; // Create an llsdXXXXXX file containing 'data' serialized to // notation. @@ -1845,32 +1950,128 @@ namespace tut // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [serialize, cdata] + (std::ostream& out) + { writeLLSDArray(serialize, out, cdata); }); - python("read C++ notation", + python("read C++ " + desc, placeholders::arg1 << import_llsd << - "def parse_each(iterable):\n" - " for item in iterable:\n" - " yield llsd.parse(item)\n" << - pydata << + "from functools import partial\n" + "import io\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " try:\n" + " frombytes = llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" + << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_XML)"); + toPythonUsing("LLSD_XML", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_XML); }); + } + + template<> template<> void TestPythonCompatibleObject::test<4>() { - set_test_name("verify sequence from Python"); + set_test_name("to Python using LLSDSerialize::serialize(LLSD_NOTATION)"); + toPythonUsing("LLSD_NOTATION", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_NOTATION); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<5>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_BINARY)"); + toPythonUsing("LLSD_BINARY", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_BINARY); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<6>() + { + set_test_name("to Python using LLSDSerialize::toXML()"); + toPythonUsing("toXML()", LLSDSerialize::toXML); + } + + template<> template<> + void TestPythonCompatibleObject::test<7>() + { + set_test_name("to Python using LLSDSerialize::toNotation()"); + toPythonUsing("toNotation()", LLSDSerialize::toNotation); + } +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("to Python using LLSDSerialize::toBinary()"); + // We don't expect this to work because, without a header, + // llsd.parse() will assume notation rather than binary. + toPythonUsing("toBinary()", LLSDSerialize::toBinary); + } +|*==========================================================================*/ + + // helper for test<8> - test<12> + bool itemFromStream(std::istream& istr, LLSD& item, const ParserFunction& parse) + { + // reset the output value for debugging clarity + item.clear(); + // We use an int length prefix as a foolproof delimiter even for + // binary serialized streams. + int length{ 0 }; + istr.read(reinterpret_cast<char*>(&length), sizeof(length)); +// return parse(istr, item, length); + // Sadly, as of 2022-12-01 it seems we can't really trust our LLSD + // parsers to honor max_bytes: this test works better when we read + // each item into its own distinct LLMemoryStream, instead of passing + // the original istr with a max_bytes constraint. + std::vector<U8> buffer(length); + istr.read(reinterpret_cast<char*>(buffer.data()), length); + LLMemoryStream stream(buffer.data(), length); + return parse(stream, item, length); + } + + // helper for test<8> - test<12> + void fromPythonUsing(const std::string& pyformatter, + const ParserFunction& parse= + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::deserialize(data, istr, max_bytes); }) + { // Create an empty data file. This is just a placeholder for our // script to write into. Create it to establish a unique name that // we know. NamedTempFile file("llsd", ""); - python("write Python notation", + python("Python " + pyformatter, placeholders::arg1 << import_llsd << + "import struct\n" + "lenformat = struct.Struct('i')\n" "DATA = [\n" " 17,\n" " 3.14,\n" @@ -1881,34 +2082,87 @@ namespace tut "]\n" // Don't forget raw-string syntax for Windows pathnames. // N.B. Using 'print' implicitly adds newlines. - "with open(r'" << file.getName() << "', 'w') as f:\n" + "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " serialized = llsd." << pyformatter << "(item)\n" + " f.write(lenformat.pack(len(serialized)))\n" + " f.write(serialized)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; - // Notice that we're not doing anything special to parse out the - // newlines: LLSDSerialize::fromNotation ignores them. While it would - // seem they're not strictly necessary, going in this direction, we - // want to ensure that notation-separated-by-newlines works in both - // directions -- since in practice, a given file might be read by - // either language. - ensure_equals("Failed to read LLSD::Integer from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_equals(item.asInteger(), 17); - ensure_equals("Failed to read LLSD::Real from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_approximately_equals("Bad LLSD::Real value from Python", - item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure_equals("Failed to read LLSD::String from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_equals(item.asString(), - "This string\n" - "has several\n" - "lines."); - + try + { + ensure("Failed to read LLSD::Integer from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asInteger(), 17); + ensure("Failed to read LLSD::Real from Python", + itemFromStream(inf, item, parse)); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure("Failed to read LLSD::String from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } + catch (const tut::failure& err) + { + std::cout << "for " << err.what() << ", item = " << item << std::endl; + throw; + } + } + + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("from Python XML using LLSDSerialize::deserialize()"); + fromPythonUsing("format_xml"); + } + + template<> template<> + void TestPythonCompatibleObject::test<9>() + { + set_test_name("from Python notation using LLSDSerialize::deserialize()"); + fromPythonUsing("format_notation"); + } + + template<> template<> + void TestPythonCompatibleObject::test<10>() + { + set_test_name("from Python binary using LLSDSerialize::deserialize()"); + fromPythonUsing("format_binary"); + } + + template<> template<> + void TestPythonCompatibleObject::test<11>() + { + set_test_name("from Python XML using fromXML()"); + // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors + fromPythonUsing("format_xml", + [](std::istream& istr, LLSD& data, llssize) + { return LLSDSerialize::fromXML(data, istr) > 0; }); + } + + template<> template<> + void TestPythonCompatibleObject::test<12>() + { + set_test_name("from Python notation using fromNotation()"); + fromPythonUsing("format_notation", + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<13>() + { + set_test_name("from Python binary using fromBinary()"); + // We don't expect this to work because format_binary() emits a + // header, but fromBinary() won't recognize a header. + fromPythonUsing("format_binary", + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); } +|*==========================================================================*/ } |