diff options
Diffstat (limited to 'indra/llcommon/llsdserialize.cpp')
-rw-r--r-- | indra/llcommon/llsdserialize.cpp | 205 |
1 files changed, 108 insertions, 97 deletions
diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 8b4a0ee6d8..b7e316da10 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -34,6 +34,9 @@ #include <iostream> #include "apr_base64.h" +#include <boost/iostreams/device/array.hpp> +#include <boost/iostreams/stream.hpp> + #ifdef LL_USESYSTEMLIBS # include <zlib.h> #else @@ -50,7 +53,7 @@ #include "lluri.h" // File constants -static const int MAX_HDR_LEN = 20; +static const size_t MAX_HDR_LEN = 20; static const S32 UNZIP_LLSD_MAX_DEPTH = 96; static const char LEGACY_NON_HEADER[] = "<llsd>"; const std::string LLSD_BINARY_HEADER("LLSD/Binary"); @@ -99,7 +102,7 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize } // static -bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) +bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) { LLPointer<LLSDParser> p = NULL; char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ @@ -252,7 +255,7 @@ F64 ll_ntohd(F64 netdouble) * @return Returns number of bytes read off of the stream. Returns * PARSE_FAILURE (-1) on failure. */ -int deserialize_string(std::istream& istr, std::string& value, S32 max_bytes); +llssize deserialize_string(std::istream& istr, std::string& value, llssize max_bytes); /** * @brief Parse a delimited string. @@ -263,7 +266,7 @@ int deserialize_string(std::istream& istr, std::string& value, S32 max_bytes); * @return Returns number of bytes read off of the stream. Returns * PARSE_FAILURE (-1) on failure. */ -int deserialize_string_delim(std::istream& istr, std::string& value, char d); +llssize deserialize_string_delim(std::istream& istr, std::string& value, char d); /** * @brief Read a raw string off the stream. @@ -277,10 +280,10 @@ int deserialize_string_delim(std::istream& istr, std::string& value, char d); * @return Returns number of bytes read off of the stream. Returns * PARSE_FAILURE (-1) on failure. */ -int deserialize_string_raw( +llssize deserialize_string_raw( std::istream& istr, std::string& value, - S32 max_bytes); + llssize max_bytes); /** * @brief helper method for dealing with the different notation boolean format. @@ -292,7 +295,7 @@ int deserialize_string_raw( * @return Returns number of bytes read off of the stream. Returns * PARSE_FAILURE (-1) on failure. */ -int deserialize_boolean( +llssize deserialize_boolean( std::istream& istr, LLSD& data, const std::string& compare, @@ -329,7 +332,7 @@ LLSDParser::LLSDParser() LLSDParser::~LLSDParser() { } -S32 LLSDParser::parse(std::istream& istr, LLSD& data, S32 max_bytes, S32 max_depth) +S32 LLSDParser::parse(std::istream& istr, LLSD& data, llssize max_bytes, S32 max_depth) { mCheckLimits = (LLSDSerialize::SIZE_UNLIMITED == max_bytes) ? false : true; mMaxBytesLeft = max_bytes; @@ -359,7 +362,7 @@ std::istream& LLSDParser::get( char delim) const { istr.get(s, n, delim); - if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); return istr; } @@ -369,7 +372,7 @@ std::istream& LLSDParser::get( char delim) const { istr.get(sb, delim); - if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); return istr; } @@ -393,11 +396,11 @@ std::istream& LLSDParser::read( std::streamsize n) const { istr.read(s, n); - if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); return istr; } -void LLSDParser::account(S32 bytes) const +void LLSDParser::account(llssize bytes) const { if(mCheckLimits) mMaxBytesLeft -= bytes; } @@ -502,7 +505,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) c c = istr.peek(); if(isalpha(c)) { - int cnt = deserialize_boolean( + auto cnt = deserialize_boolean( istr, data, NOTATION_FALSE_SERIAL, @@ -532,7 +535,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) c c = istr.peek(); if(isalpha(c)) { - int cnt = deserialize_boolean(istr,data,NOTATION_TRUE_SERIAL,true); + auto cnt = deserialize_boolean(istr,data,NOTATION_TRUE_SERIAL,true); if(PARSE_FAILURE == cnt) parse_count = cnt; else account(cnt); } @@ -608,7 +611,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) c c = get(istr); // pop the 'l' c = get(istr); // pop the delimiter std::string str; - int cnt = deserialize_string_delim(istr, str, c); + auto cnt = deserialize_string_delim(istr, str, c); if(PARSE_FAILURE == cnt) { parse_count = PARSE_FAILURE; @@ -631,7 +634,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) c c = get(istr); // pop the 'd' c = get(istr); // pop the delimiter std::string str; - int cnt = deserialize_string_delim(istr, str, c); + auto cnt = deserialize_string_delim(istr, str, c); if(PARSE_FAILURE == cnt) { parse_count = PARSE_FAILURE; @@ -663,7 +666,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) c default: parse_count = PARSE_FAILURE; - LL_INFOS() << "Unrecognized character while parsing: int(" << (int)c + LL_INFOS() << "Unrecognized character while parsing: int(" << int(c) << ")" << LL_ENDL; break; } @@ -694,7 +697,7 @@ S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) c { putback(istr, c); found_name = true; - int count = deserialize_string(istr, name, mMaxBytesLeft); + auto count = deserialize_string(istr, name, mMaxBytesLeft); if(PARSE_FAILURE == count) return PARSE_FAILURE; account(count); } @@ -776,7 +779,7 @@ S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array, S32 max_dept bool LLSDNotationParser::parseString(std::istream& istr, LLSD& data) const { std::string value; - int count = deserialize_string(istr, value, mMaxBytesLeft); + auto count = deserialize_string(istr, value, mMaxBytesLeft); if(PARSE_FAILURE == count) return false; account(count); data = value; @@ -803,13 +806,13 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const { // We probably have a valid raw binary stream. determine // the size, and read it. - S32 len = strtol(buf + 2, NULL, 0); + auto len = strtol(buf + 2, NULL, 0); if(mCheckLimits && (len > mMaxBytesLeft)) return false; std::vector<U8> value; if(len) { value.resize(len); - account((int)fullread(istr, (char *)&value[0], len)); + account(fullread(istr, (char *)&value[0], len)); } c = get(istr); // strip off the trailing double-quote data = value; @@ -1006,7 +1009,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) con case '"': { std::string value; - int cnt = deserialize_string_delim(istr, value, c); + auto cnt = deserialize_string_delim(istr, value, c); if(PARSE_FAILURE == cnt) { parse_count = PARSE_FAILURE; @@ -1093,7 +1096,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) con if(size > 0) { value.resize(size); - account((int)fullread(istr, (char*)&value[0], size)); + account(fullread(istr, (char*)&value[0], size)); } data = value; } @@ -1107,7 +1110,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) con default: parse_count = PARSE_FAILURE; - LL_INFOS() << "Unrecognized character while parsing: int(" << (int)c + LL_INFOS() << "Unrecognized character while parsing: int(" << int(c) << ")" << LL_ENDL; break; } @@ -1141,7 +1144,7 @@ S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) con case '\'': case '"': { - int cnt = deserialize_string_delim(istr, name, c); + auto cnt = deserialize_string_delim(istr, name, c); if(PARSE_FAILURE == cnt) return PARSE_FAILURE; account(cnt); break; @@ -1225,7 +1228,7 @@ bool LLSDBinaryParser::parseString( if(size) { buf.resize(size); - account((int)fullread(istr, &buf[0], size)); + account(fullread(istr, &buf[0], size)); value.assign(buf.begin(), buf.end()); } return true; @@ -1429,7 +1432,7 @@ S32 LLSDNotationFormatter::format_impl(const LLSD& data, std::ostream& ostr, ostr << std::uppercase; auto oldfill(ostr.fill('0')); auto oldwidth(ostr.width()); - for (int i = 0; i < buffer.size(); i++) + for (size_t i = 0; i < buffer.size(); i++) { // have to restate setw() before every conversion ostr << std::setw(2) << (int) buffer[i]; @@ -1592,7 +1595,7 @@ void LLSDBinaryFormatter::formatString( /** * local functions */ -int deserialize_string(std::istream& istr, std::string& value, S32 max_bytes) +llssize deserialize_string(std::istream& istr, std::string& value, llssize max_bytes) { int c = istr.get(); if(istr.fail()) @@ -1602,7 +1605,7 @@ int deserialize_string(std::istream& istr, std::string& value, S32 max_bytes) return LLSDParser::PARSE_FAILURE; } - int rv = LLSDParser::PARSE_FAILURE; + llssize rv = LLSDParser::PARSE_FAILURE; switch(c) { case '\'': @@ -1622,7 +1625,7 @@ int deserialize_string(std::istream& istr, std::string& value, S32 max_bytes) return rv + 1; // account for the character grabbed at the top. } -int deserialize_string_delim( +llssize deserialize_string_delim( std::istream& istr, std::string& value, char delim) @@ -1632,7 +1635,7 @@ int deserialize_string_delim( bool found_hex = false; bool found_digit = false; U8 byte = 0; - int count = 0; + llssize count = 0; while (true) { @@ -1647,7 +1650,7 @@ int deserialize_string_delim( } char next_char = (char)next_byte; // Now that we know it's not EOF - + if(found_escape) { // next character(s) is a special sequence. @@ -1725,16 +1728,16 @@ int deserialize_string_delim( return count; } -int deserialize_string_raw( +llssize deserialize_string_raw( std::istream& istr, std::string& value, - S32 max_bytes) + llssize max_bytes) { - int count = 0; + llssize count = 0; const S32 BUF_LEN = 20; char buf[BUF_LEN]; /* Flawfinder: ignore */ istr.get(buf, BUF_LEN - 1, ')'); - count += (int)istr.gcount(); + count += istr.gcount(); int c = istr.get(); c = istr.get(); count += 2; @@ -1743,13 +1746,13 @@ int deserialize_string_raw( // We probably have a valid raw string. determine // the size, and read it. // *FIX: This is memory inefficient. - S32 len = strtol(buf + 1, NULL, 0); + auto len = strtol(buf + 1, NULL, 0); if((max_bytes>0)&&(len>max_bytes)) return LLSDParser::PARSE_FAILURE; std::vector<char> buf; if(len) { buf.resize(len); - count += (int)fullread(istr, (char *)&buf[0], len); + count += fullread(istr, (char *)&buf[0], len); value.assign(buf.begin(), buf.end()); } c = istr.get(); @@ -2038,7 +2041,7 @@ void serialize_string(const std::string& value, std::ostream& str) } } -int deserialize_boolean( +llssize deserialize_boolean( std::istream& istr, LLSD& data, const std::string& compare, @@ -2055,7 +2058,7 @@ int deserialize_boolean( // * set data to LLSD::null // * return LLSDParser::PARSE_FAILURE (-1) // - int bytes_read = 0; + llssize bytes_read = 0; std::string::size_type ii = 0; char c = istr.peek(); while((++ii < compare.size()) @@ -2110,7 +2113,7 @@ std::string zip_llsd(LLSD& data) U8 out[CHUNK]; - strm.avail_in = source.size(); + strm.avail_in = narrow(source.size()); strm.next_in = (U8*) source.data(); U8* output = NULL; @@ -2128,7 +2131,9 @@ std::string zip_llsd(LLSD& data) { //copy result into output if (strm.avail_out >= CHUNK) { - free(output); + deflateEnd(&strm); + if(output) + free(output); LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL; return std::string(); } @@ -2151,7 +2156,9 @@ std::string zip_llsd(LLSD& data) } else { - free(output); + deflateEnd(&strm); + if(output) + free(output); LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL; return std::string(); } @@ -2162,7 +2169,8 @@ std::string zip_llsd(LLSD& data) std::string result((char*) output, size); deflateEnd(&strm); - free(output); + if(output) + free(output); return result; } @@ -2172,53 +2180,66 @@ std::string zip_llsd(LLSD& data) // and deserializes from that copy using LLSDSerialize LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, S32 size) { + std::unique_ptr<U8[]> in = std::unique_ptr<U8[]>(new(std::nothrow) U8[size]); + if (!in) + { + return ZR_MEM_ERROR; + } + is.read((char*) in.get(), size); + + return unzip_llsd(data, in.get(), size); +} + +LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, const U8* in, S32 size) +{ U8* result = NULL; U32 cur_size = 0; z_stream strm; - const U32 CHUNK = 65536; + constexpr U32 CHUNK = 1024 * 512; - U8 *in = new(std::nothrow) U8[size]; - if (!in) + static thread_local std::unique_ptr<U8[]> out; + if (!out) { - return ZR_MEM_ERROR; + out = std::unique_ptr<U8[]>(new(std::nothrow) U8[CHUNK]); } - is.read((char*) in, size); - - U8 out[CHUNK]; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; strm.avail_in = size; - strm.next_in = in; + strm.next_in = const_cast<U8*>(in); S32 ret = inflateInit(&strm); do { strm.avail_out = CHUNK; - strm.next_out = out; + strm.next_out = out.get(); ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_ERROR) + switch (ret) + { + case Z_NEED_DICT: + case Z_DATA_ERROR: { inflateEnd(&strm); free(result); - delete [] in; return ZR_DATA_ERROR; } - - switch (ret) + case Z_STREAM_ERROR: + case Z_BUF_ERROR: { - case Z_NEED_DICT: - ret = Z_DATA_ERROR; - case Z_DATA_ERROR: + inflateEnd(&strm); + free(result); + return ZR_BUFFER_ERROR; + } + case Z_MEM_ERROR: + { inflateEnd(&strm); free(result); - delete [] in; return ZR_MEM_ERROR; - break; + } } U32 have = CHUNK-strm.avail_out; @@ -2231,17 +2252,15 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, { free(result); } - delete[] in; return ZR_MEM_ERROR; } result = new_result; - memcpy(result+cur_size, out, have); + memcpy(result+cur_size, out.get(), have); cur_size += have; - } while (ret == Z_OK); + } while (ret == Z_OK && ret != Z_STREAM_END); inflateEnd(&strm); - delete [] in; if (ret != Z_STREAM_END) { @@ -2251,37 +2270,11 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, //result now points to the decompressed LLSD block { - std::istringstream istr; - // Since we are using this for meshes, data we are dealing with tend to be large. - // So string can potentially fail to allocate, make sure this won't cause problems - try - { - std::string res_str((char*)result, cur_size); + char* result_ptr = strip_deprecated_header((char*)result, cur_size); - std::string deprecated_header("<? LLSD/Binary ?>"); - - if (res_str.substr(0, deprecated_header.size()) == deprecated_header) - { - res_str = res_str.substr(deprecated_header.size() + 1, cur_size); - } - cur_size = res_str.size(); - - istr.str(res_str); - } -#ifdef LL_WINDOWS - catch (std::length_error) - { - free(result); - return ZR_SIZE_ERROR; - } -#endif - catch (std::bad_alloc&) - { - free(result); - return ZR_MEM_ERROR; - } - - if (!LLSDSerialize::fromBinary(data, istr, cur_size, UNZIP_LLSD_MAX_DEPTH)) + boost::iostreams::stream<boost::iostreams::array_source> istrm(result_ptr, cur_size); + + if (!LLSDSerialize::fromBinary(data, istrm, cur_size, UNZIP_LLSD_MAX_DEPTH)) { free(result); return ZR_PARSE_ERROR; @@ -2294,7 +2287,7 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, //This unzip function will only work with a gzip header and trailer - while the contents //of the actual compressed data is the same for either format (gzip vs zlib ), the headers //and trailers are different for the formats. -U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 size ) +U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size ) { if (size == 0) { @@ -2395,4 +2388,22 @@ U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 return result; } +char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size) +{ + const char* deprecated_header = "<? LLSD/Binary ?>"; + constexpr size_t deprecated_header_size = 17; + + if (cur_size > deprecated_header_size + && memcmp(in, deprecated_header, deprecated_header_size) == 0) + { + in = in + deprecated_header_size; + cur_size = cur_size - deprecated_header_size; + if (header_size) + { + *header_size = deprecated_header_size + 1; + } + } + + return in; +} |