diff options
Diffstat (limited to 'indra/llcommon/llsdserialize.cpp')
-rwxr-xr-x[-rw-r--r--] | indra/llcommon/llsdserialize.cpp | 410 |
1 files changed, 359 insertions, 51 deletions
diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 69e3ff266a..d49ff0feb5 100644..100755 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -34,6 +34,12 @@ #include <iostream> #include "apr_base64.h" +#ifdef LL_USESYSTEMLIBS +# include <zlib.h> +#else +# include "zlib/zlib.h" // for davep's dirty little zip functions +#endif + #if !LL_WINDOWS #include <netinet/in.h> // htonl & ntohl #endif @@ -49,6 +55,10 @@ static const char LEGACY_NON_HEADER[] = "<llsd>"; const std::string LLSD_BINARY_HEADER("LLSD/Binary"); const std::string LLSD_XML_HEADER("LLSD/XML"); +//used to deflate a gzipped asset (currently used for navmeshes) +#define windowBits 15 +#define ENABLE_ZLIB_GZIP 32 + /** * LLSDSerialize */ @@ -71,7 +81,7 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize break; default: - llwarns << "serialize request for unkown ELLSD_Serialize" << llendl; + LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL; } if (f.notNull()) @@ -104,7 +114,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ { legacy_no_header = true; - inbuf = str.gcount(); + inbuf = (int)str.gcount(); } else { @@ -159,7 +169,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) } else { - llwarns << "deserialize request for unknown ELLSD_Serialize" << llendl; + LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL; } if (p.notNull()) @@ -169,7 +179,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) } fail: - llwarns << "deserialize LLSD parse failure" << llendl; + LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; return false; } @@ -337,7 +347,7 @@ std::istream& LLSDParser::get( char delim) const { istr.get(s, n, delim); - if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); return istr; } @@ -347,7 +357,7 @@ std::istream& LLSDParser::get( char delim) const { istr.get(sb, delim); - if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); return istr; } @@ -371,7 +381,7 @@ std::istream& LLSDParser::read( std::streamsize n) const { istr.read(s, n); - if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); return istr; } @@ -435,7 +445,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading map." << llendl; + LL_INFOS() << "STREAM FAILURE reading map." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -454,7 +464,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading array." << llendl; + LL_INFOS() << "STREAM FAILURE reading array." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -490,7 +500,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading boolean." << llendl; + LL_INFOS() << "STREAM FAILURE reading boolean." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -516,7 +526,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading boolean." << llendl; + LL_INFOS() << "STREAM FAILURE reading boolean." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -529,7 +539,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const data = integer; if(istr.fail()) { - llinfos << "STREAM FAILURE reading integer." << llendl; + LL_INFOS() << "STREAM FAILURE reading integer." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -543,7 +553,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const data = real; if(istr.fail()) { - llinfos << "STREAM FAILURE reading real." << llendl; + LL_INFOS() << "STREAM FAILURE reading real." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -557,7 +567,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const data = id; if(istr.fail()) { - llinfos << "STREAM FAILURE reading uuid." << llendl; + LL_INFOS() << "STREAM FAILURE reading uuid." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -572,7 +582,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading string." << llendl; + LL_INFOS() << "STREAM FAILURE reading string." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -594,7 +604,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading link." << llendl; + LL_INFOS() << "STREAM FAILURE reading link." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -617,7 +627,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading date." << llendl; + LL_INFOS() << "STREAM FAILURE reading date." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -630,15 +640,15 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading data." << llendl; + LL_INFOS() << "STREAM FAILURE reading data." << LL_ENDL; parse_count = PARSE_FAILURE; } break; default: parse_count = PARSE_FAILURE; - llinfos << "Unrecognized character while parsing: int(" << (int)c - << ")" << llendl; + LL_INFOS() << "Unrecognized character while parsing: int(" << (int)c + << ")" << LL_ENDL; break; } if(PARSE_FAILURE == parse_count) @@ -783,7 +793,7 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const if(len) { value.resize(len); - account(fullread(istr, (char *)&value[0], len)); + account((int)fullread(istr, (char *)&value[0], len)); } c = get(istr); // strip off the trailing double-quote data = value; @@ -863,7 +873,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const { /** * Undefined: '!'<br> - * Boolean: 't' for true 'f' for false<br> + * Boolean: '1' for true '0' for false<br> * Integer: 'i' + 4 bytes network byte order<br> * Real: 'r' + 8 bytes IEEE double<br> * UUID: 'u' + 16 byte unsigned integer<br> @@ -899,7 +909,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary map." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary map." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -918,7 +928,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary array." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary array." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -943,7 +953,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const data = (S32)ntohl(value_nbo); if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary integer." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary integer." << LL_ENDL; } break; } @@ -955,7 +965,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const data = ll_ntohd(real_nbo); if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary real." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary real." << LL_ENDL; } break; } @@ -967,7 +977,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const data = id; if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary uuid." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary uuid." << LL_ENDL; } break; } @@ -988,8 +998,8 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary (notation-style) string." - << llendl; + LL_INFOS() << "STREAM FAILURE reading binary (notation-style) string." + << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -1008,7 +1018,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary string." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary string." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -1027,7 +1037,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const } if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary link." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary link." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -1040,7 +1050,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const data = LLDate(real); if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary date." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary date." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -1063,13 +1073,13 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const if(size > 0) { value.resize(size); - account(fullread(istr, (char*)&value[0], size)); + account((int)fullread(istr, (char*)&value[0], size)); } data = value; } if(istr.fail()) { - llinfos << "STREAM FAILURE reading binary." << llendl; + LL_INFOS() << "STREAM FAILURE reading binary." << LL_ENDL; parse_count = PARSE_FAILURE; } break; @@ -1077,8 +1087,8 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const default: parse_count = PARSE_FAILURE; - llinfos << "Unrecognized character while parsing: int(" << (int)c - << ")" << llendl; + LL_INFOS() << "Unrecognized character while parsing: int(" << (int)c + << ")" << LL_ENDL; break; } if(PARSE_FAILURE == parse_count) @@ -1194,7 +1204,7 @@ bool LLSDBinaryParser::parseString( if(size) { buf.resize(size); - account(fullread(istr, &buf[0], size)); + account((int)fullread(istr, &buf[0], size)); value.assign(buf.begin(), buf.end()); } return true; @@ -1251,12 +1261,37 @@ std::string LLSDNotationFormatter::escapeString(const std::string& in) // virtual S32 LLSDNotationFormatter::format(const LLSD& data, std::ostream& ostr, U32 options) const { + S32 rv = format_impl(data, ostr, options, 0); + return rv; +} + +S32 LLSDNotationFormatter::format_impl(const LLSD& data, std::ostream& ostr, U32 options, U32 level) const +{ S32 format_count = 1; + std::string pre; + std::string post; + + if (options & LLSDFormatter::OPTIONS_PRETTY) + { + for (U32 i = 0; i < level; i++) + { + pre += " "; + } + post = "\n"; + } + switch(data.type()) { case LLSD::TypeMap: { + if (0 != level) ostr << post << pre; ostr << "{"; + std::string inner_pre; + if (options & LLSDFormatter::OPTIONS_PRETTY) + { + inner_pre = pre + " "; + } + bool need_comma = false; LLSD::map_const_iterator iter = data.beginMap(); LLSD::map_const_iterator end = data.endMap(); @@ -1264,18 +1299,18 @@ S32 LLSDNotationFormatter::format(const LLSD& data, std::ostream& ostr, U32 opti { if(need_comma) ostr << ","; need_comma = true; - ostr << '\''; + ostr << post << inner_pre << '\''; serialize_string((*iter).first, ostr); ostr << "':"; - format_count += format((*iter).second, ostr); + format_count += format_impl((*iter).second, ostr, options, level + 2); } - ostr << "}"; + ostr << post << pre << "}"; break; } case LLSD::TypeArray: { - ostr << "["; + ostr << post << pre << "["; bool need_comma = false; LLSD::array_const_iterator iter = data.beginArray(); LLSD::array_const_iterator end = data.endArray(); @@ -1283,7 +1318,7 @@ S32 LLSDNotationFormatter::format(const LLSD& data, std::ostream& ostr, U32 opti { if(need_comma) ostr << ","; need_comma = true; - format_count += format(*iter, ostr); + format_count += format_impl(*iter, ostr, options, level + 1); } ostr << "]"; break; @@ -1333,7 +1368,7 @@ S32 LLSDNotationFormatter::format(const LLSD& data, std::ostream& ostr, U32 opti case LLSD::TypeString: ostr << '\''; - serialize_string(data.asString(), ostr); + serialize_string(data.asStringRef(), ostr); ostr << '\''; break; @@ -1350,9 +1385,26 @@ S32 LLSDNotationFormatter::format(const LLSD& data, std::ostream& ostr, U32 opti case LLSD::TypeBinary: { // *FIX: memory inefficient. - std::vector<U8> buffer = data.asBinary(); + const std::vector<U8>& buffer = data.asBinary(); ostr << "b(" << buffer.size() << ")\""; - if(buffer.size()) ostr.write((const char*)&buffer[0], buffer.size()); + if(buffer.size()) + { + if (options & LLSDFormatter::OPTIONS_PRETTY_BINARY) + { + std::ios_base::fmtflags old_flags = ostr.flags(); + ostr.setf( std::ios::hex, std::ios::basefield ); + ostr << "0x"; + for (int i = 0; i < buffer.size(); i++) + { + ostr << (int) buffer[i]; + } + ostr.flags(old_flags); + } + else + { + ostr.write((const char*)&buffer[0], buffer.size()); + } + } ostr << "\""; break; } @@ -1441,13 +1493,16 @@ S32 LLSDBinaryFormatter::format(const LLSD& data, std::ostream& ostr, U32 option } case LLSD::TypeUUID: + { ostr.put('u'); - ostr.write((const char*)(&(data.asUUID().mData)), UUID_BYTES); + LLUUID temp = data.asUUID(); + ostr.write((const char*)(&(temp.mData)), UUID_BYTES); break; + } case LLSD::TypeString: ostr.put('s'); - formatString(data.asString(), ostr); + formatString(data.asStringRef(), ostr); break; case LLSD::TypeDate: @@ -1465,9 +1520,8 @@ S32 LLSDBinaryFormatter::format(const LLSD& data, std::ostream& ostr, U32 option case LLSD::TypeBinary: { - // *FIX: memory inefficient. ostr.put('b'); - std::vector<U8> buffer = data.asBinary(); + const std::vector<U8>& buffer = data.asBinary(); U32 size_nbo = htonl(buffer.size()); ostr.write((const char*)(&size_nbo), sizeof(U32)); if(buffer.size()) ostr.write((const char*)&buffer[0], buffer.size()); @@ -1636,7 +1690,7 @@ int deserialize_string_raw( const S32 BUF_LEN = 20; char buf[BUF_LEN]; /* Flawfinder: ignore */ istr.get(buf, BUF_LEN - 1, ')'); - count += istr.gcount(); + count += (int)istr.gcount(); int c = istr.get(); c = istr.get(); count += 2; @@ -1651,7 +1705,7 @@ int deserialize_string_raw( if(len) { buf.resize(len); - count += fullread(istr, (char *)&buf[0], len); + count += (int)fullread(istr, (char *)&buf[0], len); value.assign(buf.begin(), buf.end()); } c = istr.get(); @@ -1983,3 +2037,257 @@ std::ostream& operator<<(std::ostream& s, const LLSD& llsd) return s; } + +//dirty little zippers -- yell at davep if these are horrid + +//return a string containing gzipped bytes of binary serialized LLSD +// VERY inefficient -- creates several copies of LLSD block in memory +std::string zip_llsd(LLSD& data) +{ + std::stringstream llsd_strm; + + LLSDSerialize::toBinary(data, llsd_strm); + + const U32 CHUNK = 65536; + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + S32 ret = deflateInit(&strm, Z_BEST_COMPRESSION); + if (ret != Z_OK) + { + LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL; + return std::string(); + } + + std::string source = llsd_strm.str(); + + U8 out[CHUNK]; + + strm.avail_in = source.size(); + strm.next_in = (U8*) source.data(); + U8* output = NULL; + + U32 cur_size = 0; + + U32 have = 0; + + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + + ret = deflate(&strm, Z_FINISH); + if (ret == Z_OK || ret == Z_STREAM_END) + { //copy result into output + if (strm.avail_out >= CHUNK) + { + free(output); + LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL; + return std::string(); + } + + have = CHUNK-strm.avail_out; + output = (U8*) realloc(output, cur_size+have); + memcpy(output+cur_size, out, have); + cur_size += have; + } + else + { + free(output); + LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL; + return std::string(); + } + } + while (ret == Z_OK); + + std::string::size_type size = cur_size; + + std::string result((char*) output, size); + deflateEnd(&strm); + free(output); + +#if 0 //verify results work with unzip_llsd + std::istringstream test(result); + LLSD test_sd; + if (!unzip_llsd(test_sd, test, result.size())) + { + LL_ERRS() << "Invalid compression result!" << LL_ENDL; + } +#endif + + return result; +} + +//decompress a block of LLSD from provided istream +// not very efficient -- creats a copy of decompressed LLSD block in memory +// and deserializes from that copy using LLSDSerialize +bool unzip_llsd(LLSD& data, std::istream& is, S32 size) +{ + U8* result = NULL; + U32 cur_size = 0; + z_stream strm; + + const U32 CHUNK = 65536; + + U8 *in = new U8[size]; + is.read((char*) in, size); + + U8 out[CHUNK]; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = size; + strm.next_in = in; + + S32 ret = inflateInit(&strm); + + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR) + { + inflateEnd(&strm); + free(result); + delete [] in; + return false; + } + + switch (ret) + { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + inflateEnd(&strm); + free(result); + delete [] in; + return false; + break; + } + + U32 have = CHUNK-strm.avail_out; + + result = (U8*) realloc(result, cur_size + have); + memcpy(result+cur_size, out, have); + cur_size += have; + + } while (ret == Z_OK); + + inflateEnd(&strm); + delete [] in; + + if (ret != Z_STREAM_END) + { + free(result); + return false; + } + + //result now points to the decompressed LLSD block + { + std::string res_str((char*) result, cur_size); + + std::string deprecated_header("<? LLSD/Binary ?>"); + + if (res_str.substr(0, deprecated_header.size()) == deprecated_header) + { + res_str = res_str.substr(deprecated_header.size()+1, cur_size); + } + cur_size = res_str.size(); + + std::istringstream istr(res_str); + + if (!LLSDSerialize::fromBinary(data, istr, cur_size)) + { + LL_WARNS() << "Failed to unzip LLSD block" << LL_ENDL; + free(result); + return false; + } + } + + free(result); + return true; +} +//This unzip function will only work with a gzip header and trailer - while the contents +//of the actual compressed data is the same for either format (gzip vs zlib ), the headers +//and trailers are different for the formats. +U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 size ) +{ + U8* result = NULL; + U32 cur_size = 0; + z_stream strm; + + const U32 CHUNK = 0x4000; + + U8 *in = new U8[size]; + is.read((char*) in, size); + + U8 out[CHUNK]; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = size; + strm.next_in = in; + + + S32 ret = inflateInit2(&strm, windowBits | ENABLE_ZLIB_GZIP ); + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR) + { + inflateEnd(&strm); + free(result); + delete [] in; + valid = false; + } + + switch (ret) + { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + inflateEnd(&strm); + free(result); + delete [] in; + valid = false; + break; + } + + U32 have = CHUNK-strm.avail_out; + + result = (U8*) realloc(result, cur_size + have); + memcpy(result+cur_size, out, have); + cur_size += have; + + } while (ret == Z_OK); + + inflateEnd(&strm); + delete [] in; + + if (ret != Z_STREAM_END) + { + free(result); + valid = false; + return NULL; + } + + //result now points to the decompressed LLSD block + { + outsize= cur_size; + valid = true; + } + + return result; +} + + |