path: root/indra/llcommon/llsdserialize.cpp

/** 
 * @file llsdserialize.cpp
 * @author Phoenix
 * @date 2006-03-05
 * @brief Implementation of LLSD parsers and formatters
 *
 * $LicenseInfo:firstyear=2006&license=viewerlgpl$
 * Second Life Viewer Source Code
 * Copyright (C) 2010, Linden Research, Inc.
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation;
 * version 2.1 of the License only.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 * 
 * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
 * $/LicenseInfo$
 */

#include "linden_common.h"
#include "llsdserialize.h"
#include "llpointer.h"
#include "llstreamtools.h" // for fullread

#include <iostream>
#include "apr_base64.h"

#include <boost/iostreams/device/array.hpp>
#include <boost/iostreams/stream.hpp>

#ifdef LL_USESYSTEMLIBS
# include <zlib.h>
#else
# include "zlib-ng/zlib.h"  // for davep's dirty little zip functions
#endif

#if !LL_WINDOWS
#include <netinet/in.h> // htonl & ntohl
#endif

#include "lldate.h"
#include "llmemorystream.h"
#include "llsd.h"
#include "llstring.h"
#include "lluri.h"

// File constants
static const size_t MAX_HDR_LEN = 20;
static const S32 UNZIP_LLSD_MAX_DEPTH = 96;
static const char LEGACY_NON_HEADER[] = "<llsd>";
const std::string LLSD_BINARY_HEADER("LLSD/Binary");
const std::string LLSD_XML_HEADER("LLSD/XML");
const std::string LLSD_NOTATION_HEADER("llsd/notation");

//used to deflate a gzipped asset (currently used for navmeshes)
#define windowBits 15
#define ENABLE_ZLIB_GZIP 32

// If we published this in llsdserialize.h, we could use it in the
// implementation of LLSDOStreamer's operator<<().
template <class Formatter>
void format_using(const LLSD& data, std::ostream& ostr,
                  LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY)
{
    LLPointer<Formatter> f{ new Formatter };
    f->format(data, ostr, options);
}

template <class Parser>
S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1)
{
    LLPointer<Parser> p{ new Parser };
    return p->parse(istr, data, max_bytes, max_depth);
}

/**
 * LLSDSerialize
 */

// static
void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize type,
							  LLSDFormatter::EFormatterOptions options)
{
	LLPointer<LLSDFormatter> f = NULL;

	switch (type)
	{
	case LLSD_BINARY:
		str << "<? " << LLSD_BINARY_HEADER << " ?>\n";
		f = new LLSDBinaryFormatter;
		break;

	case LLSD_XML:
		str << "<? " << LLSD_XML_HEADER << " ?>\n";
		f = new LLSDXMLFormatter;
		break;

	case LLSD_NOTATION:
		str << "<? " << LLSD_NOTATION_HEADER << " ?>\n";
		f = new LLSDNotationFormatter;
		break;

	default:
		LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL;
	}

	if (f.notNull())
	{
		f->format(sd, str, options);
	}
}

// static
bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes)
{
	char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */
	bool fail_if_not_legacy = false;

	/*
	 * Get the first line before anything. Don't read more than max_bytes:
	 * this get() overload reads no more than (count-1) bytes into the
	 * specified buffer. In the usual case when max_bytes exceeds
	 * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2.
	 */
	str.get(hdr_buf, llmin(max_bytes+1, sizeof(hdr_buf)-1), '\n');
	auto inbuf = str.gcount();
	// https://en.cppreference.com/w/cpp/io/basic_istream/get
	// When the get() above sees the specified delimiter '\n', it stops there
	// without pulling it from the stream. If it turns out that the stream
	// does NOT contain a header, and the content includes meaningful '\n',
	// it's important to pull that into hdr_buf too.
	if (inbuf < max_bytes && str.get(hdr_buf[inbuf]))
	{
		// got the delimiting '\n'
		++inbuf;
		// None of the following requires that hdr_buf contain a final '\0'
		// byte. We could store one if needed, since even the incremented
		// inbuf won't exceed sizeof(hdr_buf)-1, but there's no need.
	}
	std::string header{ hdr_buf, static_cast<std::string::size_type>(inbuf) };
	if (str.fail())
	{
		str.clear();
		fail_if_not_legacy = true;
	}

	if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */
	{	// Create a LLSD XML parser, and parse the first chunk read above.
		LLSDXMLParser x;
		x.parsePart(hdr_buf, inbuf);	// Parse the first part that was already read
		auto parsed = x.parse(str, sd, max_bytes - inbuf); // Parse the rest of it
		// Formally we should probably check (parsed != PARSE_FAILURE &&
		// parsed > 0), but since PARSE_FAILURE is -1, this suffices.
		return (parsed > 0);
	}

	if (fail_if_not_legacy)
	{
		LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL;
		return false;
	}

	/*
	* Remove the newline chars
	*/
	std::string::size_type lastchar = header.find_last_not_of("\r\n");
	if (lastchar != std::string::npos)
	{
		// It's important that find_last_not_of() returns size_type, which is
		// why lastchar explicitly declares the type above. erase(size_type)
		// erases from that offset to the end of the string, whereas
		// erase(iterator) erases only a single character.
		header.erase(lastchar+1);
	}

	// trim off the <? ... ?> header syntax
	auto start = header.find_first_not_of("<? ");
	if (start != std::string::npos)
	{
		auto end = header.find_first_of(" ?", start);
		if (end != std::string::npos)
		{
			header = header.substr(start, end - start);
			ws(str);
		}
	}
	/*
	 * Create the parser as appropriate
	 */
	if (0 == LLStringUtil::compareInsensitive(header, LLSD_BINARY_HEADER))
	{
		return (parse_using<LLSDBinaryParser>(str, sd, max_bytes-inbuf) > 0);
	}
	else if (0 == LLStringUtil::compareInsensitive(header, LLSD_XML_HEADER))
	{
		return (parse_using<LLSDXMLParser>(str, sd, max_bytes-inbuf) > 0);
	}
	else if (0 == LLStringUtil::compareInsensitive(header, LLSD_NOTATION_HEADER))
	{
		return (parse_using<LLSDNotationParser>(str, sd, max_bytes-inbuf) > 0);
	}
	else // no header we recognize
	{
		LLPointer<LLSDParser> p;
		if (inbuf && hdr_buf[0] == '<')
		{
			// looks like XML
			LL_DEBUGS() << "deserialize request with no header, assuming XML" << LL_ENDL;
			p = new LLSDXMLParser;
		}
		else
		{
			// assume notation
			LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL;
			p = new LLSDNotationParser;
		}
		// Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that
		// data to whatever remains in 'str'.
		LLMemoryStreamBuf already(reinterpret_cast<const U8*>(hdr_buf), inbuf);
		cat_streambuf prebuff(&already, str.rdbuf());
		std::istream  prepend(&prebuff);
#if 1
		return (p->parse(prepend, sd, max_bytes) > 0);
#else
		// debugging the reconstituted 'prepend' stream
		// allocate a buffer that we hope is big enough for the whole thing
		std::vector<char> wholemsg((max_bytes == size_t(SIZE_UNLIMITED))? 1024 : max_bytes);
		prepend.read(wholemsg.data(), std::min(max_bytes, wholemsg.size()));
		LLMemoryStream replay(reinterpret_cast<const U8*>(wholemsg.data()), prepend.gcount());
		auto success{ p->parse(replay, sd, prepend.gcount()) > 0 };
		{
			LL_DEBUGS() << (success? "parsed: $$" : "failed: '")
						<< std::string(wholemsg.data(), llmin(prepend.gcount(), 100)) << "$$"
						<< LL_ENDL;
		}
		return success;
#endif
	}
}

/**
 * Endian handlers
 */
#if LL_BIG_ENDIAN
U64 ll_htonll(U64 hostlonglong) { return hostlonglong; }
U64 ll_ntohll(U64 netlonglong) { return netlonglong; }
F64 ll_htond(F64 hostlonglong) { return hostlonglong; }
F64 ll_ntohd(F64 netlonglong) { return netlonglong; }
#else
// I read some comments one a indicating that doing an integer add
// here would be faster than a bitwise or. For now, the or has
// programmer clarity, since the intended outcome matches the
// operation.
U64 ll_htonll(U64 hostlonglong)
{
	return ((U64)(htonl((U32)((hostlonglong >> 32) & 0xFFFFFFFF))) |
			((U64)(htonl((U32)(hostlonglong & 0xFFFFFFFF))) << 32));
}
U64 ll_ntohll(U64 netlonglong)
{
	return ((U64)(ntohl((U32)((netlonglong >> 32) & 0xFFFFFFFF))) |
			((U64)(ntohl((U32)(netlonglong & 0xFFFFFFFF))) << 32));
}
union LLEndianSwapper
{
	F64 d;
	U64 i;
};
F64 ll_htond(F64 hostdouble)
{
	LLEndianSwapper tmp;
	tmp.d = hostdouble;
	tmp.i = ll_htonll(tmp.i);
	return tmp.d;
}
F64 ll_ntohd(F64 netdouble)
{
	LLEndianSwapper tmp;
	tmp.d = netdouble;
	tmp.i = ll_ntohll(tmp.i);
	return tmp.d;
}
#endif

/**
 * Local functions.
 */
/**
 * @brief Figure out what kind of string it is (raw or delimited) and handoff.
 *
 * @param istr The stream to read from.
 * @param value [out] The string which was found.
 * @param max_bytes The maximum possible length of the string. Passing in
 * a negative value will skip this check.
 * @return Returns number of bytes read off of the stream. Returns
 * PARSE_FAILURE (-1) on failure.
 */
llssize deserialize_string(std::istream& istr, std::string& value, llssize max_bytes);

/**
 * @brief Parse a delimited string. 
 *
 * @param istr The stream to read from, with the delimiter already popped.
 * @param value [out] The string which was found.
 * @param d The delimiter to use.
 * @return Returns number of bytes read off of the stream. Returns
 * PARSE_FAILURE (-1) on failure.
 */
llssize deserialize_string_delim(std::istream& istr, std::string& value, char d);

/**
 * @brief Read a raw string off the stream.
 *
 * @param istr The stream to read from, with the (len) parameter
 * leading the stream.
 * @param value [out] The string which was found.
 * @param d The delimiter to use.
 * @param max_bytes The maximum possible length of the string. Passing in
 * a negative value will skip this check.
 * @return Returns number of bytes read off of the stream. Returns
 * PARSE_FAILURE (-1) on failure.
 */
llssize deserialize_string_raw(
	std::istream& istr,
	std::string& value,
	llssize max_bytes);

/**
 * @brief helper method for dealing with the different notation boolean format.
 *
 * @param istr The stream to read from with the leading character stripped.
 * @param data [out] the result of the parse.
 * @param compare The string to compare the boolean against
 * @param vale The value to assign to data if the parse succeeds.
 * @return Returns number of bytes read off of the stream. Returns
 * PARSE_FAILURE (-1) on failure.
 */
llssize deserialize_boolean(
	std::istream& istr,
	LLSD& data,
	const std::string& compare,
	bool value);

/**
 * @brief Do notation escaping of a string to an ostream.
 *
 * @param value The string to escape and serialize
 * @param str The stream to serialize to.
 */
void serialize_string(const std::string& value, std::ostream& str);


/**
 * Local constants.
 */
static const std::string NOTATION_TRUE_SERIAL("true");
static const std::string NOTATION_FALSE_SERIAL("false");

static const char BINARY_TRUE_SERIAL = '1';
static const char BINARY_FALSE_SERIAL = '0';


/**
 * LLSDParser
 */
LLSDParser::LLSDParser()
	: mCheckLimits(true), mMaxBytesLeft(0), mParseLines(false)
{
}

// virtual
LLSDParser::~LLSDParser()
{ }

S32 LLSDParser::parse(std::istream& istr, LLSD& data, llssize max_bytes, S32 max_depth)
{
	mCheckLimits = (LLSDSerialize::SIZE_UNLIMITED == max_bytes) ? false : true;
	mMaxBytesLeft = max_bytes;
	return doParse(istr, data, max_depth);
}


// Parse using routine to get() lines, faster than parse()
S32 LLSDParser::parseLines(std::istream& istr, LLSD& data)
{
	mCheckLimits = false;
	mParseLines = true;
	return doParse(istr, data);
}


int LLSDParser::get(std::istream& istr) const
{
	if(mCheckLimits) --mMaxBytesLeft;
	return istr.get();
}

std::istream& LLSDParser::get(
	std::istream& istr,
	char* s,
	std::streamsize n,
	char delim) const
{
	istr.get(s, n, delim);
	if(mCheckLimits) mMaxBytesLeft -= istr.gcount();
	return istr;
}

std::istream& LLSDParser::get(
		std::istream& istr,
		std::streambuf& sb,
		char delim) const		
{
	istr.get(sb, delim);
	if(mCheckLimits) mMaxBytesLeft -= istr.gcount();
	return istr;
}

std::istream& LLSDParser::ignore(std::istream& istr) const
{
	istr.ignore();
	if(mCheckLimits) --mMaxBytesLeft;
	return istr;
}

std::istream& LLSDParser::putback(std::istream& istr, char c) const
{
	istr.putback(c);
	if(mCheckLimits) ++mMaxBytesLeft;
	return istr;
}

std::istream& LLSDParser::read(
	std::istream& istr,
	char* s,
	std::streamsize n) const
{
	istr.read(s, n);
	if(mCheckLimits) mMaxBytesLeft -= istr.gcount();
	return istr;
}

void LLSDParser::account(llssize bytes) const
{
	if(mCheckLimits) mMaxBytesLeft -= bytes;
}


/**
 * LLSDNotationParser
 */
LLSDNotationParser::LLSDNotationParser()
{
}	

// virtual
LLSDNotationParser::~LLSDNotationParser()
{ }

// virtual
S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) const
{
	// map: { string:object, string:object }
	// array: [ object, object, object ]
	// undef: !
	// boolean: true | false | 1 | 0 | T | F | t | f | TRUE | FALSE
	// integer: i####
	// real: r####
	// uuid: u####
	// string: "g'day" | 'have a "nice" day' | s(size)"raw data"
	// uri: l"escaped"
	// date: d"YYYY-MM-DDTHH:MM:SS.FFZ"
	// binary: b##"ff3120ab1" | b(size)"raw data"
	char c;
	c = istr.peek();
	if (max_depth == 0)
	{
		return PARSE_FAILURE;
	}
	while(isspace(c))
	{
		// pop the whitespace.
		c = get(istr);
		c = istr.peek();
		continue;
	}
	if(!istr.good())
	{
		return 0;
	}
	S32 parse_count = 1;
	switch(c)
	{
	case '{':
	{
		S32 child_count = parseMap(istr, data, max_depth - 1);
		if((child_count == PARSE_FAILURE) || data.isUndefined())
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			parse_count += child_count;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading map." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case '[':
	{
		S32 child_count = parseArray(istr, data, max_depth - 1);
		if((child_count == PARSE_FAILURE) || data.isUndefined())
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			parse_count += child_count;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading array." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case '!':
		c = get(istr);
		data.clear();
		break;

	case '0':
		c = get(istr);
		data = false;
		break;

	case 'F':
	case 'f':
		ignore(istr);
		c = istr.peek();
		if(isalpha(c))
		{
			auto cnt = deserialize_boolean(
				istr,
				data,
				NOTATION_FALSE_SERIAL,
				false);
			if(PARSE_FAILURE == cnt) parse_count = cnt;
			else account(cnt);
		}
		else
		{
			data = false;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading boolean." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;

	case '1':
		c = get(istr);
		data = true;
		break;

	case 'T':
	case 't':
		ignore(istr);
		c = istr.peek();
		if(isalpha(c))
		{
			auto cnt = deserialize_boolean(istr,data,NOTATION_TRUE_SERIAL,true);
			if(PARSE_FAILURE == cnt) parse_count = cnt;
			else account(cnt);
		}
		else
		{
			data = true;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading boolean." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;

	case 'i':
	{
		c = get(istr);
		S32 integer = 0;
		istr >> integer;
		data = integer;
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading integer." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'r':
	{
		c = get(istr);
		F64 real = 0.0;
		istr >> real;
		data = real;
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading real." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'u':
	{
		c = get(istr);
		LLUUID id;
		istr >> id;
		data = id;
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading uuid." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case '\"':
	case '\'':
	case 's':
		if(!parseString(istr, data))
		{
			parse_count = PARSE_FAILURE;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading string." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;

	case 'l':
	{
		c = get(istr); // pop the 'l'
		c = get(istr); // pop the delimiter
		std::string str;
		auto cnt = deserialize_string_delim(istr, str, c);
		if(PARSE_FAILURE == cnt)
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			data = LLURI(str);
			account(cnt);
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading link." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'd':
	{
		c = get(istr); // pop the 'd'
		c = get(istr); // pop the delimiter
		std::string str;
		auto cnt = deserialize_string_delim(istr, str, c);
		if(PARSE_FAILURE == cnt)
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			data = LLDate(str);
			account(cnt);
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading date." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'b':
		if(!parseBinary(istr, data))
		{
			parse_count = PARSE_FAILURE;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading data." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;

	default:
		parse_count = PARSE_FAILURE;
		LL_INFOS() << "Unrecognized character while parsing: int(" << int(c)
			<< ")" << LL_ENDL;
		break;
	}
	if(PARSE_FAILURE == parse_count)
	{
		data.clear();
	}
	return parse_count;
}

S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) const
{
	// map: { string:object, string:object }
	map = LLSD::emptyMap();
	S32 parse_count = 0;
	char c = get(istr);
	if(c == '{')
	{
		// eat commas, white
		bool found_name = false;
		std::string name;
		c = get(istr);
		while(c != '}' && istr.good())
		{
			if(!found_name)
			{
				if((c == '\"') || (c == '\'') || (c == 's'))
				{
					putback(istr, c);
					found_name = true;
					auto count = deserialize_string(istr, name, mMaxBytesLeft);
					if(PARSE_FAILURE == count) return PARSE_FAILURE;
					account(count);
				}
				c = get(istr);
			}
			else
			{
				if(isspace(c) || (c == ':'))
				{
					c = get(istr);
					continue;
				}
				putback(istr, c);
				LLSD child;
				S32 count = doParse(istr, child, max_depth);
				if(count > 0)
				{
					// There must be a value for every key, thus
					// child_count must be greater than 0.
					parse_count += count;
					map.insert(name, child);
				}
				else
				{
					return PARSE_FAILURE;
				}
				found_name = false;
				c = get(istr);
			}
		}
		if(c != '}')
		{
			map.clear();
			return PARSE_FAILURE;
		}
	}
	return parse_count;
}

S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth) const
{
	// array: [ object, object, object ]
	array = LLSD::emptyArray();
	S32 parse_count = 0;
	char c = get(istr);
	if(c == '[')
	{
		// eat commas, white
		c = get(istr);
		while((c != ']') && istr.good())
		{
			LLSD child;
			if(isspace(c) || (c == ','))
			{
				c = get(istr);
				continue;
			}
			putback(istr, c);
			S32 count = doParse(istr, child, max_depth);
			if(PARSE_FAILURE == count)
			{
				return PARSE_FAILURE;
			}
			else
			{
				parse_count += count;
				array.append(child);
			}
			c = get(istr);
		}
		if(c != ']')
		{
			return PARSE_FAILURE;
		}
	}
	return parse_count;
}

bool LLSDNotationParser::parseString(std::istream& istr, LLSD& data) const
{
	std::string value;
	auto count = deserialize_string(istr, value, mMaxBytesLeft);
	if(PARSE_FAILURE == count) return false;
	account(count);
	data = value;
	return true;
}

bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const
{
	// binary: b##"ff3120ab1"
	// or: b(len)"..."

	// I want to manually control those values here to make sure the
	// parser doesn't break when someone changes a constant somewhere
	// else.
	const U32 BINARY_BUFFER_SIZE = 256;
	const U32 STREAM_GET_COUNT = 255;

	// need to read the base out.
	char buf[BINARY_BUFFER_SIZE];		/* Flawfinder: ignore */
	get(istr, buf, STREAM_GET_COUNT, '"');
	char c = get(istr);
	if(c != '"') return false;
	if(0 == strncmp("b(", buf, 2))
	{
		// We probably have a valid raw binary stream. determine
		// the size, and read it.
		auto len = strtol(buf + 2, NULL, 0);
		if(mCheckLimits && (len > mMaxBytesLeft)) return false;
		std::vector<U8> value;
		if(len)
		{
			value.resize(len);
			account(fullread(istr, (char *)&value[0], len));
		}
		c = get(istr); // strip off the trailing double-quote
		data = value;
	}
	else if(0 == strncmp("b64", buf, 3))
	{
		// *FIX: A bit inefficient, but works for now. To make the
		// format better, I would need to add a hint into the
		// serialization format that indicated how long it was.
		std::stringstream coded_stream;
		get(istr, *(coded_stream.rdbuf()), '\"');
		c = get(istr);
		std::string encoded(coded_stream.str());
		S32 len = apr_base64_decode_len(encoded.c_str());
		std::vector<U8> value;
		if(len)
		{
			value.resize(len);
			len = apr_base64_decode_binary(&value[0], encoded.c_str());
			value.resize(len);
		}
		data = value;
	}
	else if(0 == strncmp("b16", buf, 3))
	{
		// yay, base 16. We pop the next character which is either a
		// double quote or base 16 data. If it's a double quote, we're
		// done parsing. If it's not, put the data back, and read the
		// stream until the next double quote.
		char* read;	 /*Flawfinder: ignore*/
		U8 byte;
		U8 byte_buffer[BINARY_BUFFER_SIZE];
		U8* write;
		std::vector<U8> value;
		c = get(istr);
		while(c != '"')
		{
			putback(istr, c);
			read = buf;
			write = byte_buffer;
			get(istr, buf, STREAM_GET_COUNT, '"');
			c = get(istr);
			while(*read != '\0')	 /*Flawfinder: ignore*/
			{
				byte = hex_as_nybble(*read++);
				byte = byte << 4;
				byte |= hex_as_nybble(*read++);
				*write++ = byte;
			}
			// copy the data out of the byte buffer
			value.insert(value.end(), byte_buffer, write);
		}
		data = value;
	}
	else
	{
		return false;
	}
	return true;
}


/**
 * LLSDBinaryParser
 */
LLSDBinaryParser::LLSDBinaryParser()
{
}

// virtual
LLSDBinaryParser::~LLSDBinaryParser()
{
}

// virtual
S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) const
{
/**
 * Undefined: '!'<br>
 * Boolean: '1' for true '0' for false<br>
 * Integer: 'i' + 4 bytes network byte order<br>
 * Real: 'r' + 8 bytes IEEE double<br>
 * UUID: 'u' + 16 byte unsigned integer<br>
 * String: 's' + 4 byte integer size + string<br>
 *  strings also secretly support the notation format
 * Date: 'd' + 8 byte IEEE double for seconds since epoch<br>
 * URI: 'l' + 4 byte integer size + string uri<br>
 * Binary: 'b' + 4 byte integer size + binary data<br>
 * Array: '[' + 4 byte integer size  + all values + ']'<br>
 * Map: '{' + 4 byte integer size  every(key + value) + '}'<br>
 *  map keys are serialized as s + 4 byte integer size + string or in the
 *  notation format.
 */
	char c;
	c = get(istr);
	if(!istr.good())
	{
		return 0;
	}
	if (max_depth == 0)
	{
		return PARSE_FAILURE;
	}
	S32 parse_count = 1;
	switch(c)
	{
	case '{':
	{
		S32 child_count = parseMap(istr, data, max_depth - 1);
		if((child_count == PARSE_FAILURE) || data.isUndefined())
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			parse_count += child_count;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary map." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case '[':
	{
		S32 child_count = parseArray(istr, data, max_depth - 1);
		if((child_count == PARSE_FAILURE) || data.isUndefined())
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			parse_count += child_count;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary array." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case '!':
		data.clear();
		break;

	case '0':
		data = false;
		break;

	case '1':
		data = true;
		break;

	case 'i':
	{
		U32 value_nbo = 0;
		read(istr, (char*)&value_nbo, sizeof(U32));	 /*Flawfinder: ignore*/
		data = (S32)ntohl(value_nbo);
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary integer." << LL_ENDL;
		}
		break;
	}

	case 'r':
	{
		F64 real_nbo = 0.0;
		read(istr, (char*)&real_nbo, sizeof(F64));	 /*Flawfinder: ignore*/
		data = ll_ntohd(real_nbo);
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary real." << LL_ENDL;
		}
		break;
	}

	case 'u':
	{
		LLUUID id;
		read(istr, (char*)(&id.mData), UUID_BYTES);	 /*Flawfinder: ignore*/
		data = id;
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary uuid." << LL_ENDL;
		}
		break;
	}

	case '\'':
	case '"':
	{
		std::string value;
		auto cnt = deserialize_string_delim(istr, value, c);
		if(PARSE_FAILURE == cnt)
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			data = value;
			account(cnt);
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary (notation-style) string."
				<< LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 's':
	{
		std::string value;
		if(parseString(istr, value))
		{
			data = value;
		}
		else
		{
			parse_count = PARSE_FAILURE;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary string." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'l':
	{
		std::string value;
		if(parseString(istr, value))
		{
			data = LLURI(value);
		}
		else
		{
			parse_count = PARSE_FAILURE;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary link." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'd':
	{
		F64 real = 0.0;
		read(istr, (char*)&real, sizeof(F64));	 /*Flawfinder: ignore*/
		data = LLDate(real);
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary date." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	case 'b':
	{
		// We probably have a valid raw binary stream. determine
		// the size, and read it.
		U32 size_nbo = 0;
		read(istr, (char*)&size_nbo, sizeof(U32));	/*Flawfinder: ignore*/
		S32 size = (S32)ntohl(size_nbo);
		if(mCheckLimits && (size > mMaxBytesLeft))
		{
			parse_count = PARSE_FAILURE;
		}
		else
		{
			std::vector<U8> value;
			if(size > 0)
			{
				value.resize(size);
				account(fullread(istr, (char*)&value[0], size));
			}
			data = value;
		}
		if(istr.fail())
		{
			LL_INFOS() << "STREAM FAILURE reading binary." << LL_ENDL;
			parse_count = PARSE_FAILURE;
		}
		break;
	}

	default:
		parse_count = PARSE_FAILURE;
		LL_INFOS() << "Unrecognized character while parsing: int(" << int(c)
			<< ")" << LL_ENDL;
		break;
	}
	if(PARSE_FAILURE == parse_count)
	{
		data.clear();
	}
	return parse_count;
}

S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) const
{
	map = LLSD::emptyMap();
	U32 value_nbo = 0;
	read(istr, (char*)&value_nbo, sizeof(U32));		 /*Flawfinder: ignore*/
	S32 size = (S32)ntohl(value_nbo);
	S32 parse_count = 0;
	S32 count = 0;
	char c = get(istr);
	while(c != '}' && (count < size) && istr.good())
	{
		std::string name;
		switch(c)
		{
		case 'k':
			if(!parseString(istr, name))
			{
				return PARSE_FAILURE;
			}
			break;
		case '\'':
		case '"':
		{
			auto cnt = deserialize_string_delim(istr, name, c);
			if(PARSE_FAILURE == cnt) return PARSE_FAILURE;
			account(cnt);
			break;
		}
		}
		LLSD child;
		S32 child_count = doParse(istr, child, max_depth);
		if(child_count > 0)
		{
			// There must be a value for every key, thus child_count
			// must be greater than 0.
			parse_count += child_count;
			map.insert(name, child);
		}
		else
		{
			return PARSE_FAILURE;
		}
		++count;
		c = get(istr);
	}
	if((c != '}') || (count < size))
	{
		// Make sure it is correctly terminated and we parsed as many
		// as were said to be there.
		return PARSE_FAILURE;
	}
	return parse_count;
}

S32 LLSDBinaryParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth) const
{
	array = LLSD::emptyArray();
	U32 value_nbo = 0;
	read(istr, (char*)&value_nbo, sizeof(U32));		 /*Flawfinder: ignore*/
	S32 size = (S32)ntohl(value_nbo);

	// *FIX: This would be a good place to reserve some space in the
	// array...

	S32 parse_count = 0;
	S32 count = 0;
	char c = istr.peek();
	while((c != ']') && (count < size) && istr.good())
	{
		LLSD child;
		S32 child_count = doParse(istr, child, max_depth);
		if(PARSE_FAILURE == child_count)
		{
			return PARSE_FAILURE;
		}
		if(child_count)
		{
			parse_count += child_count;
			array.append(child);
		}
		++count;
		c = istr.peek();
	}
	c = get(istr);
	if((c != ']') || (count < size))
	{
		// Make sure it is correctly terminated and we parsed as many
		// as were said to be there.
		return PARSE_FAILURE;
	}
	return parse_count;
}

bool LLSDBinaryParser::parseString(
	std::istream& istr,
	std::string& value) const
{
	// *FIX: This is memory inefficient.
	U32 value_nbo = 0;
	read(istr, (char*)&value_nbo, sizeof(U32));		 /*Flawfinder: ignore*/
	S32 size = (S32)ntohl(value_nbo);
	if(mCheckLimits && (size > mMaxBytesLeft)) return false;
	if(size < 0) return false;
	std::vector<char> buf;
	if(size)
	{
		buf.resize(size);
		account(fullread(istr, &buf[0], size));
		value.assign(buf.begin(), buf.end());
	}
	return true;
}


/**
 * LLSDFormatter
 */
LLSDFormatter::LLSDFormatter(bool boolAlpha, const std::string& realFmt, EFormatterOptions options):
    mOptions(options)
{
    boolalpha(boolAlpha);
    realFormat(realFmt);
}

// virtual
LLSDFormatter::~LLSDFormatter()
{ }

void LLSDFormatter::boolalpha(bool alpha)
{
	mBoolAlpha = alpha;
}

void LLSDFormatter::realFormat(const std::string& format)
{
	mRealFormat = format;
}

S32 LLSDFormatter::format(const LLSD& data, std::ostream& ostr) const
{
    // pass options captured by constructor
    return format(data, ostr, mOptions);
}

S32 LLSDFormatter::format(const LLSD& data, std::ostream& ostr, EFormatterOptions options) const
{
    return format_impl(data, ostr, options, 0);
}

void LLSDFormatter::formatReal(LLSD::Real real, std::ostream& ostr) const
{
	std::string buffer = llformat(mRealFormat.c_str(), real);
	ostr << buffer;
}

/**
 * LLSDNotationFormatter
 */
LLSDNotationFormatter::LLSDNotationFormatter(bool boolAlpha, const std::string& realFormat,
                                             EFormatterOptions options):
    LLSDFormatter(boolAlpha, realFormat, options)
{
}

// virtual
LLSDNotationFormatter::~LLSDNotationFormatter()
{ }

// static
std::string LLSDNotationFormatter::escapeString(const std::string& in)
{
	std::ostringstream ostr;
	serialize_string(in, ostr);
	return ostr.str();
}

S32 LLSDNotationFormatter::format_impl(const LLSD& data, std::ostream& ostr,
									   EFormatterOptions options, U32 level) const
{
	S32 format_count = 1;
	std::string pre;
	std::string post;

	if (options & LLSDFormatter::OPTIONS_PRETTY)
	{
		for (U32 i = 0; i < level; i++)
		{
			pre += "    ";
		}
		post = "\n";
	}

	switch(data.type())
	{
	case LLSD::TypeMap:
	{
		if (0 != level) ostr << post << pre;
		ostr << "{";
		std::string inner_pre;
		if (options & LLSDFormatter::OPTIONS_PRETTY)
		{
			inner_pre = pre + "    ";
		}

		bool need_comma = false;
		LLSD::map_const_iterator iter = data.beginMap();
		LLSD::map_const_iterator end = data.endMap();
		for(; iter != end; ++iter)
		{
			if(need_comma) ostr << ",";
			need_comma = true;
			ostr << post << inner_pre << '\'';
			serialize_string((*iter).first, ostr);
			ostr << "':";
			format_count += format_impl((*iter).second, ostr, options, level + 2);
		}
		ostr << post << pre << "}";
		break;
	}

	case LLSD::TypeArray:
	{
		ostr << post << pre << "[";
		bool need_comma = false;
		LLSD::array_const_iterator iter = data.beginArray();
		LLSD::array_const_iterator end = data.endArray();
		for(; iter != end; ++iter)
		{
			if(need_comma) ostr << ",";
			need_comma = true;
			format_count += format_impl(*iter, ostr, options, level + 1);
		}
		ostr << "]";
		break;
	}

	case LLSD::TypeUndefined:
		ostr << "!";
		break;

	case LLSD::TypeBoolean:
		if(mBoolAlpha ||
#if( LL_WINDOWS || __GNUC__ > 2)
		   (ostr.flags() & std::ios::boolalpha)
#else
		   (ostr.flags() & 0x0100)
#endif
			)
		{
			ostr << (data.asBoolean()
					 ? NOTATION_TRUE_SERIAL : NOTATION_FALSE_SERIAL);
		}
		else
		{
			ostr << (data.asBoolean() ? 1 : 0);
		}
		break;

	case LLSD::TypeInteger:
		ostr << "i" << data.asInteger();
		break;

	case LLSD::TypeReal:
		ostr << "r";
		if(mRealFormat.empty())
		{
			ostr << data.asReal();
		}
		else
		{
			formatReal(data.asReal(), ostr);
		}
		break;

	case LLSD::TypeUUID:
		ostr << "u" << data.asUUID();
		break;

	case LLSD::TypeString:
		ostr << '\'';
		serialize_string(data.asStringRef(), ostr);
		ostr << '\'';
		break;

	case LLSD::TypeDate:
		ostr << "d\"" << data.asDate() << "\"";
		break;

	case LLSD::TypeURI:
		ostr << "l\"";
		serialize_string(data.asString(), ostr);
		ostr << "\"";
		break;

	case LLSD::TypeBinary:
	{
		// *FIX: memory inefficient.
		const std::vector<U8>& buffer = data.asBinary();
		if (options & LLSDFormatter::OPTIONS_PRETTY_BINARY)
		{
			ostr << "b16\"";
			if (! buffer.empty())
			{
				std::ios_base::fmtflags old_flags = ostr.flags();
				ostr.setf( std::ios::hex, std::ios::basefield );
				// It shouldn't strictly matter whether the emitted hex digits
				// are uppercase; LLSDNotationParser handles either; but as of
				// 2020-05-13, Python's llbase.llsd requires uppercase hex.
				ostr << std::uppercase;
				auto oldfill(ostr.fill('0'));
				auto oldwidth(ostr.width());
				for (size_t i = 0; i < buffer.size(); i++)
				{
					// have to restate setw() before every conversion
					ostr << std::setw(2) << (int) buffer[i];
				}
				ostr.width(oldwidth);
				ostr.fill(oldfill);
				ostr.flags(old_flags);
			}
		}
		else                        // ! OPTIONS_PRETTY_BINARY
		{
			ostr << "b(" << buffer.size() << ")\"";
			if (! buffer.empty())
			{
				ostr.write((const char*)&buffer[0], buffer.size());
			}
		}
		ostr << "\"";
		break;
	}

	default:
		// *NOTE: This should never happen.
		ostr << "!";
		break;
	}
	return format_count;
}

/**
 * LLSDBinaryFormatter
 */
LLSDBinaryFormatter::LLSDBinaryFormatter(bool boolAlpha, const std::string& realFormat,
                                         EFormatterOptions options):
    LLSDFormatter(boolAlpha, realFormat, options)
{
}

// virtual
LLSDBinaryFormatter::~LLSDBinaryFormatter()
{ }

// virtual
S32 LLSDBinaryFormatter::format_impl(const LLSD& data, std::ostream& ostr,
									 EFormatterOptions options, U32 level) const
{
	S32 format_count = 1;
	switch(data.type())
	{
	case LLSD::TypeMap:
	{
		ostr.put('{');
		U32 size_nbo = htonl(data.size());
		ostr.write((const char*)(&size_nbo), sizeof(U32));
		LLSD::map_const_iterator iter = data.beginMap();
		LLSD::map_const_iterator end = data.endMap();
		for(; iter != end; ++iter)
		{
			ostr.put('k');
			formatString((*iter).first, ostr);
			format_count += format_impl((*iter).second, ostr, options, level+1);
		}
		ostr.put('}');
		break;
	}

	case LLSD::TypeArray:
	{
		ostr.put('[');
		U32 size_nbo = htonl(data.size());
		ostr.write((const char*)(&size_nbo), sizeof(U32));
		LLSD::array_const_iterator iter = data.beginArray();
		LLSD::array_const_iterator end = data.endArray();
		for(; iter != end; ++iter)
		{
			format_count += format_impl(*iter, ostr, options, level+1);
		}
		ostr.put(']');
		break;
	}

	case LLSD::TypeUndefined:
		ostr.put('!');
		break;

	case LLSD::TypeBoolean:
		if(data.asBoolean()) ostr.put(BINARY_TRUE_SERIAL);
		else ostr.put(BINARY_FALSE_SERIAL);
		break;

	case LLSD::TypeInteger:
	{
		ostr.put('i');
		U32 value_nbo = htonl(data.asInteger());
		ostr.write((const char*)(&value_nbo), sizeof(U32));
		break;
	}

	case LLSD::TypeReal:
	{
		ostr.put('r');
		F64 value_nbo = ll_htond(data.asReal());
		ostr.write((const char*)(&value_nbo), sizeof(F64));
		break;
	}

	case LLSD::TypeUUID:
	{
		ostr.put('u');
		LLUUID temp = data.asUUID();
		ostr.write((const char*)(&(temp.mData)), UUID_BYTES);
		break;
	}

	case LLSD::TypeString:
		ostr.put('s');
		formatString(data.asStringRef(), ostr);
		break;

	case LLSD::TypeDate:
	{
		ostr.put('d');
		F64 value = data.asReal();
		ostr.write((const char*)(&value), sizeof(F64));
		break;
	}

	case LLSD::TypeURI:
		ostr.put('l');
		formatString(data.asString(), ostr);
		break;

	case LLSD::TypeBinary:
	{
		ostr.put('b');
		const std::vector<U8>& buffer = data.asBinary();
		U32 size_nbo = htonl(buffer.size());
		ostr.write((const char*)(&size_nbo), sizeof(U32));
		if(buffer.size()) ostr.write((const char*)&buffer[0], buffer.size());
		break;
	}

	default:
		// *NOTE: This should never happen.
		ostr.put('!');
		break;
	}
	return format_count;
}

void LLSDBinaryFormatter::formatString(
	const std::string& string,
	std::ostream& ostr) const
{
	U32 size_nbo = htonl(string.size());
	ostr.write((const char*)(&size_nbo), sizeof(U32));
	ostr.write(string.c_str(), string.size());
}

/**
 * local functions
 */
llssize deserialize_string(std::istream& istr, std::string& value, llssize max_bytes)
{
	int c = istr.get();
	if(istr.fail())
	{
		// No data in stream, bail out but mention the character we
		// grabbed.
		return LLSDParser::PARSE_FAILURE;
	}

	llssize rv = LLSDParser::PARSE_FAILURE;
	switch(c)
	{
	case '\'':
	case '"':
		rv = deserialize_string_delim(istr, value, c);
		break;
	case 's':
		// technically, less than max_bytes, but this is just meant to
		// catch egregious protocol errors. parse errors will be
		// caught in the case of incorrect counts.
		rv = deserialize_string_raw(istr, value, max_bytes);
		break;
	default:
		break;
	}
	if(LLSDParser::PARSE_FAILURE == rv) return rv;
	return rv + 1; // account for the character grabbed at the top.
}

llssize deserialize_string_delim(
	std::istream& istr,
	std::string& value,
	char delim)
{
	std::ostringstream write_buffer;
	bool found_escape = false;
	bool found_hex = false;
	bool found_digit = false;
	U8 byte = 0;
	llssize count = 0;

	while (true)
	{
		int next_byte = istr.get();
		++count;

		if(istr.fail())
		{
			// If our stream is empty, break out
			value = write_buffer.str();
			return LLSDParser::PARSE_FAILURE;
		}

		char next_char = (char)next_byte; // Now that we know it's not EOF

		if(found_escape)
		{
			// next character(s) is a special sequence.
			if(found_hex)
			{
				if(found_digit)
				{
					found_digit = false;
					found_hex = false;
					found_escape = false;
					byte = byte << 4;
					byte |= hex_as_nybble(next_char);
					write_buffer << byte;
					byte = 0;
				}
				else
				{
					// next character is the first nybble of
					//
					found_digit = true;
					byte = hex_as_nybble(next_char);
				}
			}
			else if(next_char == 'x')
			{
				found_hex = true;
			}
			else
			{
				switch(next_char)
				{
				case 'a':
					write_buffer << '\a';
					break;
				case 'b':
					write_buffer << '\b';
					break;
				case 'f':
					write_buffer << '\f';
					break;
				case 'n':
					write_buffer << '\n';
					break;
				case 'r':
					write_buffer << '\r';
					break;
				case 't':
					write_buffer << '\t';
					break;
				case 'v':
					write_buffer << '\v';
					break;
				default:
					write_buffer << next_char;
					break;
				}
				found_escape = false;
			}
		}
		else if(next_char == '\\')
		{
			found_escape = true;
		}
		else if(next_char == delim)
		{
			break;
		}
		else
		{
			write_buffer << next_char;
		}
	}

	value = write_buffer.str();
	return count;
}

llssize deserialize_string_raw(
	std::istream& istr,
	std::string& value,
	llssize max_bytes)
{
	llssize count = 0;
	const S32 BUF_LEN = 20;
	char buf[BUF_LEN];		/* Flawfinder: ignore */
	istr.get(buf, BUF_LEN - 1, ')');
	count += istr.gcount();
	int c = istr.get();
	c = istr.get();
	count += 2;
	if(((c == '"') || (c == '\'')) && (buf[0] == '('))
	{
		// We probably have a valid raw string. determine
		// the size, and read it.
		// *FIX: This is memory inefficient.
		auto len = strtol(buf + 1, NULL, 0);
		if((max_bytes>0)&&(len>max_bytes)) return LLSDParser::PARSE_FAILURE;
		std::vector<char> buf;
		if(len)
		{
			buf.resize(len);
			count += fullread(istr, (char *)&buf[0], len);
			value.assign(buf.begin(), buf.end());
		}
		c = istr.get();
		++count;
		if(!((c == '"') || (c == '\'')))
		{
			return LLSDParser::PARSE_FAILURE;
		}
	}
	else
	{
		return LLSDParser::PARSE_FAILURE;
	}
	return count;
}

static const char* NOTATION_STRING_CHARACTERS[256] =
{
	"\\x00",	// 0
	"\\x01",	// 1
	"\\x02",	// 2
	"\\x03",	// 3
	"\\x04",	// 4
	"\\x05",	// 5
	"\\x06",	// 6
	"\\a",		// 7
	"\\b",		// 8
	"\\t",		// 9
	"\\n",		// 10
	"\\v",		// 11
	"\\f",		// 12
	"\\r",		// 13
	"\\x0e",	// 14
	"\\x0f",	// 15
	"\\x10",	// 16
	"\\x11",	// 17
	"\\x12",	// 18
	"\\x13",	// 19
	"\\x14",	// 20
	"\\x15",	// 21
	"\\x16",	// 22
	"\\x17",	// 23
	"\\x18",	// 24
	"\\x19",	// 25
	"\\x1a",	// 26
	"\\x1b",	// 27
	"\\x1c",	// 28
	"\\x1d",	// 29
	"\\x1e",	// 30
	"\\x1f",	// 31
	" ",		// 32
	"!",		// 33
	"\"",		// 34
	"#",		// 35
	"$",		// 36
	"%",		// 37
	"&",		// 38
	"\\'",		// 39
	"(",		// 40
	")",		// 41
	"*",		// 42
	"+",		// 43
	",",		// 44
	"-",		// 45
	".",		// 46
	"/",		// 47
	"0",		// 48
	"1",		// 49
	"2",		// 50
	"3",		// 51
	"4",		// 52
	"5",		// 53
	"6",		// 54
	"7",		// 55
	"8",		// 56
	"9",		// 57
	":",		// 58
	";",		// 59
	"<",		// 60
	"=",		// 61
	">",		// 62
	"?",		// 63
	"@",		// 64
	"A",		// 65
	"B",		// 66
	"C",		// 67
	"D",		// 68
	"E",		// 69
	"F",		// 70
	"G",		// 71
	"H",		// 72
	"I",		// 73
	"J",		// 74
	"K",		// 75
	"L",		// 76
	"M",		// 77
	"N",		// 78
	"O",		// 79
	"P",		// 80
	"Q",		// 81
	"R",		// 82
	"S",		// 83
	"T",		// 84
	"U",		// 85
	"V",		// 86
	"W",		// 87
	"X",		// 88
	"Y",		// 89
	"Z",		// 90
	"[",		// 91
	"\\\\",		// 92
	"]",		// 93
	"^",		// 94
	"_",		// 95
	"`",		// 96
	"a",		// 97
	"b",		// 98
	"c",		// 99
	"d",		// 100
	"e",		// 101
	"f",		// 102
	"g",		// 103
	"h",		// 104
	"i",		// 105
	"j",		// 106
	"k",		// 107
	"l",		// 108
	"m",		// 109
	"n",		// 110
	"o",		// 111
	"p",		// 112
	"q",		// 113
	"r",		// 114
	"s",		// 115
	"t",		// 116
	"u",		// 117
	"v",		// 118
	"w",		// 119
	"x",		// 120
	"y",		// 121
	"z",		// 122
	"{",		// 123
	"|",		// 124
	"}",		// 125
	"~",		// 126
	"\\x7f",	// 127
	"\\x80",	// 128
	"\\x81",	// 129
	"\\x82",	// 130
	"\\x83",	// 131
	"\\x84",	// 132
	"\\x85",	// 133
	"\\x86",	// 134
	"\\x87",	// 135
	"\\x88",	// 136
	"\\x89",	// 137
	"\\x8a",	// 138
	"\\x8b",	// 139
	"\\x8c",	// 140
	"\\x8d",	// 141
	"\\x8e",	// 142
	"\\x8f",	// 143
	"\\x90",	// 144
	"\\x91",	// 145
	"\\x92",	// 146
	"\\x93",	// 147
	"\\x94",	// 148
	"\\x95",	// 149
	"\\x96",	// 150
	"\\x97",	// 151
	"\\x98",	// 152
	"\\x99",	// 153
	"\\x9a",	// 154
	"\\x9b",	// 155
	"\\x9c",	// 156
	"\\x9d",	// 157
	"\\x9e",	// 158
	"\\x9f",	// 159
	"\\xa0",	// 160
	"\\xa1",	// 161
	"\\xa2",	// 162
	"\\xa3",	// 163
	"\\xa4",	// 164
	"\\xa5",	// 165
	"\\xa6",	// 166
	"\\xa7",	// 167
	"\\xa8",	// 168
	"\\xa9",	// 169
	"\\xaa",	// 170
	"\\xab",	// 171
	"\\xac",	// 172
	"\\xad",	// 173
	"\\xae",	// 174
	"\\xaf",	// 175
	"\\xb0",	// 176
	"\\xb1",	// 177
	"\\xb2",	// 178
	"\\xb3",	// 179
	"\\xb4",	// 180
	"\\xb5",	// 181
	"\\xb6",	// 182
	"\\xb7",	// 183
	"\\xb8",	// 184
	"\\xb9",	// 185
	"\\xba",	// 186
	"\\xbb",	// 187
	"\\xbc",	// 188
	"\\xbd",	// 189
	"\\xbe",	// 190
	"\\xbf",	// 191
	"\\xc0",	// 192
	"\\xc1",	// 193
	"\\xc2",	// 194
	"\\xc3",	// 195
	"\\xc4",	// 196
	"\\xc5",	// 197
	"\\xc6",	// 198
	"\\xc7",	// 199
	"\\xc8",	// 200
	"\\xc9",	// 201
	"\\xca",	// 202
	"\\xcb",	// 203
	"\\xcc",	// 204
	"\\xcd",	// 205
	"\\xce",	// 206
	"\\xcf",	// 207
	"\\xd0",	// 208
	"\\xd1",	// 209
	"\\xd2",	// 210
	"\\xd3",	// 211
	"\\xd4",	// 212
	"\\xd5",	// 213
	"\\xd6",	// 214
	"\\xd7",	// 215
	"\\xd8",	// 216
	"\\xd9",	// 217
	"\\xda",	// 218
	"\\xdb",	// 219
	"\\xdc",	// 220
	"\\xdd",	// 221
	"\\xde",	// 222
	"\\xdf",	// 223
	"\\xe0",	// 224
	"\\xe1",	// 225
	"\\xe2",	// 226
	"\\xe3",	// 227
	"\\xe4",	// 228
	"\\xe5",	// 229
	"\\xe6",	// 230
	"\\xe7",	// 231
	"\\xe8",	// 232
	"\\xe9",	// 233
	"\\xea",	// 234
	"\\xeb",	// 235
	"\\xec",	// 236
	"\\xed",	// 237
	"\\xee",	// 238
	"\\xef",	// 239
	"\\xf0",	// 240
	"\\xf1",	// 241
	"\\xf2",	// 242
	"\\xf3",	// 243
	"\\xf4",	// 244
	"\\xf5",	// 245
	"\\xf6",	// 246
	"\\xf7",	// 247
	"\\xf8",	// 248
	"\\xf9",	// 249
	"\\xfa",	// 250
	"\\xfb",	// 251
	"\\xfc",	// 252
	"\\xfd",	// 253
	"\\xfe",	// 254
	"\\xff"		// 255
};

void serialize_string(const std::string& value, std::ostream& str)
{
	std::string::const_iterator it = value.begin();
	std::string::const_iterator end = value.end();
	U8 c;
	for(; it != end; ++it)
	{
		c = (U8)(*it);
		str << NOTATION_STRING_CHARACTERS[c];
	}
}

llssize deserialize_boolean(
	std::istream& istr,
	LLSD& data,
	const std::string& compare,
	bool value)
{
	//
	// this method is a little goofy, because it gets the stream at
	// the point where the t or f has already been
	// consumed. Basically, parse for a patch to the string passed in
	// starting at index 1. If it's a match:
	//  * assign data to value
	//  * return the number of bytes read
	// otherwise:
	//  * set data to LLSD::null
	//  * return LLSDParser::PARSE_FAILURE (-1)
	//
	llssize bytes_read = 0;
	std::string::size_type ii = 0;
	char c = istr.peek();
	while((++ii < compare.size())
		  && (tolower(c) == (int)compare[ii])
		  && istr.good())
	{
		istr.ignore();
		++bytes_read;
		c = istr.peek();
	}
	if(compare.size() != ii)
	{
		data.clear();
		return LLSDParser::PARSE_FAILURE;
	}
	data = value;
	return bytes_read;
}

std::ostream& operator<<(std::ostream& s, const LLSD& llsd)
{
	s << LLSDNotationStreamer(llsd);
	return s;
}


//dirty little zippers -- yell at davep if these are horrid

//return a string containing gzipped bytes of binary serialized LLSD
// VERY inefficient -- creates several copies of LLSD block in memory
std::string zip_llsd(LLSD& data)
{ 
	std::stringstream llsd_strm;

	LLSDSerialize::toBinary(data, llsd_strm);

	const U32 CHUNK = 65536;

	z_stream strm;
	strm.zalloc = Z_NULL;
	strm.zfree = Z_NULL;
	strm.opaque = Z_NULL;

	S32 ret = deflateInit(&strm, Z_BEST_COMPRESSION);
	if (ret != Z_OK)
	{
		LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL;
		return std::string();
	}

	std::string source = llsd_strm.str();

	U8 out[CHUNK];

	strm.avail_in = narrow(source.size());
	strm.next_in = (U8*) source.data();
	U8* output = NULL;

	U32 cur_size = 0;

	U32 have = 0;

	do
	{
		strm.avail_out = CHUNK;
		strm.next_out = out;

		ret = deflate(&strm, Z_FINISH);
		if (ret == Z_OK || ret == Z_STREAM_END)
		{ //copy result into output
			if (strm.avail_out >= CHUNK)
			{
				deflateEnd(&strm);
				if(output)
					free(output);
				LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL;
				return std::string();
			}

			have = CHUNK-strm.avail_out;
			U8* new_output = (U8*) realloc(output, cur_size+have);
			if (new_output == NULL)
			{
				LL_WARNS() << "Failed to compress LLSD block: can't reallocate memory, current size: " << cur_size << " bytes; requested " << cur_size + have << " bytes." << LL_ENDL;
				deflateEnd(&strm);
				if (output)
				{
					free(output);
				}
				return std::string();
			}
			output = new_output;
			memcpy(output+cur_size, out, have);
			cur_size += have;
		}
		else 
		{
			deflateEnd(&strm);
			if(output)
				free(output);
			LL_WARNS() << "Failed to compress LLSD block." << LL_ENDL;
			return std::string();
		}
	}
	while (ret == Z_OK);

	std::string::size_type size = cur_size;

	std::string result((char*) output, size);
	deflateEnd(&strm);
	if(output)
		free(output);

	return result;
}

//decompress a block of LLSD from provided istream
// not very efficient -- creats a copy of decompressed LLSD block in memory
// and deserializes from that copy using LLSDSerialize
LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, S32 size)
{
	std::unique_ptr<U8[]> in = std::unique_ptr<U8[]>(new(std::nothrow) U8[size]);
	if (!in)
	{
		return ZR_MEM_ERROR;
	}
	is.read((char*) in.get(), size); 

	return unzip_llsd(data, in.get(), size);
}

LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, const U8* in, S32 size)
{
	U8* result = NULL;
	U32 cur_size = 0;
	z_stream strm;
		
	constexpr U32 CHUNK = 1024 * 512;

	static thread_local std::unique_ptr<U8[]> out;
	if (!out)
	{
		out = std::unique_ptr<U8[]>(new(std::nothrow) U8[CHUNK]);
	}
		
	strm.zalloc = Z_NULL;
	strm.zfree = Z_NULL;
	strm.opaque = Z_NULL;
	strm.avail_in = size;
	strm.next_in = const_cast<U8*>(in);

	S32 ret = inflateInit(&strm);
	
	do
	{
		strm.avail_out = CHUNK;
		strm.next_out = out.get();
		ret = inflate(&strm, Z_NO_FLUSH);
		switch (ret)
		{
		case Z_NEED_DICT:
		case Z_DATA_ERROR:
		{
			inflateEnd(&strm);
			free(result);
			return ZR_DATA_ERROR;
		}
		case Z_STREAM_ERROR:
		case Z_BUF_ERROR:
		{
			inflateEnd(&strm);
			free(result);
			return ZR_BUFFER_ERROR;
		}

		case Z_MEM_ERROR:
		{
			inflateEnd(&strm);
			free(result);
			return ZR_MEM_ERROR;
		}
		}

		U32 have = CHUNK-strm.avail_out;

		U8* new_result = (U8*)realloc(result, cur_size + have);
		if (new_result == NULL)
		{
			inflateEnd(&strm);
			if (result)
			{
				free(result);
			}
			return ZR_MEM_ERROR;
		}
		result = new_result;
		memcpy(result+cur_size, out.get(), have);
		cur_size += have;

	} while (ret == Z_OK && ret != Z_STREAM_END);

	inflateEnd(&strm);

	if (ret != Z_STREAM_END)
	{
		free(result);
		return ZR_DATA_ERROR;
	}

	//result now points to the decompressed LLSD block
	{
		char* result_ptr = strip_deprecated_header((char*)result, cur_size);

		boost::iostreams::stream<boost::iostreams::array_source> istrm(result_ptr, cur_size);
		
		if (!LLSDSerialize::fromBinary(data, istrm, cur_size, UNZIP_LLSD_MAX_DEPTH))
		{
			free(result);
			return ZR_PARSE_ERROR;
		}
	}

	free(result);
	return ZR_OK;
}
//This unzip function will only work with a gzip header and trailer - while the contents
//of the actual compressed data is the same for either format (gzip vs zlib ), the headers
//and trailers are different for the formats.
U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size )
{
	if (size == 0)
	{
		LL_WARNS() << "No data to unzip." << LL_ENDL;
		return NULL;
	}

	U8* result = NULL;
	U32 cur_size = 0;
	z_stream strm;
		
	const U32 CHUNK = 0x4000;

	U8 *in = new(std::nothrow) U8[size];
	if (in == NULL)
	{
		LL_WARNS() << "Memory allocation failure." << LL_ENDL;
		return NULL;
	}
	is.read((char*) in, size); 

	U8 out[CHUNK];
		
	strm.zalloc = Z_NULL;
	strm.zfree = Z_NULL;
	strm.opaque = Z_NULL;
	strm.avail_in = size;
	strm.next_in = in;

	
	S32 ret = inflateInit2(&strm,  windowBits | ENABLE_ZLIB_GZIP );
	do
	{
		strm.avail_out = CHUNK;
		strm.next_out = out;
		ret = inflate(&strm, Z_NO_FLUSH);
		if (ret == Z_STREAM_ERROR)
		{
			inflateEnd(&strm);
			free(result);
			delete [] in;
			valid = false;
		}
		
		switch (ret)
		{
		case Z_NEED_DICT:
			ret = Z_DATA_ERROR;
		case Z_DATA_ERROR:
		case Z_MEM_ERROR:
			inflateEnd(&strm);
			free(result);
			delete [] in;
			valid = false;
			break;
		}

		U32 have = CHUNK-strm.avail_out;

		U8* new_result = (U8*) realloc(result, cur_size + have);
		if (new_result == NULL)
		{
			LL_WARNS() << "Failed to unzip LLSD NavMesh block: can't reallocate memory, current size: " << cur_size
				<< " bytes; requested " << cur_size + have
				<< " bytes; total syze: ." << size << " bytes."
				<< LL_ENDL;
			inflateEnd(&strm);
			if (result)
			{
				free(result);
			}
			delete[] in;
			valid = false;
			return NULL;
		}
		result = new_result;
		memcpy(result+cur_size, out, have);
		cur_size += have;

	} while (ret == Z_OK);

	inflateEnd(&strm);
	delete [] in;

	if (ret != Z_STREAM_END)
	{
		free(result);
		valid = false;
		return NULL;
	}

	//result now points to the decompressed LLSD block
	{
		outsize= cur_size;
		valid = true;		
	}

	return result;
}

char* strip_deprecated_header(char* in, U32& cur_size, U32* header_size)
{
	const char* deprecated_header = "<? LLSD/Binary ?>";
	constexpr size_t deprecated_header_size = 17;

	if (cur_size > deprecated_header_size
		&& memcmp(in, deprecated_header, deprecated_header_size) == 0)
	{
		in = in + deprecated_header_size;
		cur_size = cur_size - deprecated_header_size;
		if (header_size)
		{
			*header_size = deprecated_header_size + 1;
		}
	}

	return in;
}