summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--indra/llcommon/llsdserialize.cpp87
-rw-r--r--indra/llcommon/llstreamtools.cpp26
-rw-r--r--indra/llcommon/llstreamtools.h27
-rw-r--r--indra/llcommon/tests/llsdserialize_test.cpp192
4 files changed, 233 insertions, 99 deletions
diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp
index 8b4a0ee6d8..9ff4676b95 100644
--- a/indra/llcommon/llsdserialize.cpp
+++ b/indra/llcommon/llsdserialize.cpp
@@ -45,6 +45,7 @@
#endif
#include "lldate.h"
+#include "llmemorystream.h"
#include "llsd.h"
#include "llstring.h"
#include "lluri.h"
@@ -61,6 +62,23 @@ const std::string LLSD_NOTATION_HEADER("llsd/notation");
#define windowBits 15
#define ENABLE_ZLIB_GZIP 32
+// If we published this in llsdserialize.h, we could use it in the
+// implementation of LLSDOStreamer's operator<<().
+template <class Formatter>
+void format_using(const LLSD& data, std::ostream& ostr,
+ LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY)
+{
+ LLPointer<Formatter> f{ new Formatter };
+ f->format(data, ostr, options);
+}
+
+template <class Parser>
+S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1)
+{
+ LLPointer<Parser> p{ new Parser };
+ return p->parse(istr, data, max_bytes, max_depth);
+}
+
/**
* LLSDSerialize
*/
@@ -83,10 +101,10 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize
f = new LLSDXMLFormatter;
break;
- case LLSD_NOTATION:
- str << "<? " << LLSD_NOTATION_HEADER << " ?>\n";
- f = new LLSDNotationFormatter;
- break;
+ case LLSD_NOTATION:
+ str << "<? " << LLSD_NOTATION_HEADER << " ?>\n";
+ f = new LLSDNotationFormatter;
+ break;
default:
LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL;
@@ -101,10 +119,7 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize
// static
bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes)
{
- LLPointer<LLSDParser> p = NULL;
char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */
- int i;
- int inbuf = 0;
bool legacy_no_header = false;
bool fail_if_not_legacy = false;
std::string header;
@@ -112,7 +127,10 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes)
/*
* Get the first line before anything.
*/
- str.get(hdr_buf, MAX_HDR_LEN, '\n');
+ // Remember str's original input position: if there's no header, we'll
+ // want to back up and retry.
+ str.get(hdr_buf, sizeof(hdr_buf), '\n');
+ auto inbuf = str.gcount();
if (str.fail())
{
str.clear();
@@ -122,16 +140,18 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes)
if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */
{
legacy_no_header = true;
- inbuf = (int)str.gcount();
}
else
{
if (fail_if_not_legacy)
- goto fail;
+ {
+ LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL;
+ return false;
+ }
/*
* Remove the newline chars
*/
- for (i = 0; i < MAX_HDR_LEN; i++)
+ for (size_t i = 0; i < sizeof(hdr_buf); i++)
{
if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' ||
hdr_buf[i] == '\n')
@@ -149,50 +169,47 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes)
{
end = header.find_first_of(" ?", start);
}
- if ((start == std::string::npos) || (end == std::string::npos))
- goto fail;
-
- header = header.substr(start, end - start);
- ws(str);
+ if (! (start == std::string::npos) || (end == std::string::npos))
+ {
+ header = header.substr(start, end - start);
+ ws(str);
+ }
}
/*
* Create the parser as appropriate
*/
if (legacy_no_header)
{ // Create a LLSD XML parser, and parse the first chunk read above
- LLSDXMLParser* x = new LLSDXMLParser();
- x->parsePart(hdr_buf, inbuf); // Parse the first part that was already read
- x->parseLines(str, sd); // Parse the rest of it
- delete x;
- return true;
+ LLSDXMLParser x;
+ x.parsePart(hdr_buf, inbuf); // Parse the first part that was already read
+ auto parsed = x.parseLines(str, sd); // Parse the rest of it
+ // Formally we should probably check (parsed != PARSE_FAILURE &&
+ // parsed > 0), but since PARSE_FAILURE is -1, this suffices.
+ return (parsed > 0);
}
if (header == LLSD_BINARY_HEADER)
{
- p = new LLSDBinaryParser;
+ return (parse_using<LLSDBinaryParser>(str, sd, max_bytes) > 0);
}
else if (header == LLSD_XML_HEADER)
{
- p = new LLSDXMLParser;
+ return (parse_using<LLSDXMLParser>(str, sd, max_bytes) > 0);
}
else if (header == LLSD_NOTATION_HEADER)
{
- p = new LLSDNotationParser;
+ return (parse_using<LLSDNotationParser>(str, sd, max_bytes) > 0);
}
else
{
- LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL;
+ LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL;
+ // Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that
+ // data to whatever remains in 'str'.
+ LLMemoryStreamBuf already(reinterpret_cast<const U8*>(hdr_buf), inbuf);
+ cat_streambuf prebuff(&already, str.rdbuf());
+ std::istream prepend(&prebuff);
+ return (parse_using<LLSDNotationParser>(prepend, sd, max_bytes) > 0);
}
-
- if (p.notNull())
- {
- p->parse(str, sd, max_bytes);
- return true;
- }
-
-fail:
- LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL;
- return false;
}
/**
diff --git a/indra/llcommon/llstreamtools.cpp b/indra/llcommon/llstreamtools.cpp
index d7a6f47932..1674f6edc2 100644
--- a/indra/llcommon/llstreamtools.cpp
+++ b/indra/llcommon/llstreamtools.cpp
@@ -523,3 +523,29 @@ std::istream& operator>>(std::istream& str, const char *tocheck)
}
return str;
}
+
+int cat_streambuf::underflow()
+{
+ if (gptr() == egptr())
+ {
+ // here because our buffer is empty
+ std::streamsize size;
+ // Until we've run out of mInputs, try reading the first of them
+ // into mBuffer. If that fetches some characters, break the loop.
+ while (! mInputs.empty()
+ && ! (size = mInputs.front()->sgetn(mBuffer.data(), mBuffer.size())))
+ {
+ // We tried to read mInputs.front() but got zero characters.
+ // Discard the first streambuf and try the next one.
+ mInputs.pop_front();
+ }
+ // Either we ran out of mInputs or we succeeded in reading some
+ // characters, that is, size != 0. Tell base class what we have.
+ setg(mBuffer.data(), mBuffer.data(), mBuffer.data() + size);
+ }
+ // If we fell out of the above loop with mBuffer still empty, return
+ // eof(), otherwise return the next character.
+ return (gptr() == egptr())
+ ? std::char_traits<char>::eof()
+ : std::char_traits<char>::to_int_type(*gptr());
+}
diff --git a/indra/llcommon/llstreamtools.h b/indra/llcommon/llstreamtools.h
index 1b04bf91d7..997f738840 100644
--- a/indra/llcommon/llstreamtools.h
+++ b/indra/llcommon/llstreamtools.h
@@ -27,8 +27,10 @@
#ifndef LL_STREAM_TOOLS_H
#define LL_STREAM_TOOLS_H
+#include <deque>
#include <iostream>
#include <string>
+#include <vector>
// unless specifed otherwise these all return input_stream.good()
@@ -113,6 +115,27 @@ LL_COMMON_API std::streamsize fullread(
LL_COMMON_API std::istream& operator>>(std::istream& str, const char *tocheck);
-#endif
-
+/**
+ * cat_streambuf is a std::streambuf subclass that accepts a variadic number
+ * of std::streambuf* (e.g. some_istream.rdbuf()) and virtually concatenates
+ * their contents.
+ */
+// derived from https://stackoverflow.com/a/49441066/5533635
+class cat_streambuf: public std::streambuf
+{
+private:
+ std::deque<std::streambuf*> mInputs;
+ std::vector<char> mBuffer;
+
+public:
+ // only valid for std::streambuf* arguments
+ template <typename... Inputs>
+ cat_streambuf(Inputs... inputs):
+ mInputs{inputs...},
+ mBuffer(1024)
+ {}
+
+ int underflow();
+};
+#endif
diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp
index c246f5ee56..0334699c7f 100644
--- a/indra/llcommon/tests/llsdserialize_test.cpp
+++ b/indra/llcommon/tests/llsdserialize_test.cpp
@@ -60,6 +60,7 @@ using namespace boost::phoenix;
#include "../test/lltut.h"
#include "../test/namedtempfile.h"
#include "stringize.h"
+#include <functional>
std::vector<U8> string_to_vector(const std::string& str)
{
@@ -112,7 +113,7 @@ namespace tut
mSD = LLUUID::null;
expected = "<llsd><uuid /></llsd>\n";
xml_test("null uuid", expected);
-
+
mSD = LLUUID("c96f9b1e-f589-4100-9774-d98643ce0bed");
expected = "<llsd><uuid>c96f9b1e-f589-4100-9774-d98643ce0bed</uuid></llsd>\n";
xml_test("uuid", expected);
@@ -136,7 +137,7 @@ namespace tut
expected = "<llsd><binary encoding=\"base64\">aGVsbG8=</binary></llsd>\n";
xml_test("binary", expected);
}
-
+
template<> template<>
void sd_xml_object::test<2>()
{
@@ -225,7 +226,7 @@ namespace tut
expected = "<llsd><map><key>baz</key><undef /><key>foo</key><string>bar</string></map></llsd>\n";
xml_test("2 element map", expected);
}
-
+
template<> template<>
void sd_xml_object::test<6>()
{
@@ -241,7 +242,7 @@ namespace tut
expected = "<llsd><binary encoding=\"base64\">Nnw2fGFzZGZoYXBweWJveHw2MGU0NGVjNS0zMDVjLTQzYzItOWExOS1iNGI4OWIxYWUyYTZ8NjBlNDRlYzUtMzA1Yy00M2MyLTlhMTktYjRiODliMWFlMmE2fDYwZTQ0ZWM1LTMwNWMtNDNjMi05YTE5LWI0Yjg5YjFhZTJhNnwwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDB8N2ZmZmZmZmZ8N2ZmZmZmZmZ8MHwwfDgyMDAwfDQ1MGZlMzk0LTI5MDQtYzlhZC0yMTRjLWEwN2ViN2ZlZWMyOXwoTm8gRGVzY3JpcHRpb24pfDB8MTB8MA==</binary></llsd>\n";
xml_test("binary", expected);
}
-
+
class TestLLSDSerializeData
{
public:
@@ -250,9 +251,34 @@ namespace tut
void doRoundTripTests(const std::string&);
void checkRoundTrip(const std::string&, const LLSD& v);
-
- LLPointer<LLSDFormatter> mFormatter;
- LLPointer<LLSDParser> mParser;
+
+ void setFormatterParser(LLPointer<LLSDFormatter> formatter, LLPointer<LLSDParser> parser)
+ {
+ mFormatter = [formatter](const LLSD& data, std::ostream& str)
+ {
+ formatter->format(data, str);
+ };
+ // this lambda must be mutable since otherwise the bound 'parser'
+ // is assumed to point to a const LLSDParser
+ mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes) mutable
+ {
+ // reset() call is needed since test code re-uses parser object
+ parser->reset();
+ return (parser->parse(istr, data, max_bytes) > 0);
+ };
+ }
+
+ void setParser(bool (*parser)(LLSD&, std::istream&, size_t))
+ {
+ // why does LLSDSerialize::deserialize() reverse the parse() params??
+ mParser = [parser](std::istream& istr, LLSD& data, size_t max_bytes)
+ {
+ return (parser(data, istr, max_bytes) > 0);
+ };
+ }
+
+ std::function<void(const LLSD& data, std::ostream& str)> mFormatter;
+ std::function<bool(std::istream& istr, LLSD& data, size_t max_bytes)> mParser;
};
TestLLSDSerializeData::TestLLSDSerializeData()
@@ -265,12 +291,11 @@ namespace tut
void TestLLSDSerializeData::checkRoundTrip(const std::string& msg, const LLSD& v)
{
- std::stringstream stream;
- mFormatter->format(v, stream);
+ std::stringstream stream;
+ mFormatter(v, stream);
//LL_INFOS() << "checkRoundTrip: length " << stream.str().length() << LL_ENDL;
LLSD w;
- mParser->reset(); // reset() call is needed since test code re-uses mParser
- mParser->parse(stream, w, stream.str().size());
+ mParser(stream, w, stream.str().size());
try
{
@@ -299,52 +324,52 @@ namespace tut
fillmap(root[key], width, depth - 1);
}
}
-
+
void TestLLSDSerializeData::doRoundTripTests(const std::string& msg)
{
LLSD v;
checkRoundTrip(msg + " undefined", v);
-
+
v = true;
checkRoundTrip(msg + " true bool", v);
-
+
v = false;
checkRoundTrip(msg + " false bool", v);
-
+
v = 1;
checkRoundTrip(msg + " positive int", v);
-
+
v = 0;
checkRoundTrip(msg + " zero int", v);
-
+
v = -1;
checkRoundTrip(msg + " negative int", v);
-
+
v = 1234.5f;
checkRoundTrip(msg + " positive float", v);
-
+
v = 0.0f;
checkRoundTrip(msg + " zero float", v);
-
+
v = -1234.5f;
checkRoundTrip(msg + " negative float", v);
-
+
// FIXME: need a NaN test
-
+
v = LLUUID::null;
checkRoundTrip(msg + " null uuid", v);
-
+
LLUUID newUUID;
newUUID.generate();
v = newUUID;
checkRoundTrip(msg + " new uuid", v);
-
+
v = "";
checkRoundTrip(msg + " empty string", v);
-
+
v = "some string";
checkRoundTrip(msg + " non-empty string", v);
-
+
v =
"Second Life is a 3-D virtual world entirely built and owned by its residents. "
"Since opening to the public in 2003, it has grown explosively and today is "
@@ -372,7 +397,7 @@ namespace tut
for (U32 block = 0x000000; block <= 0x10ffff; block += block_size)
{
std::ostringstream out;
-
+
for (U32 c = block; c < block + block_size; ++c)
{
if (c <= 0x000001f
@@ -386,7 +411,7 @@ namespace tut
if (0x00fdd0 <= c && c <= 0x00fdef) { continue; }
if ((c & 0x00fffe) == 0x00fffe) { continue; }
// see Unicode standard, section 15.8
-
+
if (c <= 0x00007f)
{
out << (char)(c & 0x7f);
@@ -410,55 +435,55 @@ namespace tut
out << (char)(0x80 | ((c >> 0) & 0x3f));
}
}
-
+
v = out.str();
std::ostringstream blockmsg;
blockmsg << msg << " unicode string block 0x" << std::hex << block;
checkRoundTrip(blockmsg.str(), v);
}
-
+
LLDate epoch;
v = epoch;
checkRoundTrip(msg + " epoch date", v);
-
+
LLDate aDay("2002-12-07T05:07:15.00Z");
v = aDay;
checkRoundTrip(msg + " date", v);
-
+
LLURI path("http://slurl.com/secondlife/Ambleside/57/104/26/");
v = path;
checkRoundTrip(msg + " url", v);
-
+
const char source[] = "it must be a blue moon again";
std::vector<U8> data;
// note, includes terminating '\0'
copy(&source[0], &source[sizeof(source)], back_inserter(data));
-
+
v = data;
checkRoundTrip(msg + " binary", v);
-
+
v = LLSD::emptyMap();
checkRoundTrip(msg + " empty map", v);
-
+
v = LLSD::emptyMap();
v["name"] = "luke"; //v.insert("name", "luke");
v["age"] = 3; //v.insert("age", 3);
checkRoundTrip(msg + " map", v);
-
+
v.clear();
v["a"]["1"] = true;
v["b"]["0"] = false;
checkRoundTrip(msg + " nested maps", v);
-
+
v = LLSD::emptyArray();
checkRoundTrip(msg + " empty array", v);
-
+
v = LLSD::emptyArray();
v.append("ali");
v.append(28);
checkRoundTrip(msg + " array", v);
-
+
v.clear();
v[0][0] = true;
v[1][0] = false;
@@ -468,7 +493,7 @@ namespace tut
fillmap(v, 10, 3); // 10^6 maps
checkRoundTrip(msg + " many nested maps", v);
}
-
+
typedef tut::test_group<TestLLSDSerializeData> TestLLSDSerializeGroup;
typedef TestLLSDSerializeGroup::object TestLLSDSerializeObject;
TestLLSDSerializeGroup gTestLLSDSerializeGroup("llsd serialization");
@@ -476,35 +501,78 @@ namespace tut
template<> template<>
void TestLLSDSerializeObject::test<1>()
{
- mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY);
- mParser = new LLSDNotationParser();
+ setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY),
+ new LLSDNotationParser());
doRoundTripTests("pretty binary notation serialization");
}
template<> template<>
void TestLLSDSerializeObject::test<2>()
{
- mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE);
- mParser = new LLSDNotationParser();
+ setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE),
+ new LLSDNotationParser());
doRoundTripTests("raw binary notation serialization");
}
template<> template<>
void TestLLSDSerializeObject::test<3>()
{
- mFormatter = new LLSDXMLFormatter();
- mParser = new LLSDXMLParser();
+ setFormatterParser(new LLSDXMLFormatter(), new LLSDXMLParser());
doRoundTripTests("xml serialization");
}
template<> template<>
void TestLLSDSerializeObject::test<4>()
{
- mFormatter = new LLSDBinaryFormatter();
- mParser = new LLSDBinaryParser();
+ setFormatterParser(new LLSDBinaryFormatter(), new LLSDBinaryParser());
doRoundTripTests("binary serialization");
}
+ template<> template<>
+ void TestLLSDSerializeObject::test<5>()
+ {
+ mFormatter = [](const LLSD& sd, std::ostream& str)
+ {
+ LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_BINARY);
+ };
+ setParser(LLSDSerialize::deserialize);
+ doRoundTripTests("serialize(LLSD_BINARY)");
+ };
+
+ template<> template<>
+ void TestLLSDSerializeObject::test<6>()
+ {
+ mFormatter = [](const LLSD& sd, std::ostream& str)
+ {
+ LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_XML);
+ };
+ setParser(LLSDSerialize::deserialize);
+ doRoundTripTests("serialize(LLSD_XML)");
+ };
+
+ template<> template<>
+ void TestLLSDSerializeObject::test<7>()
+ {
+ mFormatter = [](const LLSD& sd, std::ostream& str)
+ {
+ LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_NOTATION);
+ };
+ setParser(LLSDSerialize::deserialize);
+ // In this test, serialize(LLSD_NOTATION) emits a header recognized by
+ // deserialize().
+ doRoundTripTests("serialize(LLSD_NOTATION)");
+ };
+
+ template<> template<>
+ void TestLLSDSerializeObject::test<8>()
+ {
+ setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE),
+ new LLSDNotationParser());
+ setParser(LLSDSerialize::deserialize);
+ // This is an interesting test because LLSDNotationFormatter does not
+ // emit an llsd/notation header.
+ doRoundTripTests("LLSDNotationFormatter -> deserialize");
+ };
/**
* @class TestLLSDParsing
@@ -555,7 +623,7 @@ namespace tut
public:
TestLLSDXMLParsing() {}
};
-
+
typedef tut::test_group<TestLLSDXMLParsing> TestLLSDXMLParsingGroup;
typedef TestLLSDXMLParsingGroup::object TestLLSDXMLParsingObject;
TestLLSDXMLParsingGroup gTestLLSDXMLParsingGroup("llsd XML parsing");
@@ -586,8 +654,8 @@ namespace tut
LLSD(),
LLSDParser::PARSE_FAILURE);
}
-
-
+
+
template<> template<>
void TestLLSDXMLParsingObject::test<2>()
{
@@ -596,7 +664,7 @@ namespace tut
v["amy"] = 23;
v["bob"] = LLSD();
v["cam"] = 1.23;
-
+
ensureParse(
"unknown data type",
"<llsd><map>"
@@ -607,16 +675,16 @@ namespace tut
v,
v.size() + 1);
}
-
+
template<> template<>
void TestLLSDXMLParsingObject::test<3>()
{
// test handling of nested bad data
-
+
LLSD v;
v["amy"] = 23;
v["cam"] = 1.23;
-
+
ensureParse(
"map with html",
"<llsd><map>"
@@ -626,7 +694,7 @@ namespace tut
"</map></llsd>",
v,
v.size() + 1);
-
+
v.clear();
v["amy"] = 23;
v["cam"] = 1.23;
@@ -639,7 +707,7 @@ namespace tut
"</map></llsd>",
v,
v.size() + 1);
-
+
v.clear();
v["amy"] = 23;
v["bob"] = LLSD::emptyMap();
@@ -661,7 +729,7 @@ namespace tut
v[0] = 23;
v[1] = LLSD();
v[2] = 1.23;
-
+
ensureParse(
"array value of html",
"<llsd><array>"
@@ -671,7 +739,7 @@ namespace tut
"</array></llsd>",
v,
v.size() + 1);
-
+
v.clear();
v[0] = 23;
v[1] = LLSD::emptyMap();
@@ -1225,7 +1293,7 @@ namespace tut
vec[0] = 'a'; vec[1] = 'b'; vec[2] = 'c';
vec[3] = '3'; vec[4] = '2'; vec[5] = '1';
LLSD value = vec;
-
+
vec.resize(11);
vec[0] = 'b'; // for binary
vec[5] = 'a'; vec[6] = 'b'; vec[7] = 'c';
@@ -1909,6 +1977,6 @@ namespace tut
"This string\n"
"has several\n"
"lines.");
-
+
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-28 22:12:45 +0000