From 580f9088b4644c1d9e41a25deac42dc487e9d5c1 Mon Sep 17 00:00:00 2001 From: Dave Simmons Date: Wed, 25 Jun 2008 16:22:00 +0000 Subject: svn merge -r90394:90492 svn/branches/havok4/qar-689 --> release QAR-689 - branch/havok4/havok4-7 r89805 is ready for merge back into release --- indra/llcommon/indra_constants.h | 2 +- indra/llcommon/llsdserialize.cpp | 25 +++++-- indra/llcommon/llsdserialize.h | 41 ++++++++++- indra/llcommon/llsdserialize_xml.cpp | 128 ++++++++++++++++++++++++++++------- indra/llcommon/llstat.cpp | 5 +- indra/llcommon/llstat.h | 8 ++- indra/llmath/v3math.cpp | 66 ++++++++++++++++++ indra/llmath/v3math.h | 1 + indra/llmessage/llassetstorage.cpp | 66 +++++++++++++++++- indra/llmessage/llassetstorage.h | 15 +++- indra/newview/llviewerregion.cpp | 2 +- 11 files changed, 321 insertions(+), 38 deletions(-) diff --git a/indra/llcommon/indra_constants.h b/indra/llcommon/indra_constants.h index 5697fb9f41..e83da12beb 100644 --- a/indra/llcommon/indra_constants.h +++ b/indra/llcommon/indra_constants.h @@ -78,7 +78,7 @@ enum LAND_STAT_REPORT_TYPE const U32 STAT_FILTER_MASK = 0x1FFFFFFF; // Default maximum number of tasks/prims per region. -const U32 MAX_TASKS_PER_REGION = 15000; +const U32 DEFAULT_MAX_REGION_WIDE_PRIM_COUNT = 15000; const F32 MIN_AGENT_DEPTH = 0.30f; const F32 DEFAULT_AGENT_DEPTH = 0.45f; diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 2183792bb1..6bb75439a2 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -146,12 +146,15 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) * Create the parser as appropriate */ if (legacy_no_header) - { - LLSDXMLParser* x = new LLSDXMLParser; - x->parsePart(hdr_buf, inbuf); - p = x; + { // Create a LLSD XML parser, and parse the first chunk read above + LLSDXMLParser* x = new LLSDXMLParser(); + x->parsePart(hdr_buf, inbuf); // Parse the first part that was already read + x->parseLines(str, sd); // Parse the rest of it + delete x; + return true; } - else if (header == LLSD_BINARY_HEADER) + + if (header == LLSD_BINARY_HEADER) { p = new LLSDBinaryParser; } @@ -300,7 +303,8 @@ static const char BINARY_FALSE_SERIAL = '0'; /** * LLSDParser */ -LLSDParser::LLSDParser() : mCheckLimits(true), mMaxBytesLeft(0) +LLSDParser::LLSDParser() + : mCheckLimits(true), mMaxBytesLeft(0), mParseLines(false) { } @@ -316,6 +320,15 @@ S32 LLSDParser::parse(std::istream& istr, LLSD& data, S32 max_bytes) } +// Parse using routine to get() lines, faster than parse() +S32 LLSDParser::parseLines(std::istream& istr, LLSD& data) +{ + mCheckLimits = false; + mParseLines = true; + return doParse(istr, data); +} + + int LLSDParser::get(std::istream& istr) const { if(mCheckLimits) --mMaxBytesLeft; diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 5e88070130..df78bb44f4 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -83,6 +83,18 @@ public: */ S32 parse(std::istream& istr, LLSD& data, S32 max_bytes); + /** Like parse(), but uses a different call (istream.getline()) to read by lines + * This API is better suited for XML, where the parse cannot tell + * where the document actually ends. + */ + S32 parseLines(std::istream& istr, LLSD& data); + + /** + * @brief Resets the parser so parse() or parseLines() can be called again for another chunk. + */ + void reset() { doReset(); }; + + protected: /** * @brief Pure virtual base for doing the parse. @@ -100,6 +112,11 @@ protected: */ virtual S32 doParse(std::istream& istr, LLSD& data) const = 0; + /** + * @brief Virtual default function for resetting the parser + */ + virtual void doReset() {}; + /* @name Simple istream helper methods * * These helper methods exist to help correctly use the @@ -191,6 +208,11 @@ protected: * @brief The maximum number of bytes left to be parsed. */ mutable S32 mMaxBytesLeft; + + /** + * @brief Use line-based reading to get text + */ + bool mParseLines; }; /** @@ -301,6 +323,11 @@ protected: */ virtual S32 doParse(std::istream& istr, LLSD& data) const; + /** + * @brief Virtual default function for resetting the parser + */ + virtual void doReset(); + private: class Impl; Impl& impl; @@ -674,7 +701,7 @@ public: U32 options = LLSDFormatter::OPTIONS_NONE); /** - * @breif Examine a stream, and parse 1 sd object out based on contents. + * @brief Examine a stream, and parse 1 sd object out based on contents. * * @param sd [out] The data found on the stream * @param str The incoming stream @@ -718,13 +745,23 @@ public: return f->format(sd, str, LLSDFormatter::OPTIONS_PRETTY); } - static S32 fromXML(LLSD& sd, std::istream& str) + static S32 fromXMLEmbedded(LLSD& sd, std::istream& str) { // no need for max_bytes since xml formatting is not // subvertable by bad sizes. LLPointer p = new LLSDXMLParser; return p->parse(str, sd, LLSDSerialize::SIZE_UNLIMITED); } + static S32 fromXMLDocument(LLSD& sd, std::istream& str) + { + LLPointer p = new LLSDXMLParser(); + return p->parseLines(str, sd); + } + static S32 fromXML(LLSD& sd, std::istream& str) + { + return fromXMLEmbedded(sd, str); +// return fromXMLDocument(sd, str); + } /* * Binary Methods diff --git a/indra/llcommon/llsdserialize_xml.cpp b/indra/llcommon/llsdserialize_xml.cpp index cddb243faf..592dfc9bc0 100644 --- a/indra/llcommon/llsdserialize_xml.cpp +++ b/indra/llcommon/llsdserialize_xml.cpp @@ -262,12 +262,13 @@ public: ~Impl(); S32 parse(std::istream& input, LLSD& data); + S32 parseLines(std::istream& input, LLSD& data); void parsePart(const char *buf, int len); -private: void reset(); - + +private: void startElementHandler(const XML_Char* name, const XML_Char** attributes); void endElementHandler(const XML_Char* name); void characterDataHandler(const XML_Char* data, int length); @@ -307,8 +308,8 @@ private: LLSD mResult; S32 mParseCount; - bool mInLLSDElement; - bool mGracefullStop; + bool mInLLSDElement; // true if we're on LLSD + bool mGracefullStop; // true if we found the LLSDRefStack; LLSDRefStack mStack; @@ -319,15 +320,12 @@ private: std::string mCurrentKey; std::ostringstream mCurrentContent; - - bool mPreStaged; }; LLSDXMLParser::Impl::Impl() { mParser = XML_ParserCreate(NULL); - mPreStaged = false; reset(); } @@ -336,7 +334,7 @@ LLSDXMLParser::Impl::~Impl() XML_ParserFree(mParser); } -bool is_eol(char c) +inline bool is_eol(char c) { return (c == '\n' || c == '\r'); } @@ -356,9 +354,9 @@ static unsigned get_till_eol(std::istream& input, char *buf, unsigned bufsize) unsigned count = 0; while (count < bufsize && input.good()) { - input.get(buf[count]); - count++; - if (is_eol(buf[count - 1])) + char c = input.get(); + buf[count++] = c; + if (is_eol(c)) break; } return count; @@ -366,7 +364,6 @@ static unsigned get_till_eol(std::istream& input, char *buf, unsigned bufsize) S32 LLSDXMLParser::Impl::parse(std::istream& input, LLSD& data) { - reset(); XML_Status status; static const int BUFFER_SIZE = 1024; @@ -420,14 +417,86 @@ S32 LLSDXMLParser::Impl::parse(std::istream& input, LLSD& data) return mParseCount; } -void LLSDXMLParser::Impl::reset() + +S32 LLSDXMLParser::Impl::parseLines(std::istream& input, LLSD& data) { - if (mPreStaged) + XML_Status status = XML_STATUS_OK; + + data = LLSD(); + + static const int BUFFER_SIZE = 1024; + + //static char last_buffer[ BUFFER_SIZE ]; + //std::streamsize last_num_read; + + // Must get rid of any leading \n, otherwise the stream gets into an error/eof state + clear_eol(input); + + while( !mGracefullStop + && input.good() + && !input.eof()) { - mPreStaged = false; - return; + void* buffer = XML_GetBuffer(mParser, BUFFER_SIZE); + /* + * If we happened to end our last buffer right at the end of the llsd, but the + * stream is still going we will get a null buffer here. Check for mGracefullStop. + * -- I don't think this is actually true - zero 2008-05-09 + */ + if (!buffer) + { + break; + } + + // Get one line + input.getline((char*)buffer, BUFFER_SIZE); + std::streamsize num_read = input.gcount(); + + //memcpy( last_buffer, buffer, num_read ); + //last_num_read = num_read; + + if ( num_read > 0 ) + { + if (!input.good() ) + { // Clear state that's set when we run out of buffer + input.clear(); + } + + // Don't parse the NULL at the end which might be added if \n was absorbed by getline() + char * text = (char *) buffer; + if ( text[num_read - 1] == 0) + { + num_read--; + } + } + + status = XML_ParseBuffer(mParser, num_read, false); + if (status == XML_STATUS_ERROR) + { + break; + } + } + + if (status != XML_STATUS_ERROR + && !mGracefullStop) + { // Parse last bit + status = XML_ParseBuffer(mParser, 0, true); + } + + if (status == XML_STATUS_ERROR + && !mGracefullStop) + { + llinfos << "LLSDXMLParser::Impl::parseLines: XML_STATUS_ERROR" << llendl; + return LLSDParser::PARSE_FAILURE; } + clear_eol(input); + data = mResult; + return mParseCount; +} + + +void LLSDXMLParser::Impl::reset() +{ mResult.clear(); mParseCount = 0; @@ -476,14 +545,15 @@ LLSDXMLParser::Impl::findAttribute(const XML_Char* name, const XML_Char** pairs) void LLSDXMLParser::Impl::parsePart(const char* buf, int len) { - void * buffer = XML_GetBuffer(mParser, len); - if (buffer != NULL && buf != NULL) + if ( buf != NULL + && len > 0 ) { - memcpy(buffer, buf, len); + XML_Status status = XML_Parse(mParser, buf, len, false); + if (status == XML_STATUS_ERROR) + { + llinfos << "Unexpected XML parsing error at start" << llendl; + } } - XML_ParseBuffer(mParser, len, false); - - mPreStaged = true; } void LLSDXMLParser::Impl::startElementHandler(const XML_Char* name, const XML_Char** attributes) @@ -738,5 +808,17 @@ void LLSDXMLParser::parsePart(const char *buf, int len) // virtual S32 LLSDXMLParser::doParse(std::istream& input, LLSD& data) const { - return impl.parse(input, data); + if (mParseLines) + { + // Use line-based reading (faster code) + return impl.parseLines(input, data); + } + + return impl.parse(input, data); +} + +// virtual +void LLSDXMLParser::doReset() +{ + impl.reset(); } diff --git a/indra/llcommon/llstat.cpp b/indra/llcommon/llstat.cpp index e999934b62..21b723de71 100644 --- a/indra/llcommon/llstat.cpp +++ b/indra/llcommon/llstat.cpp @@ -76,10 +76,11 @@ public: U64 LLStatAccum::impl::sScaleTimes[IMPL_NUM_SCALES] = { + USEC_PER_SEC / 10, // 100 millisec USEC_PER_SEC * 1, // seconds USEC_PER_SEC * 60, // minutes - USEC_PER_SEC * 60 * 2 // minutes -#if 0 + USEC_PER_SEC * 60 * 2 // two minutes +#if ENABLE_LONG_TIME_STATS // enable these when more time scales are desired USEC_PER_SEC * 60*60, // hours USEC_PER_SEC * 24*60*60, // days diff --git a/indra/llcommon/llstat.h b/indra/llcommon/llstat.h index e20c01f67f..5fa46fca75 100644 --- a/indra/llcommon/llstat.h +++ b/indra/llcommon/llstat.h @@ -37,6 +37,9 @@ #include "lltimer.h" #include "llframetimer.h" +// Set this if longer stats are needed +#define ENABLE_LONG_TIME_STATS 0 + // // Accumulates statistics for an arbitrary length of time. // Does this by maintaining a chain of accumulators, each one @@ -52,19 +55,22 @@ protected: public: enum TimeScale { + SCALE_100MS, SCALE_SECOND, SCALE_MINUTE, SCALE_TWO_MINUTE, +#if ENABLE_LONG_TIME_STATS SCALE_HOUR, SCALE_DAY, SCALE_WEEK, - +#endif NUM_SCALES }; F32 meanValue(TimeScale scale) const; // see the subclasses for the specific meaning of value + F32 meanValueOverLast100ms() const { return meanValue(SCALE_100MS); } F32 meanValueOverLastSecond() const { return meanValue(SCALE_SECOND); } F32 meanValueOverLastMinute() const { return meanValue(SCALE_MINUTE); } diff --git a/indra/llmath/v3math.cpp b/indra/llmath/v3math.cpp index f1fe1a780e..6299bbd1b7 100644 --- a/indra/llmath/v3math.cpp +++ b/indra/llmath/v3math.cpp @@ -73,6 +73,72 @@ BOOL LLVector3::clamp(F32 min, F32 max) return ret; } +// Clamps length to an upper limit. +// Returns TRUE if the data changed +BOOL LLVector3::clampLength( F32 length_limit ) +{ + BOOL changed = FALSE; + + F32 len = length(); + if (llfinite(len)) + { + if ( len > length_limit) + { + normalize(); + if (length_limit < 0.f) + { + length_limit = 0.f; + } + mV[0] *= length_limit; + mV[1] *= length_limit; + mV[2] *= length_limit; + changed = TRUE; + } + } + else + { // this vector may still be salvagable + F32 max_abs_component = 0.f; + for (S32 i = 0; i < 3; ++i) + { + F32 abs_component = fabs(mV[i]); + if (llfinite(abs_component)) + { + if (abs_component > max_abs_component) + { + max_abs_component = abs_component; + } + } + else + { + // no it can't be salvaged --> clear it + clear(); + changed = TRUE; + break; + } + } + if (!changed) + { + // yes it can be salvaged --> + // bring the components down before we normalize + mV[0] /= max_abs_component; + mV[1] /= max_abs_component; + mV[2] /= max_abs_component; + normalize(); + + if (length_limit < 0.f) + { + length_limit = 0.f; + } + mV[0] *= length_limit; + mV[1] *= length_limit; + mV[2] *= length_limit; + } + } + + return changed; +} + + // Sets all values to absolute value of their original values // Returns TRUE if data changed BOOL LLVector3::abs() diff --git a/indra/llmath/v3math.h b/indra/llmath/v3math.h index 03c780a1f4..d8f3bec193 100644 --- a/indra/llmath/v3math.h +++ b/indra/llmath/v3math.h @@ -74,6 +74,7 @@ class LLVector3 inline BOOL isFinite() const; // checks to see if all values of LLVector3 are finite BOOL clamp(F32 min, F32 max); // Clamps all values to (min,max), returns TRUE if data changed + BOOL clampLength( F32 length_limit ); // Scales vector to limit length to a value void quantize16(F32 lowerxy, F32 upperxy, F32 lowerz, F32 upperz); // changes the vector to reflect quatization void quantize8(F32 lowerxy, F32 upperxy, F32 lowerz, F32 upperz); // changes the vector to reflect quatization diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 2c8e7ce8a6..9d32afc2f0 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -42,6 +42,7 @@ #include "llstring.h" #include "lldir.h" #include "llsd.h" +#include "llframetimer.h" // this library includes #include "message.h" @@ -60,6 +61,9 @@ LLMetrics *LLAssetStorage::metric_recipient = NULL; const LLUUID CATEGORIZE_LOST_AND_FOUND_ID("00000000-0000-0000-0000-000000000010"); +const U64 TOXIC_ASSET_LIFETIME = (120 * 1000000); // microseconds + + ///---------------------------------------------------------------------------- /// LLAssetInfo ///---------------------------------------------------------------------------- @@ -314,6 +318,9 @@ LLAssetStorage::~LLAssetStorage() // unregister our callbacks with the message system gMessageSystem->setHandlerFuncFast(_PREHASH_AssetUploadComplete, NULL, NULL); } + + // Clear the toxic asset map + mToxicAssetMap.clear(); } void LLAssetStorage::setUpstream(const LLHost &upstream_host) @@ -1233,7 +1240,11 @@ void LLAssetStorage::legacyGetDataCallback(LLVFS *vfs, const LLUUID &uuid, LLAss LLLegacyAssetRequest *legacy = (LLLegacyAssetRequest *)user_data; char filename[LL_MAX_PATH] = ""; /* Flawfinder: ignore */ - if (! status) + // Check if the asset is marked toxic, and don't load bad stuff + BOOL toxic = gAssetStorage->isAssetToxic( uuid ); + + if ( !status + && !toxic ) { LLVFile file(vfs, uuid, type); @@ -1431,3 +1442,56 @@ void LLAssetStorage::reportMetric( const LLUUID& asset_id, const LLAssetType::ET metric_recipient->recordEvent(metric_name, message, success); } } + + +// Check if an asset is in the toxic map. If it is, the entry is updated +BOOL LLAssetStorage::isAssetToxic( const LLUUID& uuid ) +{ + BOOL is_toxic = FALSE; + + if ( !uuid.isNull() ) + { + toxic_asset_map_t::iterator iter = mToxicAssetMap.find( uuid ); + if ( iter != mToxicAssetMap.end() ) + { // Found toxic asset + (*iter).second = LLFrameTimer::getTotalTime() + TOXIC_ASSET_LIFETIME; + is_toxic = TRUE; + } + } + return is_toxic; +} + + + + +// Clean the toxic asset list, remove old entries +void LLAssetStorage::flushOldToxicAssets( BOOL force_it ) +{ + // Scan and look for old entries + U64 now = LLFrameTimer::getTotalTime(); + toxic_asset_map_t::iterator iter = mToxicAssetMap.begin(); + while ( iter != mToxicAssetMap.end() ) + { + if ( force_it + || (*iter).second < now ) + { // Too old - remove it + mToxicAssetMap.erase( iter++ ); + } + else + { + iter++; + } + } +} + + +// Add an item to the toxic asset map +void LLAssetStorage::markAssetToxic( const LLUUID& uuid ) +{ + if ( !uuid.isNull() ) + { + // Set the value to the current time. Creates a new entry if needed + mToxicAssetMap[ uuid ] = LLFrameTimer::getTotalTime() + TOXIC_ASSET_LIFETIME; + } +} + diff --git a/indra/llmessage/llassetstorage.h b/indra/llmessage/llassetstorage.h index 8da3926c63..b1007e83c6 100644 --- a/indra/llmessage/llassetstorage.h +++ b/indra/llmessage/llassetstorage.h @@ -197,7 +197,8 @@ public: }; - +// Map of known bad assets +typedef std::map toxic_asset_map_t; typedef void (*LLGetAssetCallback)(LLVFS *vfs, const LLUUID &asset_id, LLAssetType::EType asset_type, void *user_data, S32 status, LLExtStat ext_status); @@ -231,6 +232,9 @@ protected: request_list_t mPendingUploads; request_list_t mPendingLocalUploads; + // Map of toxic assets - these caused problems when recently rezzed, so avoid them + toxic_asset_map_t mToxicAssetMap; // Objects in this list are known to cause problems and are not loaded + public: LLAssetStorage(LLMessageSystem *msg, LLXferManager *xfer, LLVFS *vfs, const LLHost &upstream_host); @@ -291,6 +295,15 @@ public: const LLUUID &asset_id, LLAssetType::EType atype, LLGetAssetCallback cb, void *user_data, BOOL is_priority = FALSE); // Get a particular inventory item. + // Check if an asset is in the toxic map. If it is, the entry is updated + BOOL isAssetToxic( const LLUUID& uuid ); + + // Clean the toxic asset list, remove old entries + void flushOldToxicAssets( BOOL force_it ); + + // Add an item to the toxic asset map + void markAssetToxic( const LLUUID& uuid ); + protected: virtual LLSD getPendingDetails(const request_list_t* requests, LLAssetType::EType asset_type, diff --git a/indra/newview/llviewerregion.cpp b/indra/newview/llviewerregion.cpp index b847f4d04c..41c97818dc 100644 --- a/indra/newview/llviewerregion.cpp +++ b/indra/newview/llviewerregion.cpp @@ -163,7 +163,7 @@ LLViewerRegion::LLViewerRegion(const U64 &handle, mRegionFlags( REGION_FLAGS_DEFAULT ), mSimAccess( SIM_ACCESS_MIN ), mBillableFactor(1.0), - mMaxTasks(MAX_TASKS_PER_REGION), + mMaxTasks(DEFAULT_MAX_REGION_WIDE_PRIM_COUNT), mCacheLoaded(FALSE), mCacheEntriesCount(0), mCacheID(), -- cgit v1.2.3