From 25dc8660efedecfd9f021142ac1e4ff3639e80d4 Mon Sep 17 00:00:00 2001 From: Rye Date: Tue, 30 Dec 2025 11:39:16 -0500 Subject: Clean up LLFile and LLFilesystem operations to reduce temporaries allocations and utf8->utf16 conversions Fix LLFile::tmpdir failing when temp directory is located in a unicode path on windows Add Tracy profiler tagging to LLFile that's disabled by default Replace LLFile::utf8StringToWstring with our fsyspath std::filesystem::path adapter Introduce std::filesystem::path to llifstream/llofstream wrappers Add move operators to fsyspath to match copy operator Restore LLUniqueFile RAII class for direct c file io wrapping and reducing merge conflicts with downstream code that may depend upon it Signed-off-by: Rye --- indra/llmessage/message.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llmessage/message.cpp') diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index e2937490ba..5b3b57f11e 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -2376,7 +2376,7 @@ void dump_prehash_files() { U32 i; std::string filename("../../indra/llmessage/message_prehash.h"); - LLFILE* fp = LLFile::fopen(filename, "w"); /* Flawfinder: ignore */ + LLFILE* fp = LLFile::fopen(filename, TEXT("w")); /* Flawfinder: ignore */ if (fp) { fprintf( @@ -2407,7 +2407,7 @@ void dump_prehash_files() fclose(fp); } filename = std::string("../../indra/llmessage/message_prehash.cpp"); - fp = LLFile::fopen(filename, "w"); /* Flawfinder: ignore */ + fp = LLFile::fopen(filename, TEXT("w")); /* Flawfinder: ignore */ if (fp) { fprintf( -- cgit v1.3 From 9873abfb94f0d9cbc6df60fef9df622ef51355ec Mon Sep 17 00:00:00 2001 From: Rye Date: Mon, 5 Jan 2026 15:01:23 -0500 Subject: Address feedback from copilot Signed-off-by: Rye --- indra/llappearance/llpolymesh.cpp | 2 +- indra/llcommon/llcrc.cpp | 2 +- indra/llcommon/llcrc.h | 2 +- indra/llcommon/llfile.h | 10 ++++------ indra/llcommon/llprocessor.cpp | 4 ++-- indra/llcommon/llsys.cpp | 6 +++--- indra/llfilesystem/lldir.cpp | 2 +- indra/llfilesystem/tests/lldir_test.cpp | 2 +- indra/llimage/llimagedimensionsinfo.cpp | 2 +- indra/llimage/llimagetga.cpp | 2 +- indra/llmessage/llassetstorage.cpp | 2 +- indra/llmessage/lltransfersourcefile.cpp | 2 +- indra/llmessage/lltransfertargetfile.cpp | 2 +- indra/llmessage/llxfer_file.cpp | 16 ++++++++-------- indra/llmessage/message.cpp | 4 ++-- indra/llmessage/tests/lldatapacker_test.cpp | 4 ++-- indra/llrender/llshadermgr.cpp | 4 ++-- indra/llui/lluicolortable.cpp | 2 +- indra/llui/llxuiparser.cpp | 4 ++-- indra/llxml/llcontrol.cpp | 2 +- indra/llxml/llxmlparser.cpp | 4 ++-- indra/newview/llconversationlog.cpp | 4 ++-- indra/newview/lldrawpoolbump.cpp | 2 +- indra/newview/llfloateruipreview.cpp | 2 +- indra/newview/llinventorymodel.cpp | 4 ++-- indra/newview/llkeyconflict.cpp | 2 +- indra/newview/lllogchat.cpp | 12 ++++++------ indra/newview/llmutelist.cpp | 8 ++++---- indra/newview/llpreviewnotecard.cpp | 4 ++-- indra/newview/llpreviewscript.cpp | 4 ++-- indra/newview/llstartup.cpp | 6 +++--- indra/newview/lltoolbarview.cpp | 2 +- indra/newview/llviewerassetstorage.cpp | 2 +- indra/newview/llviewerinput.cpp | 2 +- indra/newview/llviewermenufile.cpp | 2 +- indra/newview/llviewerstatsrecorder.cpp | 2 +- 36 files changed, 68 insertions(+), 70 deletions(-) (limited to 'indra/llmessage/message.cpp') diff --git a/indra/llappearance/llpolymesh.cpp b/indra/llappearance/llpolymesh.cpp index b579dc007b..45f51950ec 100644 --- a/indra/llappearance/llpolymesh.cpp +++ b/indra/llappearance/llpolymesh.cpp @@ -280,7 +280,7 @@ bool LLPolyMeshSharedData::loadMesh( const std::string& fileName ) LL_ERRS() << "Filename is Empty!" << LL_ENDL; return false; } - LLFILE* fp = LLFile::fopen(fileName, TEXT("rb")); /*Flawfinder: ignore*/ + LLFILE* fp = LLFile::fopen(fileName, LLFILE_MODE("rb")); /*Flawfinder: ignore*/ if (!fp) { LLError::LLUserWarningMsg::showMissingFiles(); diff --git a/indra/llcommon/llcrc.cpp b/indra/llcommon/llcrc.cpp index f836a12c35..33bf01a29b 100644 --- a/indra/llcommon/llcrc.cpp +++ b/indra/llcommon/llcrc.cpp @@ -166,7 +166,7 @@ void LLCRC::update(const std::string& filename) return; } - FILE* fp = LLFile::fopen(filename, TEXT("rb")); /* Flawfinder: ignore */ + FILE* fp = LLFile::fopen(filename, LLFILE_MODE("rb")); /* Flawfinder: ignore */ if (fp) { diff --git a/indra/llcommon/llcrc.h b/indra/llcommon/llcrc.h index cbf4cf4e75..2ea5fbc3b0 100644 --- a/indra/llcommon/llcrc.h +++ b/indra/llcommon/llcrc.h @@ -36,7 +36,7 @@ // example (don't try this at work kids): // // LLCRC crc; -// FILE* fp = LLFile::fopen(filename,TEXT("rb")); +// FILE* fp = LLFile::fopen(filename,LLFILE_MODE("rb")); // while(!feof(fp)) { // crc.update(fgetc(fp)); // } diff --git a/indra/llcommon/llfile.h b/indra/llcommon/llfile.h index 7e8e114098..8de3f5c9b4 100755 --- a/indra/llcommon/llfile.h +++ b/indra/llcommon/llfile.h @@ -56,12 +56,10 @@ typedef FILE LLFILE; #include "fsyspath.h" #include "llstring.h" // safe char* -> std::string conversion -#ifndef TEXT #if LL_WINDOWS -#define TEXT(flags) L##flags +#define LLFILE_MODE(flags) L##flags #else -#define TEXT(flags) flags -#endif +#define LLFILE_MODE(flags) flags #endif /// This class provides a selection of functions to operate on files through names and @@ -415,7 +413,7 @@ public: std::error_code ec; return copy(source, target, std::filesystem::copy_options::overwrite_existing, ec); } - inline static bool copy(const std::string& source, char* target) + inline static bool copy(const std::string& source, const char* target) { std::error_code ec; return copy(source, target, std::filesystem::copy_options::overwrite_existing, ec); @@ -686,7 +684,7 @@ private: #else typedef int llfile_handle_t; const llfile_handle_t InvalidHandle = -1; - llfile_handle_t mHandle = -1;; // The file handle/descriptor + llfile_handle_t mHandle = -1; // The file handle/descriptor #endif std::ios_base::openmode mOpen{}; // Used to emulate std::ios_base::app under Windows diff --git a/indra/llcommon/llprocessor.cpp b/indra/llcommon/llprocessor.cpp index 7656986fe2..9d0ef45bee 100644 --- a/indra/llcommon/llprocessor.cpp +++ b/indra/llcommon/llprocessor.cpp @@ -837,7 +837,7 @@ private: void get_proc_cpuinfo() { std::map< std::string, std::string > cpuinfo; - LLFILE* cpuinfo_fp = LLFile::fopen(CPUINFO_FILE, TEXT("rb")); + LLFILE* cpuinfo_fp = LLFile::fopen(CPUINFO_FILE, LLFILE_MODE("rb")); if(cpuinfo_fp) { char line[MAX_STRING]; @@ -954,7 +954,7 @@ private: std::ostringstream s; // *NOTE:Mani - This is for linux only. - LLFILE* cpuinfo = LLFile::fopen(CPUINFO_FILE, TEXT("rb")); + LLFILE* cpuinfo = LLFile::fopen(CPUINFO_FILE, LLFILE_MODE("rb")); if(cpuinfo) { char line[MAX_STRING]; diff --git a/indra/llcommon/llsys.cpp b/indra/llcommon/llsys.cpp index 499de996e2..5ef042b5a7 100644 --- a/indra/llcommon/llsys.cpp +++ b/indra/llcommon/llsys.cpp @@ -514,7 +514,7 @@ namespace { #if LL_LINUX constexpr U32 STATUS_SIZE = 2048; - LLFILE* status_filep = LLFile::fopen("/proc/self/status", TEXT("rb")); + LLFILE* status_filep = LLFile::fopen("/proc/self/status", LLFILE_MODE("rb")); if (status_filep) { char buff[STATUS_SIZE]; /* Flawfinder: ignore */ @@ -1273,7 +1273,7 @@ bool gunzip_file(const std::string& srcfile, const std::string& dstfile) src = gzopen(srcfile.c_str(), "rb"); #endif if (! src) goto err; - dst = LLFile::fopen(tmpfile, TEXT("wb")); /* Flawfinder: ignore */ + dst = LLFile::fopen(tmpfile, LLFILE_MODE("wb")); /* Flawfinder: ignore */ if (! dst) goto err; do { @@ -1314,7 +1314,7 @@ bool gzip_file(const std::string& srcfile, const std::string& dstfile) #endif if (! dst) goto err; - src = LLFile::fopen(srcfile, TEXT("rb")); /* Flawfinder: ignore */ + src = LLFile::fopen(srcfile, LLFILE_MODE("rb")); /* Flawfinder: ignore */ if (! src) goto err; while ((bytes = (S32)fread(buffer, sizeof(U8), COMPRESS_BUFFER_SIZE, src)) > 0) diff --git a/indra/llfilesystem/lldir.cpp b/indra/llfilesystem/lldir.cpp index 3ec8983e8a..1a647362c5 100755 --- a/indra/llfilesystem/lldir.cpp +++ b/indra/llfilesystem/lldir.cpp @@ -1010,7 +1010,7 @@ bool LLDir::setCacheDir(const std::string &path) { LLFile::mkdir(path); std::string tempname = add(path, "temp"); - LLFILE* file = LLFile::fopen(tempname, TEXT("wt")); + LLFILE* file = LLFile::fopen(tempname, LLFILE_MODE("wt")); if (file) { fclose(file); diff --git a/indra/llfilesystem/tests/lldir_test.cpp b/indra/llfilesystem/tests/lldir_test.cpp index b879fef48a..412feb147b 100644 --- a/indra/llfilesystem/tests/lldir_test.cpp +++ b/indra/llfilesystem/tests/lldir_test.cpp @@ -405,7 +405,7 @@ namespace tut std::string makeTestFile( const std::string& dir, const std::string& file ) { std::string path = dir + file; - LLFILE* handle = LLFile::fopen(path, TEXT("w")); + LLFILE* handle = LLFile::fopen(path, LLFILE_MODE("w")); ensure("failed to open test file '"+path+"'", handle != NULL ); // Harbison & Steele, 4th ed., p. 366: "If an error occurs, fputs // returns EOF; otherwise, it returns some other, nonnegative value." diff --git a/indra/llimage/llimagedimensionsinfo.cpp b/indra/llimage/llimagedimensionsinfo.cpp index 1d93014e1b..2ae7760d4c 100644 --- a/indra/llimage/llimagedimensionsinfo.cpp +++ b/indra/llimage/llimagedimensionsinfo.cpp @@ -166,7 +166,7 @@ bool LLImageDimensionsInfo::getImageDimensionsJpeg() { sJpegErrorEncountered = false; clean(); - FILE* fp = LLFile::fopen(mSrcFilename, TEXT("rb")); + FILE* fp = LLFile::fopen(mSrcFilename, LLFILE_MODE("rb")); if (!fp) { setLastError("Unable to open file for reading", mSrcFilename); diff --git a/indra/llimage/llimagetga.cpp b/indra/llimage/llimagetga.cpp index 7c6686db95..7b09553974 100644 --- a/indra/llimage/llimagetga.cpp +++ b/indra/llimage/llimagetga.cpp @@ -1191,7 +1191,7 @@ bool LLImageTGA::loadFile( const std::string& path ) return false; } - LLFILE* file = LLFile::fopen(path, TEXT("rb")); /* Flawfinder: ignore */ + LLFILE* file = LLFile::fopen(path, LLFILE_MODE("rb")); /* Flawfinder: ignore */ if( !file ) { LL_WARNS() << "Couldn't open file " << path << LL_ENDL; diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 13be0161b4..a8f0554a71 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -1402,7 +1402,7 @@ void LLAssetStorage::legacyGetDataCallback(const LLUUID &uuid, uuid.toString(uuid_str); filename = llformat("%s.%s",gDirUtilp->getExpandedFilename(LL_PATH_CACHE,uuid_str).c_str(),LLAssetType::lookup(type)); - LLFILE* fp = LLFile::fopen(filename, TEXT("wb")); /* Flawfinder: ignore */ + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("wb")); /* Flawfinder: ignore */ if (fp) { const S32 buf_size = 65536; diff --git a/indra/llmessage/lltransfersourcefile.cpp b/indra/llmessage/lltransfersourcefile.cpp index c2bf5c2afa..d2d264c520 100644 --- a/indra/llmessage/lltransfersourcefile.cpp +++ b/indra/llmessage/lltransfersourcefile.cpp @@ -62,7 +62,7 @@ void LLTransferSourceFile::initTransfer() return; } // Look for the file. - mFP = LLFile::fopen(mParams.getFilename(), TEXT("rb")); /* Flawfinder: ignore */ + mFP = LLFile::fopen(mParams.getFilename(), LLFILE_MODE("rb")); /* Flawfinder: ignore */ if (!mFP) { sendTransferStatus(LLTS_ERROR); diff --git a/indra/llmessage/lltransfertargetfile.cpp b/indra/llmessage/lltransfertargetfile.cpp index cc66befcae..05dba219ed 100644 --- a/indra/llmessage/lltransfertargetfile.cpp +++ b/indra/llmessage/lltransfertargetfile.cpp @@ -75,7 +75,7 @@ LLTSCode LLTransferTargetFile::dataCallback(const S32 packet_id, U8 *in_datap, c if (!mFP) { - mFP = LLFile::fopen(mParams.mFilename, TEXT("wb")); /* Flawfinder: ignore */ + mFP = LLFile::fopen(mParams.mFilename, LLFILE_MODE("wb")); /* Flawfinder: ignore */ if (!mFP) { diff --git a/indra/llmessage/llxfer_file.cpp b/indra/llmessage/llxfer_file.cpp index f0310c6297..5f27cc4827 100644 --- a/indra/llmessage/llxfer_file.cpp +++ b/indra/llmessage/llxfer_file.cpp @@ -161,7 +161,7 @@ S32 LLXfer_File::initializeRequest(U64 xfer_id, S32 LLXfer_File::startDownload() { S32 retval = 0; // presume success - mFp = LLFile::fopen(mTempFilename,TEXT("w+b")); /* Flawfinder : ignore */ + mFp = LLFile::fopen(mTempFilename,LLFILE_MODE("w+b")); /* Flawfinder : ignore */ if (mFp) { fclose(mFp); @@ -209,7 +209,7 @@ S32 LLXfer_File::startSend (U64 xfer_id, const LLHost &remote_host) mBufferStartOffset = 0; // We leave the file open, assuming we'll start reading and sending soon - mFp = LLFile::fopen(mLocalFilename,TEXT("rb")); /* Flawfinder : ignore */ + mFp = LLFile::fopen(mLocalFilename, LLFILE_MODE("rb")); /* Flawfinder : ignore */ if (mFp) { fseek(mFp,0,SEEK_END); @@ -252,7 +252,7 @@ S32 LLXfer_File::reopenFileHandle() if (mFp == NULL) { - mFp = LLFile::fopen(mLocalFilename,TEXT("rb")); /* Flawfinder : ignore */ + mFp = LLFile::fopen(mLocalFilename, LLFILE_MODE("rb")); /* Flawfinder : ignore */ if (mFp == NULL) { LL_INFOS("Xfer") << "Warning: " << mLocalFilename << " not found when re-opening file" << LL_ENDL; @@ -313,7 +313,7 @@ S32 LLXfer_File::flush() { LL_ERRS("Xfer") << "Overwriting open file pointer!" << LL_ENDL; } - mFp = LLFile::fopen(mTempFilename,TEXT("a+b")); /* Flawfinder : ignore */ + mFp = LLFile::fopen(mTempFilename, LLFILE_MODE("a+b")); /* Flawfinder : ignore */ if (mFp) { @@ -381,11 +381,11 @@ S32 LLXfer_File::processEOF() } else { - //LLFILE* fp = LLFile::fopen(mTempFilename, TEXT("r")); + //LLFILE* fp = LLFile::fopen(mTempFilename, LLFILE_MODE("r")); //LL_WARNS() << "File " << mTempFilename << " does " // << (!fp ? "not" : "" ) << " exit." << LL_ENDL; //if(fp) fclose(fp); - //fp = LLFile::fopen(mLocalFilename, TEXT("r")); + //fp = LLFile::fopen(mLocalFilename, LLFILE_MODE("r")); //LL_WARNS() << "File " << mLocalFilename << " does " // << (!fp ? "not" : "" ) << " exit." << LL_ENDL; //if(fp) fclose(fp); @@ -452,8 +452,8 @@ U32 LLXfer_File::getXferTypeTag() S32 copy_file(const std::string& from, const std::string& to) { S32 rv = 0; - LLFILE* in = LLFile::fopen(from, TEXT("rb")); /*Flawfinder: ignore*/ - LLFILE* out = LLFile::fopen(to, TEXT("wb")); /*Flawfinder: ignore*/ + LLFILE* in = LLFile::fopen(from, LLFILE_MODE("rb")); /*Flawfinder: ignore*/ + LLFILE* out = LLFile::fopen(to, LLFILE_MODE("wb")); /*Flawfinder: ignore*/ if(in && out) { S32 read = 0; diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index 5b3b57f11e..df58c8a7f3 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -2376,7 +2376,7 @@ void dump_prehash_files() { U32 i; std::string filename("../../indra/llmessage/message_prehash.h"); - LLFILE* fp = LLFile::fopen(filename, TEXT("w")); /* Flawfinder: ignore */ + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("w")); /* Flawfinder: ignore */ if (fp) { fprintf( @@ -2407,7 +2407,7 @@ void dump_prehash_files() fclose(fp); } filename = std::string("../../indra/llmessage/message_prehash.cpp"); - fp = LLFile::fopen(filename, TEXT("w")); /* Flawfinder: ignore */ + fp = LLFile::fopen(filename, LLFILE_MODE("w")); /* Flawfinder: ignore */ if (fp) { fprintf( diff --git a/indra/llmessage/tests/lldatapacker_test.cpp b/indra/llmessage/tests/lldatapacker_test.cpp index 92a2c31ca8..5823980d43 100644 --- a/indra/llmessage/tests/lldatapacker_test.cpp +++ b/indra/llmessage/tests/lldatapacker_test.cpp @@ -348,7 +348,7 @@ namespace tut { F32 f_val = 44.44f, f_unpkval; - LLFILE* fp = LLFile::fopen(TEST_FILE_NAME, TEXT("w+")); + LLFILE* fp = LLFile::fopen(TEST_FILE_NAME, LLFILE_MODE("w+")); if(!fp) { LL_ERRS() << "File couldnt be open" << LL_ENDL; @@ -403,7 +403,7 @@ namespace tut LLVector4 unpkllvec4; LLUUID unpkuuid; - LLFILE* fp = LLFile::fopen(TEST_FILE_NAME,TEXT("w+")); + LLFILE* fp = LLFile::fopen(TEST_FILE_NAME, LLFILE_MODE("w+")); if(!fp) { LL_ERRS() << "File couldnt be open" << LL_ENDL; diff --git a/indra/llrender/llshadermgr.cpp b/indra/llrender/llshadermgr.cpp index 0ed98f2e7e..f6738364a6 100644 --- a/indra/llrender/llshadermgr.cpp +++ b/indra/llrender/llshadermgr.cpp @@ -501,7 +501,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev open_file_name = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "shaders/errorF.glsl"); } - file = LLFile::fopen(open_file_name, TEXT("r")); + file = LLFile::fopen(open_file_name, LLFILE_MODE("r")); } else #endif @@ -529,7 +529,7 @@ GLuint LLShaderMgr::loadShaderFile(const std::string& filename, S32 & shader_lev */ LL_DEBUGS("ShaderLoading") << "Looking in " << open_file_name << LL_ENDL; - file = LLFile::fopen(open_file_name, TEXT("r")); /* Flawfinder: ignore */ + file = LLFile::fopen(open_file_name, LLFILE_MODE("r")); /* Flawfinder: ignore */ if (file) { LL_DEBUGS("ShaderLoading") << "Loading file: " << open_file_name << " (Want class " << gpu_class << ")" << LL_ENDL; diff --git a/indra/llui/lluicolortable.cpp b/indra/llui/lluicolortable.cpp index 206d02b366..cd062b23b8 100644 --- a/indra/llui/lluicolortable.cpp +++ b/indra/llui/lluicolortable.cpp @@ -298,7 +298,7 @@ void LLUIColorTable::saveUserSettings() const if(!output_node->isNull()) { const std::string& filename = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, "colors.xml"); - LLFILE *fp = LLFile::fopen(filename, TEXT("w")); + LLFILE *fp = LLFile::fopen(filename, LLFILE_MODE("w")); if(fp != NULL) { diff --git a/indra/llui/llxuiparser.cpp b/indra/llui/llxuiparser.cpp index e3d806bdd0..9bf2f4220c 100644 --- a/indra/llui/llxuiparser.cpp +++ b/indra/llui/llxuiparser.cpp @@ -637,7 +637,7 @@ void LLXUIXSDWriter::writeXSD(const std::string& type_name, const std::string& p } } - LLFILE* xsd_file = LLFile::fopen(file_name.c_str(), TEXT("w")); + LLFILE* xsd_file = LLFile::fopen(file_name.c_str(), LLFILE_MODE("w")); LLXMLNode::writeHeaderToFile(xsd_file); root_nodep->writeToFile(xsd_file); fclose(xsd_file); @@ -1397,7 +1397,7 @@ bool LLSimpleXUIParser::readXUI(const std::string& filename, LLInitParam::BaseBl mCurReadDepth = 0; setParseSilently(silent); - ScopedFile file(filename, TEXT("rb")); + ScopedFile file(filename, LLFILE_MODE("rb")); if( !file.isOpen() ) { LL_WARNS("ReadXUI") << "Unable to open file " << filename << LL_ENDL; diff --git a/indra/llxml/llcontrol.cpp b/indra/llxml/llcontrol.cpp index a8e44f7bdd..24d2facc4c 100644 --- a/indra/llxml/llcontrol.cpp +++ b/indra/llxml/llcontrol.cpp @@ -399,7 +399,7 @@ void LLControlGroup::cleanup() if(mSettingsProfile && getCount.size() != 0) { std::string file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, SETTINGS_PROFILE); - LLFILE* out = LLFile::fopen(file, TEXT("w")); /* Flawfinder: ignore */ + LLFILE* out = LLFile::fopen(file, LLFILE_MODE("w")); /* Flawfinder: ignore */ if(!out) { LL_WARNS("SettingsProfile") << "Error opening " << SETTINGS_PROFILE << LL_ENDL; diff --git a/indra/llxml/llxmlparser.cpp b/indra/llxml/llxmlparser.cpp index 5aac9100dd..8afd412cda 100644 --- a/indra/llxml/llxmlparser.cpp +++ b/indra/llxml/llxmlparser.cpp @@ -72,7 +72,7 @@ bool LLXmlParser::parseFile(const std::string &path) bool success = true; - LLFILE* file = LLFile::fopen(path, TEXT("rb")); /* Flawfinder: ignore */ + LLFILE* file = LLFile::fopen(path, LLFILE_MODE("rb")); /* Flawfinder: ignore */ if( !file ) { mAuxErrorString = llformat( "Couldn't open file %s", path.c_str()); @@ -388,7 +388,7 @@ int main() { char buf[1024]; - LLFILE* file = LLFile::fopen("test.xml", TEXT("rb")); + LLFILE* file = LLFile::fopen("test.xml", LLFILE_MODE("rb")); if( !file ) { return 1; diff --git a/indra/newview/llconversationlog.cpp b/indra/newview/llconversationlog.cpp index 63a3743ef7..bfcc2e4182 100644 --- a/indra/newview/llconversationlog.cpp +++ b/indra/newview/llconversationlog.cpp @@ -512,7 +512,7 @@ bool LLConversationLog::saveToFile(const std::string& filename) return false; } - LLFILE* fp = LLFile::fopen(filename, TEXT("wb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("wb")); if (!fp) { LL_WARNS() << "Couldn't open call log list" << filename << LL_ENDL; @@ -556,7 +556,7 @@ bool LLConversationLog::loadFromFile(const std::string& filename) return false; } - LLFILE* fp = LLFile::fopen(filename, TEXT("rb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("rb")); if (!fp) { LL_WARNS() << "Couldn't open call log list" << filename << LL_ENDL; diff --git a/indra/newview/lldrawpoolbump.cpp b/indra/newview/lldrawpoolbump.cpp index 200fce7b5b..a42972205a 100644 --- a/indra/newview/lldrawpoolbump.cpp +++ b/indra/newview/lldrawpoolbump.cpp @@ -114,7 +114,7 @@ void LLStandardBumpmap::addstandard() gStandardBumpmapList[LLStandardBumpmap::sStandardBumpmapCount++] = LLStandardBumpmap("Darkness"); // BE_DARKNESS std::string file_name = gDirUtilp->getExpandedFilename( LL_PATH_APP_SETTINGS, "std_bump.ini" ); - LLFILE* file = LLFile::fopen( file_name, TEXT("rt") ); /*Flawfinder: ignore*/ + LLFILE* file = LLFile::fopen( file_name, LLFILE_MODE("rt") ); /*Flawfinder: ignore*/ if( !file ) { LL_WARNS() << "Could not open std_bump <" << file_name << ">" << LL_ENDL; diff --git a/indra/newview/llfloateruipreview.cpp b/indra/newview/llfloateruipreview.cpp index cf710d45dd..637871da0d 100755 --- a/indra/newview/llfloateruipreview.cpp +++ b/indra/newview/llfloateruipreview.cpp @@ -574,7 +574,7 @@ void LLFloaterUIPreview::onClickExportSchema() // std::string file_name(template_path + gDirUtilp->getDirDelimiter() + widget_name + ".rng"); - // LLFILE* rng_file = LLFile::fopen(file_name.c_str(), TEXT("w")); + // LLFILE* rng_file = LLFile::fopen(file_name.c_str(), LLFILE_MODE("w")); // { // LLXMLNode::writeHeaderToFile(rng_file); // const bool use_type_decorations = false; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 9491c9d610..c34df48f7d 100644 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -2745,7 +2745,7 @@ bool LLInventoryModel::loadSkeleton( const S32 NO_VERSION = LLViewerInventoryCategory::VERSION_UNKNOWN; std::string gzip_filename(inventory_filename); gzip_filename.append(".gz"); - LLFILE* fp = LLFile::fopen(gzip_filename, TEXT("rb")); + LLFILE* fp = LLFile::fopen(gzip_filename, LLFILE_MODE("rb")); bool remove_inventory_file = false; if (LLAppViewer::instance()->isSecondInstance()) { @@ -4908,7 +4908,7 @@ bool decompress_file(const char* src_filename, const char* dst_filename) // open the files src = gzopen(src_filename, "rb"); if(!src) goto err_decompress; - dst = LLFile::fopen(dst_filename, TEXT("wb")); + dst = LLFile::fopen(dst_filename, LLFILE_MODE("wb")); if(!dst) goto err_decompress; // decompress. diff --git a/indra/newview/llkeyconflict.cpp b/indra/newview/llkeyconflict.cpp index 458d5225e1..c082c632f1 100644 --- a/indra/newview/llkeyconflict.cpp +++ b/indra/newview/llkeyconflict.cpp @@ -575,7 +575,7 @@ void LLKeyConflictHandler::saveToSettings(bool temporary) // Write the resulting XML to file if (!output_node->isNull()) { - LLFILE *fp = LLFile::fopen(filename, TEXT("w")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("w")); if (fp != NULL) { LLXMLNode::writeHeaderToFile(fp); diff --git a/indra/newview/lllogchat.cpp b/indra/newview/lllogchat.cpp index 405530f5bd..59f6993a0b 100644 --- a/indra/newview/lllogchat.cpp +++ b/indra/newview/lllogchat.cpp @@ -453,7 +453,7 @@ void LLLogChat::loadChatHistory(const std::string& file_name, std::list& m // If we got here, we managed to stat the file. // Open the file to read in binary mode to prevent interpreting other characters as EOF - LLFILE* fptr = LLFile::fopen(log_file_name, TEXT("rb")); /*Flawfinder: ignore*/ + LLFILE* fptr = LLFile::fopen(log_file_name, LLFILE_MODE("rb")); /*Flawfinder: ignore*/ if (!fptr) { // Ok, this is strange but not really tragic in the big picture of things LL_WARNS("ChatHistory") << "Unable to read file " << log_file_name << " after stat was successful" << LL_ENDL; @@ -890,7 +890,7 @@ bool LLLogChat::isAdHocTranscriptExist(std::string file_name) bool LLLogChat::isTranscriptFileFound(std::string fullname) { bool result = false; - LLFILE * filep = LLFile::fopen(fullname, TEXT("rb")); + LLFILE * filep = LLFile::fopen(fullname, LLFILE_MODE("rb")); if (NULL != filep) { if (makeLogFileName("chat") == fullname) @@ -1183,7 +1183,7 @@ void LLLoadHistoryThread::loadHistory(const std::string& file_name, std::listgetExpandedFilename(LL_PATH_PER_SL_ACCOUNT, "render_mute_settings.txt"); - LLFILE* fp = LLFile::fopen(filename, TEXT("wb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("wb")); if (!fp) { LL_WARNS() << "Couldn't open render mute list file: " << filename << LL_ENDL; @@ -963,7 +963,7 @@ bool LLRenderMuteList::saveToFile() bool LLRenderMuteList::loadFromFile() { std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_PER_SL_ACCOUNT, "render_mute_settings.txt"); - LLFILE* fp = LLFile::fopen(filename, TEXT("rb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("rb")); if (!fp) { LL_WARNS() << "Couldn't open render mute list file: " << filename << LL_ENDL; diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp index 6928d7e6c3..cecda23e7b 100644 --- a/indra/newview/llpreviewnotecard.cpp +++ b/indra/newview/llpreviewnotecard.cpp @@ -801,7 +801,7 @@ bool LLPreviewNotecard::loadNotecardText(const std::string& filename) return false; } - LLFILE* file = LLFile::fopen(filename, TEXT("rb")); /*Flawfinder: ignore*/ + LLFILE* file = LLFile::fopen(filename, LLFILE_MODE("rb")); /*Flawfinder: ignore*/ if (!file) { LL_WARNS() << "Error opening " << filename << LL_ENDL; @@ -832,7 +832,7 @@ bool LLPreviewNotecard::loadNotecardText(const std::string& filename) bool LLPreviewNotecard::writeToFile(const std::string& filename) { - LLFILE* fp = LLFile::fopen(filename, TEXT("wb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("wb")); if (!fp) { LL_WARNS() << "Unable to write to " << filename << LL_ENDL; diff --git a/indra/newview/llpreviewscript.cpp b/indra/newview/llpreviewscript.cpp index c73976e540..53d0a712a8 100644 --- a/indra/newview/llpreviewscript.cpp +++ b/indra/newview/llpreviewscript.cpp @@ -632,7 +632,7 @@ bool LLScriptEdCore::loadScriptText(const std::string& filename) return false; } - LLFILE* file = LLFile::fopen(filename, TEXT("rb")); /*Flawfinder: ignore*/ + LLFILE* file = LLFile::fopen(filename, LLFILE_MODE("rb")); /*Flawfinder: ignore*/ if (!file) { LL_WARNS() << "Error opening " << filename << LL_ENDL; @@ -663,7 +663,7 @@ bool LLScriptEdCore::loadScriptText(const std::string& filename) bool LLScriptEdCore::writeToFile(const std::string& filename) { - LLFILE* fp = LLFile::fopen(filename, TEXT("wb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("wb")); if (!fp) { LL_WARNS() << "Unable to write to " << filename << LL_ENDL; diff --git a/indra/newview/llstartup.cpp b/indra/newview/llstartup.cpp index 37609f74ed..2cb2a7f8d5 100644 --- a/indra/newview/llstartup.cpp +++ b/indra/newview/llstartup.cpp @@ -587,7 +587,7 @@ bool idle_startup() std::string message_template_path = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS,"message_template.msg"); LLFILE* found_template = NULL; - found_template = LLFile::fopen(message_template_path, TEXT("r")); /* Flawfinder: ignore */ + found_template = LLFile::fopen(message_template_path, LLFILE_MODE("r")); /* Flawfinder: ignore */ #if LL_WINDOWS // On the windows dev builds, unpackaged, the message_template.msg @@ -596,7 +596,7 @@ bool idle_startup() if (!found_template) { message_template_path = gDirUtilp->getExpandedFilename(LL_PATH_EXECUTABLE, "app_settings", "message_template.msg"); - found_template = LLFile::fopen(message_template_path.c_str(), TEXT("r")); /* Flawfinder: ignore */ + found_template = LLFile::fopen(message_template_path.c_str(), LLFILE_MODE("r")); /* Flawfinder: ignore */ } #elif LL_DARWIN // On Mac dev builds, message_template.msg lives in: @@ -606,7 +606,7 @@ bool idle_startup() message_template_path = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "message_template.msg"); - found_template = LLFile::fopen(message_template_path.c_str(), TEXT("r")); /* Flawfinder: ignore */ + found_template = LLFile::fopen(message_template_path.c_str(), LLFILE_MODE("r")); /* Flawfinder: ignore */ } #endif diff --git a/indra/newview/lltoolbarview.cpp b/indra/newview/lltoolbarview.cpp index d1e36676af..c401a00fd7 100644 --- a/indra/newview/lltoolbarview.cpp +++ b/indra/newview/lltoolbarview.cpp @@ -420,7 +420,7 @@ void LLToolBarView::saveToolbars() const if(!output_node->isNull()) { const std::string& filename = gDirUtilp->getExpandedFilename(LL_PATH_PER_SL_ACCOUNT, "toolbars.xml"); - LLFILE *fp = LLFile::fopen(filename, TEXT("w")); + LLFILE *fp = LLFile::fopen(filename, LLFILE_MODE("w")); if (fp != NULL) { LLXMLNode::writeHeaderToFile(fp); diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp index c28f3dc73a..867d47c893 100644 --- a/indra/newview/llviewerassetstorage.cpp +++ b/indra/newview/llviewerassetstorage.cpp @@ -279,7 +279,7 @@ void LLViewerAssetStorage::storeAssetData( LL_DEBUGS("AssetStorage") << "ASSET_ID: " << asset_id << LL_ENDL; S32 size = 0; - LLFILE* fp = LLFile::fopen(filename, TEXT("rb")); + LLFILE* fp = LLFile::fopen(filename, LLFILE_MODE("rb")); if (fp) { fseek(fp, 0, SEEK_END); diff --git a/indra/newview/llviewerinput.cpp b/indra/newview/llviewerinput.cpp index 9b39f7b0e9..5741d7f5fb 100644 --- a/indra/newview/llviewerinput.cpp +++ b/indra/newview/llviewerinput.cpp @@ -1430,7 +1430,7 @@ S32 LLViewerInput::loadBindingsXML(const std::string& filename) // file in app_settings is supposed to be up to date // this is only for the file from user_settings LL_INFOS("ViewerInput") << "Updating file " << filename << " to a newer version" << LL_ENDL; - LLFILE *fp = LLFile::fopen(filename, TEXT("w")); + LLFILE *fp = LLFile::fopen(filename, LLFILE_MODE("w")); if (fp != NULL) { LLXMLNode::writeHeaderToFile(fp); diff --git a/indra/newview/llviewermenufile.cpp b/indra/newview/llviewermenufile.cpp index a8ed2da9e0..423bb5f31d 100644 --- a/indra/newview/llviewermenufile.cpp +++ b/indra/newview/llviewermenufile.cpp @@ -1161,7 +1161,7 @@ void handle_compress_image() // so doing dirty, but OS independent fopen and fseek size_t get_file_size(std::string &filename) { - LLFILE* file = LLFile::fopen(filename, TEXT("rb")); /*Flawfinder: ignore*/ + LLFILE* file = LLFile::fopen(filename, LLFILE_MODE("rb")); /*Flawfinder: ignore*/ if (!file) { LL_WARNS() << "Error opening " << filename << LL_ENDL; diff --git a/indra/newview/llviewerstatsrecorder.cpp b/indra/newview/llviewerstatsrecorder.cpp index a2141f4eed..a183d7c8de 100644 --- a/indra/newview/llviewerstatsrecorder.cpp +++ b/indra/newview/llviewerstatsrecorder.cpp @@ -194,7 +194,7 @@ void LLViewerStatsRecorder::writeToLog( F32 interval ) // Open the data file makeStatsFileName(); - mStatsFile = LLFile::fopen(mStatsFileName, TEXT("wb")); + mStatsFile = LLFile::fopen(mStatsFileName, LLFILE_MODE("wb")); if (mStatsFile) { -- cgit v1.3 From a1b68fb8ed6c2589dd13bd9ba1fab7050ba38226 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 27 May 2026 10:18:29 -0700 Subject: Drop fewer packets on UDP packet flood (#5565) * ACK received-but-unprocessed packets when queued * Prioritize packets between high- and low-priority (high-priority are: reliable packets that need an ACK) * Increase max size of LLPacketRing buffer --- indra/llmessage/llcircuit.cpp | 5 +- indra/llmessage/llpacketbuffer.h | 3 + indra/llmessage/llpacketring.cpp | 315 ++++++------------------------------- indra/llmessage/llpacketring.h | 88 +++-------- indra/llmessage/message.cpp | 315 ++++++++++++++++++++++++++++++++++--- indra/llmessage/message.h | 97 +++++++++++- indra/llmessage/net.cpp | 2 +- indra/newview/llappviewer.cpp | 79 +++++----- indra/newview/llstartup.cpp | 4 +- indra/newview/llviewermenu.cpp | 2 +- indra/newview/llviewerstats.cpp | 2 +- indra/newview/llviewerthrottle.cpp | 23 ++- indra/newview/llviewerthrottle.h | 4 +- indra/newview/llworld.cpp | 10 +- 14 files changed, 529 insertions(+), 420 deletions(-) (limited to 'indra/llmessage/message.cpp') diff --git a/indra/llmessage/llcircuit.cpp b/indra/llmessage/llcircuit.cpp index c2b1a2f069..eaee7b0112 100644 --- a/indra/llmessage/llcircuit.cpp +++ b/indra/llmessage/llcircuit.cpp @@ -346,8 +346,7 @@ S32 LLCircuitData::resendUnackedPackets(const F64Seconds now) packetp->mBuffer[0] |= LL_RESENT_FLAG; // tag packet id as being a resend - gMessageSystem->mPacketRing.sendPacket(packetp->mSocket, - (char *)packetp->mBuffer, packetp->mBufferLength, + gMessageSystem->sendPacketToSocket((char *)packetp->mBuffer, packetp->mBufferLength, packetp->mHost); mThrottles.throttleOverflow(TC_RESEND, packetp->mBufferLength * 8.f); @@ -973,7 +972,7 @@ bool LLCircuitData::updateWatchDogTimers(LLMessageSystem *msgsys) { // let's call this one a loss! mPacketsLost++; - gMessageSystem->mDroppedPackets++; + gMessageSystem->mLostPackets++; if(gMessageSystem->mVerboseLog) { std::ostringstream str; diff --git a/indra/llmessage/llpacketbuffer.h b/indra/llmessage/llpacketbuffer.h index ac4012d330..b0505e7afd 100644 --- a/indra/llmessage/llpacketbuffer.h +++ b/indra/llmessage/llpacketbuffer.h @@ -38,6 +38,9 @@ public: LLPacketBuffer(S32 hSocket); // receive a packet ~LLPacketBuffer(); + LLPacketBuffer(const LLPacketBuffer&) = default; + LLPacketBuffer& operator=(const LLPacketBuffer&) = default; + S32 getSize() const { return mSize; } const char *getData() const { return mData; } LLHost getHost() const { return mHost; } diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index b8284334ea..bade413e61 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -28,344 +28,121 @@ #include "llpacketring.h" -#if LL_WINDOWS - #include -#else - #include - #include -#endif - -// linden library includes #include "llerror.h" -#include "lltimer.h" -#include "llproxy.h" -#include "llrand.h" -#include "message.h" -#include "u64.h" -constexpr S16 MAX_BUFFER_RING_SIZE = 1024; +constexpr S16 MAX_BUFFER_RING_SIZE = 8192; + +// DANGER: don't adjust DEFAULT_BUFFER_RING_SIZE unless you know what +// you're doing. Its value affects the "buffer load rate" which is used +// to supply backpressure to an overloaded nework queue. constexpr S16 DEFAULT_BUFFER_RING_SIZE = 256; -LLPacketRing::LLPacketRing () - : mPacketRing(DEFAULT_BUFFER_RING_SIZE, nullptr) +LLPacketRing::LLPacketRing() + : mRing(DEFAULT_BUFFER_RING_SIZE, nullptr) { LLHost invalid_host; - for (size_t i = 0; i < mPacketRing.size(); ++i) + for (size_t i = 0; i < mRing.size(); ++i) { - mPacketRing[i] = new LLPacketBuffer(invalid_host, nullptr, 0); + mRing[i] = new LLPacketBuffer(invalid_host, nullptr, 0); } } -LLPacketRing::~LLPacketRing () +LLPacketRing::~LLPacketRing() { - for (auto packet : mPacketRing) + for (auto* packet : mRing) { delete packet; } - mPacketRing.clear(); + mRing.clear(); mNumBufferedPackets = 0; mNumBufferedBytes = 0; mHeadIndex = 0; } -S32 LLPacketRing::receivePacket (S32 socket, char *datap) -{ - bool drop = computeDrop(); - return (mNumBufferedPackets > 0) ? - receiveOrDropBufferedPacket(datap, drop) : - receiveOrDropPacket(socket, datap, drop); -} - -bool send_packet_helper(int socket, const char * datap, S32 data_size, LLHost host) -{ - if (!LLProxy::isSOCKSProxyEnabled()) - { - return send_packet(socket, datap, data_size, host.getAddress(), host.getPort()); - } - - char headered_send_buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; - - proxywrap_t *socks_header = static_cast(static_cast(&headered_send_buffer)); - socks_header->rsv = 0; - socks_header->addr = host.getAddress(); - socks_header->port = htons(host.getPort()); - socks_header->atype = ADDRESS_IPV4; - socks_header->frag = 0; - - memcpy(headered_send_buffer + SOCKS_HEADER_SIZE, datap, data_size); - - return send_packet( socket, - headered_send_buffer, - data_size + SOCKS_HEADER_SIZE, - LLProxy::getInstance()->getUDPProxy().getAddress(), - LLProxy::getInstance()->getUDPProxy().getPort()); -} - -bool LLPacketRing::sendPacket(int socket, const char * datap, S32 data_size, LLHost host) +void LLPacketRing::pushPacket(const LLPacketBuffer& packet) { - mActualBytesOut += data_size; - return send_packet_helper(socket, datap, data_size, host); -} - -void LLPacketRing::dropPackets (U32 num_to_drop) -{ - mPacketsToDrop += num_to_drop; -} - -void LLPacketRing::setDropPercentage (F32 percent_to_drop) -{ - mDropPercentage = percent_to_drop; -} - -bool LLPacketRing::computeDrop() -{ - bool drop= (mDropPercentage > 0.0f && (ll_frand(100.f) < mDropPercentage)); - if (drop) - { - ++mPacketsToDrop; - } - if (mPacketsToDrop > 0) - { - --mPacketsToDrop; - drop = true; - } - return drop; -} - -S32 LLPacketRing::receiveOrDropPacket(S32 socket, char *datap, bool drop) -{ - S32 packet_size = 0; - - // pull straight from socket - if (LLProxy::isSOCKSProxyEnabled()) + S16 ring_size = (S16)mRing.size(); + if (mNumBufferedPackets >= ring_size && ring_size < MAX_BUFFER_RING_SIZE) { - char buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; /* Flawfinder ignore */ - packet_size = receive_packet(socket, buffer); - if (packet_size > 0) - { - mActualBytesIn += packet_size; - } - - if (packet_size > SOCKS_HEADER_SIZE) - { - if (drop) - { - packet_size = 0; - } - else - { - // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) - packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size - memcpy(datap, buffer + SOCKS_HEADER_SIZE, packet_size); - proxywrap_t * header = static_cast(static_cast(buffer)); - mLastSender.setAddress(header->addr); - mLastSender.setPort(ntohs(header->port)); - mLastReceivingIF = ::get_receiving_interface(); - } - } - else - { - packet_size = 0; - } - } - else - { - packet_size = receive_packet(socket, datap); - if (packet_size > 0) - { - mActualBytesIn += packet_size; - if (drop) - { - packet_size = 0; - } - else - { - mLastSender = ::get_sender(); - mLastReceivingIF = ::get_receiving_interface(); - } - } + expandRing(); + ring_size = (S16)mRing.size(); } - return packet_size; -} -S32 LLPacketRing::receiveOrDropBufferedPacket(char *datap, bool drop) -{ - assert(mNumBufferedPackets > 0); - S32 packet_size = 0; + LLPacketBuffer* slot = mRing[mHeadIndex]; + S32 old_size = slot->getSize(); - S16 ring_size = (S16)(mPacketRing.size()); - S16 packet_index = (mHeadIndex + ring_size - mNumBufferedPackets) % ring_size; - LLPacketBuffer* packet = mPacketRing[packet_index]; - packet_size = packet->getSize(); - mLastSender = packet->getHost(); - mLastReceivingIF = packet->getReceivingInterface(); + *slot = packet; - --mNumBufferedPackets; - mNumBufferedBytes -= packet_size; - if (mNumBufferedPackets == 0) - { - assert(mNumBufferedBytes == 0); - } + mHeadIndex = (mHeadIndex + 1) % ring_size; - if (!drop) + if (mNumBufferedPackets < ring_size) { - if (packet_size > 0) - { - memcpy(datap, packet->getData(), packet_size); - } - else - { - assert(false); - } + ++mNumBufferedPackets; + mNumBufferedBytes += packet.getSize(); } else { - packet_size = 0; + // Ring is at maximum capacity; oldest packet was overwritten. + // This is VERY BAD because we've already ACKed the packet we're loosing + // (if it was "reliable"). + LL_WARNS("PacketRing") << "buffer overflow at " << mNumBufferedPackets << " packets" << LL_ENDL; + mNumBufferedBytes += packet.getSize() - old_size; } - return packet_size; } -S32 LLPacketRing::bufferInboundPacket(S32 socket) +bool LLPacketRing::popPacket(LLPacketBuffer& packet) { - if (mNumBufferedPackets == mPacketRing.size() && mNumBufferedPackets < MAX_BUFFER_RING_SIZE) + if (mNumBufferedPackets <= 0) { - expandRing(); + return false; } - LLPacketBuffer* packet = mPacketRing[mHeadIndex]; - S32 old_packet_size = packet->getSize(); - S32 packet_size = 0; - if (LLProxy::isSOCKSProxyEnabled()) - { - char buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; /* Flawfinder ignore */ - packet_size = receive_packet(socket, buffer); - if (packet_size > 0) - { - mActualBytesIn += packet_size; - if (packet_size > SOCKS_HEADER_SIZE) - { - // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) + S16 ring_size = (S16)mRing.size(); + S16 tail_index = (mHeadIndex + ring_size - mNumBufferedPackets) % ring_size; - proxywrap_t * header = static_cast(static_cast(buffer)); - LLHost sender; - sender.setAddress(header->addr); - sender.setPort(ntohs(header->port)); + LLPacketBuffer* slot = mRing[tail_index]; + S32 packet_size = slot->getSize(); - packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size - packet->init(buffer + SOCKS_HEADER_SIZE, packet_size, sender); + packet = *slot; - mHeadIndex = (mHeadIndex + 1) % (S16)(mPacketRing.size()); - if (mNumBufferedPackets < MAX_BUFFER_RING_SIZE) - { - ++mNumBufferedPackets; - mNumBufferedBytes += packet_size; - } - else - { - // we overwrote an older packet - mNumBufferedBytes += packet_size - old_packet_size; - } - } - else - { - packet_size = 0; - } - } - } - else - { - packet->init(socket); - packet_size = packet->getSize(); - if (packet_size > 0) - { - mActualBytesIn += packet_size; + --mNumBufferedPackets; + mNumBufferedBytes -= packet_size; - mHeadIndex = (mHeadIndex + 1) % (S16)(mPacketRing.size()); - if (mNumBufferedPackets < MAX_BUFFER_RING_SIZE) - { - ++mNumBufferedPackets; - mNumBufferedBytes += packet_size; - } - else - { - // we overwrote an older packet - mNumBufferedBytes += packet_size - old_packet_size; - } - } - } - return packet_size; -} + llassert(mNumBufferedPackets > 0 || mNumBufferedBytes == 0); -S32 LLPacketRing::drainSocket(S32 socket) -{ - // drain into buffer - S32 packet_size = 1; - S32 num_loops = 0; - S32 old_num_packets = mNumBufferedPackets; - while (packet_size > 0) - { - packet_size = bufferInboundPacket(socket); - ++num_loops; - } - S32 num_dropped_packets = (num_loops - 1 + old_num_packets) - mNumBufferedPackets; - if (num_dropped_packets > 0) - { - // It will eventually be accounted by mDroppedPackets - // and mPacketsLost, but track it here for logging purposes. - mNumDroppedPackets += num_dropped_packets; - } - return (S32)(mNumBufferedPackets); + return true; } bool LLPacketRing::expandRing() { - // compute larger size - constexpr S16 BUFFER_RING_EXPANSION = 256; - S16 old_size = (S16)(mPacketRing.size()); + constexpr S16 BUFFER_RING_EXPANSION = 512; + S16 old_size = (S16)mRing.size(); S16 new_size = llmin(old_size + BUFFER_RING_EXPANSION, MAX_BUFFER_RING_SIZE); if (new_size == old_size) { - // mPacketRing is already maxed out return false; } - // make a larger ring and copy packet pointers + // Lay existing entries out linearly in FIFO order starting at index 0. std::vector new_ring(new_size, nullptr); for (S16 i = 0; i < old_size; ++i) { S16 j = (mHeadIndex + i) % old_size; - new_ring[i] = mPacketRing[j]; + new_ring[i] = mRing[j]; } - // allocate new packets for the remainder of new_ring LLHost invalid_host; for (S16 i = old_size; i < new_size; ++i) { new_ring[i] = new LLPacketBuffer(invalid_host, nullptr, 0); } - // swap the rings and reset mHeadIndex - mPacketRing.swap(new_ring); + mRing.swap(new_ring); mHeadIndex = mNumBufferedPackets; return true; } F32 LLPacketRing::getBufferLoadRate() const { - // goes up to MAX_BUFFER_RING_SIZE return (F32)mNumBufferedPackets / (F32)DEFAULT_BUFFER_RING_SIZE; } - -void LLPacketRing::dumpPacketRingStats() -{ - mNumDroppedPacketsTotal += mNumDroppedPackets; - LL_INFOS("Messaging") << "Packet ring stats: " << std::endl - << "Buffered packets: " << mNumBufferedPackets << std::endl - << "Buffered bytes: " << mNumBufferedBytes << std::endl - << "Dropped packets current: " << mNumDroppedPackets << std::endl - << "Dropped packets total: " << mNumDroppedPacketsTotal << std::endl - << "Dropped packets percentage: " << mDropPercentage << "%" << std::endl - << "Actual in bytes: " << mActualBytesIn << std::endl - << "Actual out bytes: " << mActualBytesOut << LL_ENDL; - mNumDroppedPackets = 0; -} diff --git a/indra/llmessage/llpacketring.h b/indra/llmessage/llpacketring.h index 572dcbd271..5315e5a5bd 100644 --- a/indra/llmessage/llpacketring.h +++ b/indra/llmessage/llpacketring.h @@ -1,7 +1,16 @@ /** * @file llpacketring.h - * @brief definition of LLPacketRing class for implementing a resend, - * drop, or delay in packet transmissions + * @brief LLPacketRing: a simple ring buffer for LLPacketBuffers. + * + * LLPacketRing stores incoming UDP packets that have already been received + * from the network socket. It has no socket or proxy awareness; callers + * push packets in with pushPacket() and retrieve them in FIFO order with + * popPacket(). + * + * The ring starts at DEFAULT_BUFFER_RING_SIZE slots and grows in increments + * of BUFFER_RING_EXPANSION up to MAX_BUFFER_RING_SIZE. Once at the ceiling, + * pushPacket() silently overwrites the oldest queued packet to make room for + * the incoming one. * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code @@ -29,9 +38,7 @@ #include -#include "llhost.h" #include "llpacketbuffer.h" -#include "llthrottle.h" class LLPacketRing @@ -40,71 +47,26 @@ public: LLPacketRing(); ~LLPacketRing(); - // receive one packet: either buffered or from the socket - S32 receivePacket (S32 socket, char *datap); - - // send one packet - bool sendPacket(int h_socket, const char * send_buffer, S32 buf_size, LLHost host); - - // drains packets from socket and returns final mNumBufferedPackets - S32 drainSocket(S32 socket); + // Copy 'packet' onto the tail of the ring, growing the ring or + // overwriting the oldest entry when the ring is at capacity. + void pushPacket(const LLPacketBuffer& packet); - void dropPackets(U32); - void setDropPercentage (F32 percent_to_drop); - - inline LLHost getLastSender() const; - inline LLHost getLastReceivingInterface() const; - - S32 getActualInBytes() const { return mActualBytesIn; } - S32 getActualOutBytes() const { return mActualBytesOut; } - S32 getAndResetActualInBits() { S32 bits = mActualBytesIn * 8; mActualBytesIn = 0; return bits;} - S32 getAndResetActualOutBits() { S32 bits = mActualBytesOut * 8; mActualBytesOut = 0; return bits;} + // Copy the head (oldest) packet into 'packet'. + // Returns true if a packet was available, false if the ring was empty. + bool popPacket(LLPacketBuffer& packet); S32 getNumBufferedPackets() const { return (S32)(mNumBufferedPackets); } - S32 getNumBufferedBytes() const { return mNumBufferedBytes; } - S32 getNumDroppedPackets() const { return mNumDroppedPacketsTotal + mNumDroppedPackets; } - - F32 getBufferLoadRate() const; // from 0 to 4 (0 - empty, 1 - default size is full) - void dumpPacketRingStats(); -protected: - // returns 'true' if we should intentionally drop a packet - bool computeDrop(); - - // returns packet_size of received packet, zero or less if no packet found - S32 receiveOrDropPacket(S32 socket, char *datap, bool drop); - S32 receiveOrDropBufferedPacket(char *datap, bool drop); + S32 getNumBufferedBytes() const { return mNumBufferedBytes; } - // returns packet_size of packet buffered - S32 bufferInboundPacket(S32 socket); + // Ratio of buffered packets to DEFAULT_BUFFER_RING_SIZE (0 = empty, 1 = nominal full). + F32 getBufferLoadRate() const; - // returns 'true' if ring was expanded +private: + // Returns true if the ring was expanded, false if already at the ceiling. bool expandRing(); -protected: - std::vector mPacketRing; - S16 mHeadIndex { 0 }; + std::vector mRing; + S16 mHeadIndex { 0 }; S16 mNumBufferedPackets { 0 }; - S32 mNumDroppedPackets { 0 }; - S32 mNumDroppedPacketsTotal { 0 }; - S32 mNumBufferedBytes { 0 }; - - S32 mActualBytesIn { 0 }; - S32 mActualBytesOut { 0 }; - F32 mDropPercentage { 0.0f }; // % of inbound packets to drop - U32 mPacketsToDrop { 0 }; // drop next inbound n packets - - // These are the sender and receiving_interface for the last packet delivered by receivePacket() - LLHost mLastSender; - LLHost mLastReceivingIF; + S32 mNumBufferedBytes { 0 }; }; - - -inline LLHost LLPacketRing::getLastSender() const -{ - return mLastSender; -} - -inline LLHost LLPacketRing::getLastReceivingInterface() const -{ - return mLastReceivingIF; -} diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index e2937490ba..b11e4598e7 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -78,6 +78,8 @@ #include "lltransfertargetvfile.h" #include "llcorehttputil.h" #include "llpounceable.h" +#include "llproxy.h" +#include "llrand.h" // Constants //const char* MESSAGE_LOG_FILENAME = "message.log"; @@ -173,7 +175,7 @@ void LLMessageSystem::init() mTotalBytesIn = 0; mTotalBytesOut = 0; - mDroppedPackets = 0; // total dropped packets in + mLostPackets = 0; // total lost packets out mResentPackets = 0; // total resent packets out mFailedResendPackets = 0; // total resend failure packets out mOffCircuitPackets = 0; // total # of off-circuit packets rejected @@ -184,6 +186,13 @@ void LLMessageSystem::init() mIncomingCompressedSize = 0; mCurrentRecvPacketID = 0; + mActualBytesIn = 0; + mActualBytesOut = 0; + mDropPercentage = 0.0f; + mPacketsToDrop = 0; + mNumDroppedPackets = 0; + mNumDroppedPacketsTotal = 0; + mMessageFileVersionNumber = 0.f; mTimingCallback = NULL; @@ -508,18 +517,16 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) bool recv_reliable = false; bool recv_resent = false; - S32 acks = 0; + S32 num_acks = 0; S32 true_rcv_size = 0; U8* buffer = mTrueReceiveBuffer; - mTrueReceiveSize = mPacketRing.receivePacket(mSocket, (char *)mTrueReceiveBuffer); + mTrueReceiveSize = receivePacketOrDrop((char *)mTrueReceiveBuffer); // If you want to dump all received packets into SecondLife.log, uncomment this //dumpPacketToLog(); receive_size = mTrueReceiveSize; - mLastSender = mPacketRing.getLastSender(); - mLastReceivingIF = mPacketRing.getLastReceivingInterface(); if (receive_size < (S32) LL_MINIMUM_VALID_PACKET_SIZE) { @@ -535,24 +542,23 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) } else { - LLHost host; - LLCircuitData* cdp; - - // note if packet acks are appended. + // handle any packet ACKs (for outbound messages) found at tail of inbound message if(buffer[0] & LL_ACK_FLAG) { - acks += buffer[--receive_size]; + // Note: these ACKs may have already been handled if this was a buffered message + // but it doesn't hurt to handle them again. + num_acks += buffer[--receive_size]; true_rcv_size = receive_size; - if(receive_size >= ((S32)(acks * sizeof(TPACKETID) + LL_MINIMUM_VALID_PACKET_SIZE))) + if(receive_size >= ((S32)(num_acks * sizeof(TPACKETID) + LL_MINIMUM_VALID_PACKET_SIZE))) { - receive_size -= acks * sizeof(TPACKETID); + receive_size -= num_acks * sizeof(TPACKETID); } else { // mal-formed packet. ignore it and continue with // the next one LL_WARNS("Messaging") << "Malformed packet received. Packet size " - << receive_size << " with invalid no. of acks " << acks + << receive_size << " with invalid no. of acks " << num_acks << LL_ENDL; valid_packet = false; continue; @@ -562,20 +568,20 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) // process the message as normal mIncomingCompressedSize = zeroCodeExpand(&buffer, &receive_size); mCurrentRecvPacketID = ntohl(*((U32*)(&buffer[1]))); - host = getSender(); + LLHost host = getSender(); const bool resetPacketId = true; - cdp = findCircuit(host, resetPacketId); + LLCircuitData* cdp = findCircuit(host, resetPacketId); // At this point, cdp is now a pointer to the circuit that // this message came in on if it's valid, and NULL if the // circuit was bogus. - if(cdp && (acks > 0) && ((S32)(acks * sizeof(TPACKETID)) < (true_rcv_size))) + if(cdp && (num_acks > 0) && ((S32)(num_acks * sizeof(TPACKETID)) < (true_rcv_size))) { TPACKETID packet_id; U32 mem_id=0; - for(S32 i = 0; i < acks; ++i) + for(S32 i = 0; i < num_acks; ++i) { true_rcv_size -= sizeof(TPACKETID); memcpy(&mem_id, &mTrueReceiveBuffer[true_rcv_size], /* Flawfinder: ignore*/ @@ -630,7 +636,7 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) str << tbuf << "(unknown)" << (recv_reliable ? " reliable" : "") << " resent " - << ((acks > 0) ? "acks" : "") + << ((num_acks > 0) ? "acks" : "") << " DISCARD DUPLICATE"; LL_INFOS("Messaging") << str.str() << LL_ENDL; } @@ -680,7 +686,7 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) if ( valid_packet ) { - logValidMsg(cdp, host, recv_reliable, recv_resent, acks>0 ); + logValidMsg(cdp, host, recv_reliable, recv_resent, num_acks>0 ); valid_packet = mTemplateMessageReader->readMessage(buffer, host); } @@ -724,7 +730,7 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) // Check to see if we need to print debug info if ((mt_sec - mCircuitPrintTime) > mCircuitPrintFreq) { - mPacketRing.dumpPacketRingStats(); + dumpPacketRingStats(); dumpCircuitInfo(); mCircuitPrintTime = mt_sec; } @@ -749,6 +755,10 @@ S32 LLMessageSystem::getReceiveBytes() const } } +F32 LLMessageSystem::getBufferLoadRate() const +{ + return llmax(mHighPriorityInbound.getBufferLoadRate(), mLowPriorityInbound.getBufferLoadRate()); +} void LLMessageSystem::processAcks(LockMessageChecker&, F32 collect_time) { @@ -822,7 +832,264 @@ void LLMessageSystem::processAcks(LockMessageChecker&, F32 collect_time) S32 LLMessageSystem::drainUdpSocket() { - return mPacketRing.drainSocket(mSocket); + S32 packet_size = 1; + S32 num_loops = 0; + S32 old_num_buffered_packets = getNumBufferedPackets(); + while (packet_size > 0) + { + packet_size = bufferInboundPacket(); + ++num_loops; + } + S32 num_dropped_packets = (num_loops - 1 + old_num_buffered_packets) - getNumBufferedPackets(); + if (num_dropped_packets > 0) + { + mNumDroppedPackets += num_dropped_packets; + } + return getNumBufferedPackets(); +} + +bool LLMessageSystem::computeDrop() +{ + bool drop = (mDropPercentage > 0.0f && (ll_frand(100.f) < mDropPercentage)); + if (drop) + { + ++mPacketsToDrop; + } + if (mPacketsToDrop > 0) + { + --mPacketsToDrop; + drop = true; + } + return drop; +} + +bool LLMessageSystem::isHighPriorityMessage(const LLPacketBuffer& pkt) const +{ + S32 size = pkt.getSize(); + if (size < LL_PACKET_ID_SIZE + 1) + { + return false; + } + + // We want to prioritize crucial messages use to establish viewer <--> simulator connection, + // which are all low-frequency. A simple approximation is to just prioritize all non high- + // frequency messages. + // + // High frequency messages use a single byte for message_id whereas all low- and medium- + // frequency messages have 255 at the first byte of the message_id (which is after the + // LL_PACKET_ID_SIZE bytes of packet_id). + return *((const U8*)pkt.getData() + LL_PACKET_ID_SIZE) == 255; +} + +void LLMessageSystem::dropPackets(U32 num_to_drop) +{ + mPacketsToDrop += num_to_drop; +} + +void LLMessageSystem::setDropPercentage(F32 percent_to_drop) +{ + mDropPercentage = percent_to_drop; +} + +S32 LLMessageSystem::receivePacketOrDrop(char* datap) +{ + if (getNumBufferedPackets() > 0) + { + LLHost invalid_host; + LLPacketBuffer pkt(invalid_host, nullptr, 0); + if (!mHighPriorityInbound.popPacket(pkt)) + { + mLowPriorityInbound.popPacket(pkt); + } + + S32 packet_size = pkt.getSize(); + mLastSender = pkt.getHost(); + mLastReceivingIF = pkt.getReceivingInterface(); + + if (packet_size > 0) + { + memcpy(datap, pkt.getData(), packet_size); + } + return packet_size; + } + + // Read directly from the socket. + bool drop = computeDrop(); + S32 packet_size = 0; + if (LLProxy::isSOCKSProxyEnabled()) + { + char buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; /* Flawfinder ignore */ + packet_size = receive_packet(mSocket, buffer); + if (packet_size > 0) + { + mActualBytesIn += packet_size; + } + if (packet_size > SOCKS_HEADER_SIZE) + { + if (drop) + { + packet_size = 0; + } + else + { + // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) + packet_size -= SOCKS_HEADER_SIZE; + memcpy(datap, buffer + SOCKS_HEADER_SIZE, packet_size); + proxywrap_t* header = static_cast(static_cast(buffer)); + mLastSender.setAddress(header->addr); + mLastSender.setPort(ntohs(header->port)); + mLastReceivingIF = ::get_receiving_interface(); + } + } + else + { + packet_size = 0; + } + } + else + { + packet_size = receive_packet(mSocket, datap); + if (packet_size > 0) + { + mActualBytesIn += packet_size; + if (drop) + { + packet_size = 0; + } + else + { + mLastSender = ::get_sender(); + mLastReceivingIF = ::get_receiving_interface(); + } + } + } + return packet_size; +} + +S32 LLMessageSystem::bufferInboundPacket() +{ + LLHost invalid_host; + LLPacketBuffer pkt(invalid_host, nullptr, 0); + S32 packet_size = 0; + + if (LLProxy::isSOCKSProxyEnabled()) + { + char buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; /* Flawfinder ignore */ + packet_size = receive_packet(mSocket, buffer); + if (packet_size > 0) + { + mActualBytesIn += packet_size; + if (packet_size > SOCKS_HEADER_SIZE) + { + // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) + proxywrap_t* header = static_cast(static_cast(buffer)); + LLHost sender; + sender.setAddress(header->addr); + sender.setPort(ntohs(header->port)); + packet_size -= SOCKS_HEADER_SIZE; + pkt.init(buffer + SOCKS_HEADER_SIZE, packet_size, sender); + } + else + { + packet_size = 0; + } + } + } + else + { + pkt.init(mSocket); + packet_size = pkt.getSize(); + if (packet_size > 0) + { + mActualBytesIn += packet_size; + } + } + + if (packet_size >= (S32)LL_MINIMUM_VALID_PACKET_SIZE && !computeDrop()) + { + const char* data = pkt.getData(); + LLCircuitData* cdp = mCircuitInfo.findCircuit(pkt.getHost()); + TPACKETID recv_packet_id = ntohl(*((U32*)(&data[1]))); + + // Harvest piggybacked ACKs for outbound messages from the packet tail of this inbound message + if (cdp && (data[0] & LL_ACK_FLAG)) + { + U8 num_acks = (U8)data[packet_size - 1]; + S32 true_rcv_size = packet_size - 1; + if (true_rcv_size >= (S32)(num_acks * sizeof(TPACKETID) + LL_MINIMUM_VALID_PACKET_SIZE)) + { + TPACKETID ack_id; + U32 mem_id = 0; + for (S32 i = 0; i < num_acks; ++i) + { + true_rcv_size -= sizeof(TPACKETID); + memcpy(&mem_id, &data[true_rcv_size], sizeof(TPACKETID)); /* Flawfinder: ignore */ + ack_id = ntohl(mem_id); + cdp->ackReliablePacket(ack_id); + } + if (!cdp->getUnackedPacketCount()) + { + mCircuitInfo.mUnackedCircuitMap.erase(cdp->mHost); + } + } + } + + // ACK inbound reliable packet ASAP + if (cdp && (data[0] & LL_RELIABLE_FLAG)) + { + cdp->collectRAck(recv_packet_id); + } + + if (isHighPriorityMessage(pkt)) + { + mHighPriorityInbound.pushPacket(pkt); + } + else + { + mLowPriorityInbound.pushPacket(pkt); + } + } + + return packet_size; +} + +bool LLMessageSystem::sendPacketToSocket(const char* datap, S32 data_size, LLHost host) +{ + mActualBytesOut += data_size; + if (!LLProxy::isSOCKSProxyEnabled()) + { + return send_packet(mSocket, datap, data_size, host.getAddress(), host.getPort()); + } + + char headered_send_buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; + + proxywrap_t* socks_header = static_cast(static_cast(&headered_send_buffer)); + socks_header->rsv = 0; + socks_header->addr = host.getAddress(); + socks_header->port = htons(host.getPort()); + socks_header->atype = ADDRESS_IPV4; + socks_header->frag = 0; + + memcpy(headered_send_buffer + SOCKS_HEADER_SIZE, datap, data_size); + + return send_packet(mSocket, + headered_send_buffer, + data_size + SOCKS_HEADER_SIZE, + LLProxy::getInstance()->getUDPProxy().getAddress(), + LLProxy::getInstance()->getUDPProxy().getPort()); +} + +void LLMessageSystem::dumpPacketRingStats() +{ + mNumDroppedPacketsTotal += mNumDroppedPackets; + LL_INFOS("Messaging") << "buffered_packets=" << getNumBufferedPackets() + << "buffered_bytes=" << (mHighPriorityInbound.getNumBufferedBytes() + mLowPriorityInbound.getNumBufferedBytes()) + << "recently_dropped=" << mNumDroppedPackets + << "total_dropped=" << mNumDroppedPacketsTotal + << "dropped_percentage=" << mDropPercentage << "%" + << "bytes_IN=" << mActualBytesIn + << "bytes_OUT=" << mActualBytesOut << LL_ENDL; + mNumDroppedPackets = 0; } void LLMessageSystem::copyMessageReceivedToSend() @@ -1277,7 +1544,7 @@ S32 LLMessageSystem::sendMessage(const LLHost &host) } bool success; - success = mPacketRing.sendPacket(mSocket, (char *)buf_ptr, buffer_length, host); + success = sendPacketToSocket((char *)buf_ptr, buffer_length, host); if (!success) { @@ -2607,7 +2874,7 @@ void LLMessageSystem::summarizeLogs(std::ostream& str) str << buffer << std::endl << std::endl; buffer = llformat( "SendPacket failures: %20d", mSendPacketFailureCount); str << buffer << std::endl; - buffer = llformat( "Dropped packets: %20d", mDroppedPackets); + buffer = llformat( "Dropped packets: %20d", getTotalNumDroppedPackets()); str << buffer << std::endl; buffer = llformat( "Resent packets: %20d", mResentPackets); str << buffer << std::endl; @@ -3333,7 +3600,7 @@ void LLMessageSystem::establishBidirectionalTrust(const LLHost &host, S64 frame_ void LLMessageSystem::dumpPacketToLog() { - LL_WARNS("Messaging") << "Packet Dump from:" << mPacketRing.getLastSender() << LL_ENDL; + LL_WARNS("Messaging") << "Packet Dump from:" << mLastSender << LL_ENDL; LL_WARNS("Messaging") << "Packet Size:" << mTrueReceiveSize << LL_ENDL; char line_buffer[256]; /* Flawfinder: ignore */ S32 i; diff --git a/indra/llmessage/message.h b/indra/llmessage/message.h index 14cdc48a07..e81cfe16a9 100644 --- a/indra/llmessage/message.h +++ b/indra/llmessage/message.h @@ -119,6 +119,49 @@ public: // Repeat for number of messages in file // +// UDP Packet Buffer Layout +// +// Every UDP message sent or received by LLMessageSystem uses the following +// on-wire layout. Offsets are defined in EPacketHeaderLayout below. +// +// Byte(s) Name Description +// ------- ---- ----------- +// 0 Flags Bit-field (see flag constants below): +// 0x80 LL_ZERO_CODE_FLAG – body is zero-run-length encoded +// 0x40 LL_RELIABLE_FLAG – sender expects a packet ACK +// 0x20 LL_RESENT_FLAG – this is a retransmission +// 0x10 LL_ACK_FLAG – piggybacked ACKs are appended +// at the tail of this packet +// 1-4 Packet ID Sequence number, U32 in network (big-endian) byte order. +// 5 Offset Byte offset from PHL_NAME to the start of the message +// body (past the message-ID bytes). Zero for most messages. +// 6+ Message ID Variable-length message type identifier: +// High-frequency (1 byte, values 0x01–0xFE) +// Medium-frequency (2 bytes, 0xFF hh) +// Low-frequency (4 bytes, 0xFF 0xFF hh ll) +// ... Body Message block data, described by the message template. +// Present only when LL_ZERO_CODE_FLAG is clear; otherwise +// the body (everything after byte 5) is zero-coded (see +// below). The 6-byte header is never zero-coded. +// +// Optional ACK tail (present when LL_ACK_FLAG is set in the flags byte): +// +// ... ACK IDs N packet-sequence IDs being acknowledged, each a U32 in +// network byte order, packed contiguously immediately before +// the ACK count byte. Read in reverse: walk backwards from +// just before the count byte, 4 bytes at a time. +// last ACK Count U8 giving N, the number of appended ACK IDs (max 255). +// This is the very last byte of the UDP payload. +// +// Zero-coding (applied when LL_ZERO_CODE_FLAG is set): +// +// Runs of zero bytes in the body are replaced by a two-byte token: +// 0x00 N – represents (N + 1) zero bytes, for N in 1..254 +// 0x00 0x00 N – represents (256 + N) zero bytes (wrap/overflow case) +// A literal 0x00 byte that starts no run is encoded as 0x00 0x00 0x00. +// The six-byte packet header is excluded from zero-coding and is always +// transmitted as-is. + // Constants const S32 MAX_MESSAGE_INTERNAL_NAME_SIZE = 255; const S32 MAX_BUFFER_SIZE = NET_BUFFER_SIZE; @@ -291,8 +334,11 @@ class LLMessageSystem : public LLMessageSenderInterface bool mBlockUntrustedInterface; LLHost mUntrustedInterface; + protected: + LLPacketRing mHighPriorityInbound; + LLPacketRing mLowPriorityInbound; + public: - LLPacketRing mPacketRing; LLReliablePacketParams mReliablePacketParams; // Set this flag to true when you want *very* verbose logs. @@ -334,7 +380,7 @@ public: U32 mReliablePacketsIn; // total reliable packets in U32 mReliablePacketsOut; // total reliable packets out - U32 mDroppedPackets; // total dropped packets in + U32 mLostPackets; // total reliable outbound packets declared lost U32 mResentPackets; // total resent packets out U32 mFailedResendPackets; // total resend failure packets out U32 mOffCircuitPackets; // total # of off-circuit packets rejected @@ -420,6 +466,25 @@ public: // returns total number of buffered packets after the drain S32 drainUdpSocket(); + // Inbound Packet-loss simulation controls + void dropPackets(U32 num_to_drop); + void setDropPercentage(F32 percent_to_drop); + + // UDP byte-accounting + S32 getActualInBytes() const { return mActualBytesIn; } + S32 getActualOutBytes() const { return mActualBytesOut; } + S32 getAndResetActualInBits() { S32 bits = mActualBytesIn * 8; mActualBytesIn = 0; return bits; } + S32 getAndResetActualOutBits() { S32 bits = mActualBytesOut * 8; mActualBytesOut = 0; return bits; } + + // Get number of "dropped" inbound packets + S32 getTotalNumDroppedPackets() const { return mNumDroppedPacketsTotal + mNumDroppedPackets; } + + S32 getNumBufferedPackets() const { return mHighPriorityInbound.getNumBufferedPackets() + mLowPriorityInbound.getNumBufferedPackets(); } + void dumpPacketRingStats(); + + // Send datap to host via mSocket (with SOCKS proxy support if enabled). + bool sendPacketToSocket(const char* datap, S32 data_size, LLHost host); + bool isMessageFast(const char *msg); bool isMessage(const char *msg) { @@ -754,7 +819,7 @@ public: S32 getReceiveBytes() const; S32 getUnackedListSize() const { return mUnackedListSize; } - F32 getBufferLoadRate() const { return mPacketRing.getBufferLoadRate(); } + F32 getBufferLoadRate() const; //const char* getCurrentSMessageName() const { return mCurrentSMessageName; } //const char* getCurrentSBlockName() const { return mCurrentSBlockName; } @@ -898,6 +963,32 @@ private: S32 mIncomingCompressedSize; // original size of compressed msg (0 if uncomp.) TPACKETID mCurrentRecvPacketID; // packet ID of current receive packet (for reporting) + // Socket I/O helpers + + // Receive one packet: pop from ring if buffered, else read from mSocket. + // Sets mLastSender and mLastReceivingIF. + // Returns packet_size, or 0 if no packet or packet was dropped. + S32 receivePacketOrDrop(char* datap); + + // Read one raw packet from mSocket into inbound message queues + // Returns packet_size (0 if no packet was available). + S32 bufferInboundPacket(); + + // Returns true if the next inbound packet should be intentionally dropped. + bool computeDrop(); + + // Returns true if pkt carries a high-priority message and should be queued + // in mHighPriorityInbound. + bool isHighPriorityMessage(const LLPacketBuffer& pkt) const; + + // Packet-loss simulation and byte-accounting state + S32 mActualBytesIn; + S32 mActualBytesOut; + F32 mDropPercentage; // % of inbound packets to drop + U32 mPacketsToDrop; // drop next N inbound packets + S32 mNumDroppedPackets; // inbound + S32 mNumDroppedPacketsTotal;// inbound + LLMessageBuilder* mMessageBuilder; LLTemplateMessageBuilder* mTemplateMessageBuilder; LLSDMessageBuilder* mLLSDMessageBuilder; diff --git a/indra/llmessage/net.cpp b/indra/llmessage/net.cpp index 2be5a9e5b6..0df38d040a 100644 --- a/indra/llmessage/net.cpp +++ b/indra/llmessage/net.cpp @@ -326,7 +326,7 @@ S32 receive_packet(int hSocket, char * receiveBuffer) return 0; if (WSAECONNRESET == WSAGetLastError()) return 0; - LL_INFOS() << "receivePacket() failed, Error: " << WSAGetLastError() << LL_ENDL; + LL_INFOS() << "receive_packet() failed, Error: " << WSAGetLastError() << LL_ENDL; } return nRet; diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 79752d2315..7f06f991b0 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5894,7 +5894,6 @@ void LLAppViewer::idleNetwork() pingMainloopTimeout("idleNetwork"); gObjectList.mNumNewObjects = 0; - S32 total_decoded = 0; static LLCachedControl speed_test(gSavedSettings, "SpeedTest", false); if (!speed_test()) @@ -5902,64 +5901,56 @@ void LLAppViewer::idleNetwork() LL_PROFILE_ZONE_NAMED_CATEGORY_NETWORK("idle network"); //LL_RECORD_BLOCK_TIME(FTM_IDLE_NETWORK); // decode LLTimer check_message_timer; - // Read all available packets from network const S64 frame_count = gFrameCount; // U32->S64 - F32 total_time = 0.0f; + S32 total_decoded = 0; + // Process packets from network + LockMessageChecker lmc(gMessageSystem); + while (lmc.checkAllMessages(frame_count, gServicePump)) { - bool needs_drain = false; - LockMessageChecker lmc(gMessageSystem); - while (lmc.checkAllMessages(frame_count, gServicePump)) - { - if (gDoDisconnect) - { - // We're disconnecting, don't process any more messages from the server - // We're usually disconnecting due to either network corruption or a - // server going down, so this is OK. - break; - } - - total_decoded++; + ++total_decoded; - if (total_decoded > MESSAGE_MAX_PER_FRAME) + // Time-box processing of network packets to prevent framerate catastrophe + if (check_message_timer.getElapsedTimeF32() >= CheckMessagesMaxTime) + { + // Drain the socket buffer so we know how many messages remain to process + S32 num_buffered_packets = gMessageSystem->drainUdpSocket(); + if (num_buffered_packets > total_decoded) { - needs_drain = true; - break; + // Grow CheckMessagesMaxTime until we process more packets each frame than arrive. + // This might spiral out of control on very slow computers on fast networks when + // the bandwidth settings are too high. There is a mechanism for providing backpressure + // to network bandwidth but it may be inadequate for the task + // (see LLViewerThrottle::updateDynamicThrottle() for more details). + CheckMessagesMaxTime *= 1.035f; // 3.5% ~= 2x in 20 frames, ~8x in 60 frames } - - // Prevent slow packets from completely destroying the frame rate. - // This usually happens due to clumps of avatars taking huge amount - // of network processing time (which needs to be fixed, but this is - // a good limit anyway). - total_time = check_message_timer.getElapsedTimeF32(); - if (total_time >= CheckMessagesMaxTime) + else if (num_buffered_packets == 0) { - needs_drain = true; - break; + // Reset CheckMessagesMaxTime to default value + CheckMessagesMaxTime = CHECK_MESSAGES_DEFAULT_MAX_TIME; } + break; } - if (needs_drain || gMessageSystem->mPacketRing.getNumBufferedPackets() > 0) + + if (total_decoded > MESSAGE_MAX_PER_FRAME) { - // Rather than allow packets to silently backup on the socket - // we drain them into our own buffer so we know how many exist. - S32 num_buffered_packets = gMessageSystem->drainUdpSocket(); - if (num_buffered_packets > 0) - { - // Increase CheckMessagesMaxTime so that we will eventually catch up - CheckMessagesMaxTime *= 1.035f; // 3.5% ~= 2x in 20 frames, ~8x in 60 frames - } + // MESSAGE_MAX_PER_FRAME is very high (400) + // We expect to run out of time before reaching here, but just in case... + gMessageSystem->drainUdpSocket(); + break; } - else + + if (gDoDisconnect) { - // Reset CheckMessagesMaxTime to default value - CheckMessagesMaxTime = CHECK_MESSAGES_DEFAULT_MAX_TIME; + // We're disconnecting so no need to process packets. + break; } + } - // Handle per-frame message system processing. + // Handle per-frame message system processing. - static LLCachedControl ack_collection_time(gSavedSettings, "AckCollectTime", 0.1f); - lmc.processAcks(ack_collection_time()); - } + static LLCachedControl ack_collection_time(gSavedSettings, "AckCollectTime", 0.1f); + lmc.processAcks(ack_collection_time()); } add(LLStatViewer::NUM_NEW_OBJECTS, gObjectList.mNumNewObjects); diff --git a/indra/newview/llstartup.cpp b/indra/newview/llstartup.cpp index 6a85ca069a..cd4c9d40b1 100644 --- a/indra/newview/llstartup.cpp +++ b/indra/newview/llstartup.cpp @@ -401,7 +401,7 @@ void do_startup_frame() break; } } - if (needs_drain || gMessageSystem->mPacketRing.getNumBufferedPackets() > 0) + if (needs_drain || gMessageSystem->getNumBufferedPackets() > 0) { gMessageSystem->drainUdpSocket(); } @@ -785,7 +785,7 @@ bool idle_startup() F32 dropPercent = gSavedSettings.getF32("PacketDropPercentage"); - msg->mPacketRing.setDropPercentage(dropPercent); + msg->setDropPercentage(dropPercent); } LL_INFOS("AppInit") << "Message System Initialized." << LL_ENDL; diff --git a/indra/newview/llviewermenu.cpp b/indra/newview/llviewermenu.cpp index ff111dd1bc..7f36244685 100644 --- a/indra/newview/llviewermenu.cpp +++ b/indra/newview/llviewermenu.cpp @@ -2298,7 +2298,7 @@ class LLAdvancedDropPacket : public view_listener_t { bool handleEvent(const LLSD& userdata) { - gMessageSystem->mPacketRing.dropPackets(1); + gMessageSystem->dropPackets(1); return true; } }; diff --git a/indra/newview/llviewerstats.cpp b/indra/newview/llviewerstats.cpp index 6490fe76a2..3b2796572b 100644 --- a/indra/newview/llviewerstats.cpp +++ b/indra/newview/llviewerstats.cpp @@ -791,7 +791,7 @@ void send_viewer_stats(bool include_preferences) LLSD &fail = body["stats"]["failures"]; fail["send_packet"] = (S32) gMessageSystem->mSendPacketFailureCount; - fail["dropped"] = (S32) gMessageSystem->mDroppedPackets; + fail["dropped"] = (S32) gMessageSystem->getTotalNumDroppedPackets(); fail["resent"] = (S32) gMessageSystem->mResentPackets; fail["failed_resends"] = (S32) gMessageSystem->mFailedResendPackets; fail["off_circuit"] = (S32) gMessageSystem->mOffCircuitPackets; diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index 3ccfbea6e2..90a60c7a94 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -45,8 +45,8 @@ using namespace LLOldEvents; const F32 MAX_FRACTIONAL = 1.5f; const F32 MIN_FRACTIONAL = 0.2f; -const F32 MIN_BANDWIDTH = 50.f; -const F32 MAX_BANDWIDTH = 6000.f; +const F32 MIN_BANDWIDTH = 50.f; // Kbps +const F32 MAX_BANDWIDTH = 6000.f; // Kbps const F32 STEP_FRACTIONAL = 0.1f; const F32 HIGH_BUFFER_LOAD_TRESHOLD = 1.f; const F32 LOW_BUFFER_LOAD_TRESHOLD = 0.8f; @@ -244,6 +244,8 @@ void LLViewerThrottle::sendToSim() const F32 LLViewerThrottle::getMaxBandwidthKbps() { + // Why are thse different than the constants with the same names higher up? + // Anybody know? -- Leviathan constexpr F32 MIN_BANDWIDTH = 100.0f; // 100 Kbps constexpr F32 MAX_BANDWIDTH = 10000.0f; // 10 Mbps @@ -309,6 +311,21 @@ void LLViewerThrottle::resetDynamicThrottle() void LLViewerThrottle::updateDynamicThrottle() { + // User configurable bandwidth settings are merely vague aspirations. Translating those + // to what should be sent to the servers is complicated. Servers will tend to spike data + // transmission upon arrival, packets will arrive faster than we can process them, and this + // can overflow the buffer, which causes packet loss. + // + // In an attempt to avoid catastrophe we periodically measure buffer load and packet loss + // and then transmit a modified desired bandwidth to the server. Unfortunately, this system + // does not work well for spikey data bursts because: + // (1) the response takes too long (up to 5 seconds) to kick in + // (2) once it starts it tends to ramp up too slowly + // (3) it doesn't know which region is providing the flood; it just assumes it is + // all from the main region + // + // TODO: fix those ^^^ problems + if (mUpdateTimer.getElapsedTimeF32() < DYNAMIC_UPDATE_DURATION) { return; @@ -347,5 +364,7 @@ void LLViewerThrottle::updateDynamicThrottle() LL_INFOS() << "Easing network throttle to " << mCurrentBandwidth << LL_ENDL; } + // Now that we've used mBufferLoadRate we reset it to zero because otherwise it only + // increases (see clamping behavior in setBufferLoadRate()). mBufferLoadRate = 0; } diff --git a/indra/newview/llviewerthrottle.h b/indra/newview/llviewerthrottle.h index ef898a97d7..7ff0a7d1f0 100644 --- a/indra/newview/llviewerthrottle.h +++ b/indra/newview/llviewerthrottle.h @@ -77,8 +77,8 @@ public: static const std::string sNames[TC_EOF]; protected: - F32 mMaxBandwidth; - F32 mCurrentBandwidth; + F32 mMaxBandwidth; // bps + F32 mCurrentBandwidth; // bps F32 mBufferLoadRate = 0; LLViewerThrottleGroup mCurrent; diff --git a/indra/newview/llworld.cpp b/indra/newview/llworld.cpp index d02694de7d..cad36828d1 100644 --- a/indra/newview/llworld.cpp +++ b/indra/newview/llworld.cpp @@ -794,10 +794,10 @@ void LLWorld::updateNetStats() S32 packets_in = gMessageSystem->mPacketsIn - mLastPacketsIn; S32 packets_out = gMessageSystem->mPacketsOut - mLastPacketsOut; - S32 packets_lost = gMessageSystem->mDroppedPackets - mLastPacketsLost; + S32 packets_lost = gMessageSystem->mLostPackets - mLastPacketsLost; - F64Bits actual_in_bits(gMessageSystem->mPacketRing.getAndResetActualInBits()); - F64Bits actual_out_bits(gMessageSystem->mPacketRing.getAndResetActualOutBits()); + F64Bits actual_in_bits(gMessageSystem->getAndResetActualInBits()); + F64Bits actual_out_bits(gMessageSystem->getAndResetActualOutBits()); add(LLStatViewer::MESSAGE_SYSTEM_DATA_IN, actual_in_bits); add(LLStatViewer::MESSAGE_SYSTEM_DATA_OUT, actual_out_bits); @@ -815,7 +815,7 @@ void LLWorld::updateNetStats() mLastPacketsIn = gMessageSystem->mPacketsIn; mLastPacketsOut = gMessageSystem->mPacketsOut; - mLastPacketsLost = gMessageSystem->mDroppedPackets; + mLastPacketsLost = gMessageSystem->mLostPackets; } @@ -838,7 +838,7 @@ void LLWorld::printPacketsLost() << " packets lost: " << cdp->getPacketsLost() << LL_ENDL; } } - LL_INFOS() << "Packets dropped by Packet Ring: " << gMessageSystem->mPacketRing.getNumDroppedPackets() << LL_ENDL; + LL_INFOS() << "Packets dropped by Packet Ring: " << gMessageSystem->getTotalNumDroppedPackets() << LL_ENDL; } void LLWorld::processCoarseUpdate(LLMessageSystem* msg, void** user_data) -- cgit v1.3 From 5986568a23836f9ce448a1912406eaedd3794175 Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev <117672381+akleshchev@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:21:50 +0300 Subject: #6014 More tracy coverage --- indra/llappearance/llpolymesh.cpp | 1 + indra/llcommon/llcoros.cpp | 1 + indra/llcommon/llwatchdog.cpp | 1 + indra/llfilesystem/lllfsthread.cpp | 1 + indra/llmessage/message.cpp | 1 + indra/llphysicsextensionsos/llphysicsextensions.cpp | 1 + indra/llplugin/llpluginclassmedia.cpp | 1 + indra/llxml/llcontrol.cpp | 2 ++ 8 files changed, 9 insertions(+) (limited to 'indra/llmessage/message.cpp') diff --git a/indra/llappearance/llpolymesh.cpp b/indra/llappearance/llpolymesh.cpp index d5323e0b84..1162efb868 100644 --- a/indra/llappearance/llpolymesh.cpp +++ b/indra/llappearance/llpolymesh.cpp @@ -862,6 +862,7 @@ LLPolyMesh *LLPolyMesh::getMesh(const std::string &name, LLPolyMesh* reference_m //----------------------------------------------------------------------------- void LLPolyMesh::freeAllMeshes() { + LL_PROFILE_ZONE_SCOPED; // delete each item in the global lists for_each(sGlobalSharedMeshList.begin(), sGlobalSharedMeshList.end(), DeletePairedPointer()); sGlobalSharedMeshList.clear(); diff --git a/indra/llcommon/llcoros.cpp b/indra/llcommon/llcoros.cpp index 9e95d9c85f..ae7c50e9a3 100644 --- a/indra/llcommon/llcoros.cpp +++ b/indra/llcommon/llcoros.cpp @@ -252,6 +252,7 @@ void LLCoros::setStackSize(S32 stacksize) void LLCoros::printActiveCoroutines(const std::string& when) { + LL_PROFILE_ZONE_SCOPED; LL_INFOS("LLCoros") << "Number of active coroutines " << when << ": " << CoroData::instanceCount() << LL_ENDL; if (CoroData::instanceCount() > 0) diff --git a/indra/llcommon/llwatchdog.cpp b/indra/llcommon/llwatchdog.cpp index 056a0cc940..b17b68dbe7 100644 --- a/indra/llcommon/llwatchdog.cpp +++ b/indra/llcommon/llwatchdog.cpp @@ -236,6 +236,7 @@ void LLWatchdog::shutdown() void LLWatchdog::cleanup() { + LL_PROFILE_ZONE_SCOPED; if (mTimer) { mTimer->stop(); diff --git a/indra/llfilesystem/lllfsthread.cpp b/indra/llfilesystem/lllfsthread.cpp index 6a882f64a8..69f7085290 100644 --- a/indra/llfilesystem/lllfsthread.cpp +++ b/indra/llfilesystem/lllfsthread.cpp @@ -51,6 +51,7 @@ S32 LLLFSThread::updateClass(U32 ms_elapsed) //static void LLLFSThread::cleanupClass() { + LL_PROFILE_ZONE_SCOPED; llassert(sLocal != NULL); sLocal->setQuitting(); while (sLocal->getPending()) diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index b11e4598e7..52180b20f1 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -2906,6 +2906,7 @@ void LLMessageSystem::summarizeLogs(std::ostream& str) void end_messaging_system(bool print_summary) { + LL_PROFILE_ZONE_SCOPED; gTransferManager.cleanup(); LLTransferTargetVFile::updateQueue(true); // shutdown LLTransferTargetVFile if (gMessageSystem) diff --git a/indra/llphysicsextensionsos/llphysicsextensions.cpp b/indra/llphysicsextensionsos/llphysicsextensions.cpp index 3bb8ffbf1a..295ab41a17 100644 --- a/indra/llphysicsextensionsos/llphysicsextensions.cpp +++ b/indra/llphysicsextensionsos/llphysicsextensions.cpp @@ -72,6 +72,7 @@ //============================================================================= /*static */bool LLPhysicsExtensions::quitSystem() { + LL_PROFILE_ZONE_SCOPED; return LLPhysicsExtensionsImpl::quitSystem(); } //============================================================================= diff --git a/indra/llplugin/llpluginclassmedia.cpp b/indra/llplugin/llpluginclassmedia.cpp index 94c29d9457..9a1f408b34 100644 --- a/indra/llplugin/llpluginclassmedia.cpp +++ b/indra/llplugin/llpluginclassmedia.cpp @@ -92,6 +92,7 @@ bool LLPluginClassMedia::init(const std::string &launcher_filename, const std::s void LLPluginClassMedia::reset() { + LL_PROFILE_ZONE_SCOPED; if(mPlugin) { mPlugin->requestShutdown(); diff --git a/indra/llxml/llcontrol.cpp b/indra/llxml/llcontrol.cpp index 5a37167f18..be53a93a2d 100644 --- a/indra/llxml/llcontrol.cpp +++ b/indra/llxml/llcontrol.cpp @@ -407,6 +407,7 @@ static bool compareRoutine(settings_pair_t lhs, settings_pair_t rhs) void LLControlGroup::cleanup() { + LL_PROFILE_ZONE_SCOPED; if(mSettingsProfile && getCount.size() != 0) { std::string file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, SETTINGS_PROFILE); @@ -982,6 +983,7 @@ U32 LLControlGroup::loadFromFileLegacy(const std::string& filename, bool require U32 LLControlGroup::saveToFile(const std::string& filename, bool nondefault_only) { + LL_PROFILE_ZONE_SCOPED; LLSD settings; int num_saved = 0; for (ctrl_name_table_t::iterator iter = mNameTable.begin(); -- cgit v1.3 From b906284977b120c4c554fd08b78fac62759b160a Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 22 Jul 2026 09:01:47 -0700 Subject: Defer processing AvatarAppearance until after first ObjectUpdate (#6017) * make AvatarAppearance a low-priority message * remember agent_id of unexpected AvatarAppearance message for later --- indra/llmessage/llpacketbuffer.h | 7 ++++ indra/llmessage/message.cpp | 80 +++++++++++++++++++++++++++++++-------- indra/llmessage/message.h | 6 ++- indra/llprimitive/llprimitive.cpp | 8 +++- indra/llprimitive/llprimitive.h | 3 ++ indra/newview/llagent.cpp | 1 + indra/newview/llviewermessage.cpp | 9 +++-- indra/newview/llvoavatar.cpp | 18 ++++++++- indra/newview/llvoavatar.h | 21 +++++++++- 9 files changed, 128 insertions(+), 25 deletions(-) (limited to 'indra/llmessage/message.cpp') diff --git a/indra/llmessage/llpacketbuffer.h b/indra/llmessage/llpacketbuffer.h index b0505e7afd..a08e6c22e5 100644 --- a/indra/llmessage/llpacketbuffer.h +++ b/indra/llmessage/llpacketbuffer.h @@ -49,11 +49,18 @@ public: void init(S32 hSocket); void init(const char* buffer, S32 data_size, const LLHost& host); + // Whether LLCircuitData::checkPacketInID() has already been run for this + // packet (done at socket-read time, before this packet was sorted into + // the high/low priority inbound queue). + bool getPacketIDChecked() const { return mPacketIDChecked; } + void setPacketIDChecked(bool checked) { mPacketIDChecked = checked; } + protected: char mData[NET_BUFFER_SIZE]; // packet data /* Flawfinder : ignore */ S32 mSize; // size of buffer in bytes LLHost mHost; // source/dest IP and port LLHost mReceivingIF; // source/dest IP and port + bool mPacketIDChecked = false; }; #endif diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index 52180b20f1..8d86856706 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -519,10 +519,11 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) bool recv_resent = false; S32 num_acks = 0; S32 true_rcv_size = 0; + bool recv_packet_id_checked = false; U8* buffer = mTrueReceiveBuffer; - mTrueReceiveSize = receivePacketOrDrop((char *)mTrueReceiveBuffer); + mTrueReceiveSize = receivePacketOrDrop((char *)mTrueReceiveBuffer, recv_packet_id_checked); // If you want to dump all received packets into SecondLife.log, uncomment this //dumpPacketToLog(); @@ -686,7 +687,7 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) if ( valid_packet ) { - logValidMsg(cdp, host, recv_reliable, recv_resent, num_acks>0 ); + logValidMsg(cdp, host, recv_reliable, recv_resent, num_acks>0, recv_packet_id_checked ); valid_packet = mTemplateMessageReader->readMessage(buffer, host); } @@ -866,19 +867,37 @@ bool LLMessageSystem::computeDrop() bool LLMessageSystem::isHighPriorityMessage(const LLPacketBuffer& pkt) const { S32 size = pkt.getSize(); - if (size < LL_PACKET_ID_SIZE + 1) + + // avoid stepping out of bounds + const S32 MIN_VALID_PACKET_SIZE = LL_PACKET_ID_SIZE + 4; + if (size < MIN_VALID_PACKET_SIZE) { return false; } - // We want to prioritize crucial messages use to establish viewer <--> simulator connection, + // We want to prioritize crucial messages used to establish viewer <--> simulator connection, // which are all low-frequency. A simple approximation is to just prioritize all non high- // frequency messages. // - // High frequency messages use a single byte for message_id whereas all low- and medium- - // frequency messages have 255 at the first byte of the message_id (which is after the - // LL_PACKET_ID_SIZE bytes of packet_id). - return *((const U8*)pkt.getData() + LL_PACKET_ID_SIZE) == 255; + // High frequency messages use a single byte for message_id whereas medium- and low- + // frequency messages have 255 at the first byte (which is after the LL_PACKET_ID_SIZE + // bytes of packet_id). + const U8* header = (const U8*)pkt.getData() + LL_PACKET_ID_SIZE; + if (header[0] != 255) + { + // high-frequency message + return false; + } + + // BUG: The low-frequency AvatarAppearance message will be ignored if its agent is unknown + // and the agent is created upon receipt of its first high-frequency ObjectUpdate message. + // This race condition will be exacerbated by our default prioritization strategy. + // + // WORKAROUND: All middle- and low-frequency messages are high-priority except AvatarAppearance + // + // AvatarAppearance is "Low 158" which means it is stored in four bytes: 0xff 0xff 0x00 0x9E + // the last two bytes represent 158 in a BigEndian U16 + return header[1] != 255 && header[2] != 0 && header[3] != 158; } void LLMessageSystem::dropPackets(U32 num_to_drop) @@ -891,8 +910,10 @@ void LLMessageSystem::setDropPercentage(F32 percent_to_drop) mDropPercentage = percent_to_drop; } -S32 LLMessageSystem::receivePacketOrDrop(char* datap) +S32 LLMessageSystem::receivePacketOrDrop(char* datap, bool& packet_id_already_checked) { + packet_id_already_checked = false; + if (getNumBufferedPackets() > 0) { LLHost invalid_host; @@ -905,6 +926,7 @@ S32 LLMessageSystem::receivePacketOrDrop(char* datap) S32 packet_size = pkt.getSize(); mLastSender = pkt.getHost(); mLastReceivingIF = pkt.getReceivingInterface(); + packet_id_already_checked = pkt.getPacketIDChecked(); if (packet_size > 0) { @@ -913,7 +935,8 @@ S32 LLMessageSystem::receivePacketOrDrop(char* datap) return packet_size; } - // Read directly from the socket. + // Read directly from the socket. checkPacketInID() has not run yet for + // this packet. bool drop = computeDrop(); S32 packet_size = 0; if (LLProxy::isSOCKSProxyEnabled()) @@ -1034,10 +1057,24 @@ S32 LLMessageSystem::bufferInboundPacket() } } - // ACK inbound reliable packet ASAP - if (cdp && (data[0] & LL_RELIABLE_FLAG)) + if (cdp) { - cdp->collectRAck(recv_packet_id); + // ACK inbound reliable packet ASAP + if ((data[0] & LL_RELIABLE_FLAG)) + { + cdp->collectRAck(recv_packet_id); + } + + // Check packet sequencing here, in true socket-arrival order, before + // this packet is sorted into the high/low priority inbound queue. + // Skip genuine duplicate resends, same as checkMessages()/logValidMsg() + // would do further downstream. + bool recv_resent = (data[0] & LL_RESENT_FLAG) != 0; + if (!recv_resent || !cdp->isDuplicateResend(recv_packet_id)) + { + cdp->checkPacketInID(recv_packet_id, recv_resent); + pkt.setPacketIDChecked(true); + } } if (isHighPriorityMessage(pkt)) @@ -1668,7 +1705,13 @@ void LLMessageSystem::logTrustedMsgFromUntrustedCircuit( const LLHost& host ) } } -void LLMessageSystem::logValidMsg(LLCircuitData *cdp, const LLHost& host, bool recv_reliable, bool recv_resent, bool recv_acks ) +void LLMessageSystem::logValidMsg( + LLCircuitData *cdp, + const LLHost& host, + bool recv_reliable, + bool recv_resent, + bool recv_acks, + bool skip_packet_id_check ) { if (mNumMessageCounts >= MAX_MESSAGE_COUNT_NUM) { @@ -1685,8 +1728,13 @@ void LLMessageSystem::logValidMsg(LLCircuitData *cdp, const LLHost& host, bool r if (cdp) { - // update circuit packet ID tracking (missing/out of order packets) - cdp->checkPacketInID( mCurrentRecvPacketID, recv_resent ); + if (!skip_packet_id_check) + { + // update circuit packet ID tracking (missing/out of order packets) + // Already done in bufferInboundPacket(), in true socket-arrival + // order, if this packet came off the high/low priority queues. + cdp->checkPacketInID( mCurrentRecvPacketID, recv_resent ); + } cdp->addBytesIn( (S32Bytes)mTrueReceiveSize ); } diff --git a/indra/llmessage/message.h b/indra/llmessage/message.h index e81cfe16a9..e23a67c04a 100644 --- a/indra/llmessage/message.h +++ b/indra/llmessage/message.h @@ -910,7 +910,7 @@ private: void logMsgFromInvalidCircuit( const LLHost& sender, bool recv_reliable ); void logTrustedMsgFromUntrustedCircuit( const LLHost& sender ); - void logValidMsg(LLCircuitData *cdp, const LLHost& sender, bool recv_reliable, bool recv_resent, bool recv_acks ); + void logValidMsg(LLCircuitData *cdp, const LLHost& sender, bool recv_reliable, bool recv_resent, bool recv_acks, bool skip_packet_id_check ); class LLMessageCountInfo { @@ -967,8 +967,10 @@ private: // Receive one packet: pop from ring if buffered, else read from mSocket. // Sets mLastSender and mLastReceivingIF. + // Sets packet_id_already_checked to whether checkPacketInID() was already + // run for this packet back when it was buffered (see bufferInboundPacket()). // Returns packet_size, or 0 if no packet or packet was dropped. - S32 receivePacketOrDrop(char* datap); + S32 receivePacketOrDrop(char* datap, bool& packet_id_already_checked); // Read one raw packet from mSocket into inbound message queues // Returns packet_size (0 if no packet was available). diff --git a/indra/llprimitive/llprimitive.cpp b/indra/llprimitive/llprimitive.cpp index 7a638dd625..fd755e35dd 100644 --- a/indra/llprimitive/llprimitive.cpp +++ b/indra/llprimitive/llprimitive.cpp @@ -1369,6 +1369,12 @@ bool LLPrimitive::packTEMessage(LLDataPacker &dp) const } S32 LLPrimitive::parseTEMessage(LLMessageSystem* mesgsys, char const* block_name, const S32 block_num, LLTEContents& tec) +{ + return parseTEMessage(mesgsys, block_name, block_num, tec, getNumTEs()); +} + +// static +S32 LLPrimitive::parseTEMessage(LLMessageSystem* mesgsys, char const* block_name, const S32 block_num, LLTEContents& tec, U8 face_count) { S32 retval = 0; // temp buffer for material ID processing @@ -1403,7 +1409,7 @@ S32 LLPrimitive::parseTEMessage(LLMessageSystem* mesgsys, char const* block_name tec.packed_buffer[tec.size] = 0x00; ++tec.size; - tec.face_count = llmin((U32)getNumTEs(),(U32)LLTEContents::MAX_TES); + tec.face_count = llmin((U32)face_count,(U32)LLTEContents::MAX_TES); U8 *cur_ptr = tec.packed_buffer; LL_DEBUGS("TEXTUREENTRY") << "Texture Entry with buffere sized: " << tec.size << LL_ENDL; diff --git a/indra/llprimitive/llprimitive.h b/indra/llprimitive/llprimitive.h index c3e3e19ee9..21bf52610e 100644 --- a/indra/llprimitive/llprimitive.h +++ b/indra/llprimitive/llprimitive.h @@ -502,6 +502,9 @@ public: S32 unpackTEMessage(LLMessageSystem* mesgsys, char const* block_name, const S32 block_num); // Variable num of blocks S32 unpackTEMessage(LLDataPacker &dp); S32 parseTEMessage(LLMessageSystem* mesgsys, char const* block_name, const S32 block_num, LLTEContents& tec); + // Same as above, but usable before an instance exists to derive the face count from + // (e.g. decoding a message for an object that hasn't been created yet). + static S32 parseTEMessage(LLMessageSystem* mesgsys, char const* block_name, const S32 block_num, LLTEContents& tec, U8 face_count); S32 applyParsedTEMessage(LLTEContents& tec); #ifdef CHECK_FOR_FINITE diff --git a/indra/newview/llagent.cpp b/indra/newview/llagent.cpp index 35d60e6595..2f0ad15d18 100644 --- a/indra/newview/llagent.cpp +++ b/indra/newview/llagent.cpp @@ -4151,6 +4151,7 @@ void LLAgent::handleTeleportFinished() } clearTeleportRequest(); mTeleportCanceled.reset(); + LLVOAvatar::resetEarlyAppearanceList(); if (mIsMaturityRatingChangingDuringTeleport) { // notify user that the maturity preference has been changed diff --git a/indra/newview/llviewermessage.cpp b/indra/newview/llviewermessage.cpp index 1f1ff30b8f..e435ce08e3 100644 --- a/indra/newview/llviewermessage.cpp +++ b/indra/newview/llviewermessage.cpp @@ -4181,11 +4181,12 @@ void process_avatar_appearance(LLMessageSystem *mesgsys, void **user_data) if (avatarp) { avatarp->processAvatarAppearance( mesgsys ); + return; } - else - { - LL_WARNS("Messaging") << "avatar_appearance sent for unknown avatar " << uuid << LL_ENDL; - } + // The avatar object doesn't exist yet. + // We will re-request its appearance data after it is created. + LLVOAvatar::registerEarlyAppearance(uuid); + LL_WARNS("Messaging") << "AvatarAppearance received for avatar " << uuid << " before object created" << LL_ENDL; } void process_camera_constraint(LLMessageSystem *mesgsys, void **user_data) diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp index cd71a2c660..89ee5ede2a 100644 --- a/indra/newview/llvoavatar.cpp +++ b/indra/newview/llvoavatar.cpp @@ -615,6 +615,8 @@ const LLUUID LLVOAvatar::sStepSounds[LL_MCODE_END] = SND_RUBBER_RUBBER }; +uuid_list_t LLVOAvatar::sEarlyAppearanceList; + S32 LLVOAvatar::sRenderName = RENDER_NAME_ALWAYS; S32 LLVOAvatar::sRenderGroupTitles = RENDER_GROUP_TITLE_ALWAYS; S32 LLVOAvatar::sNumVisibleChatBubbles = 0; @@ -779,6 +781,20 @@ LLVOAvatar::LLVOAvatar(const LLUUID& id, mVisuallyMuteSetting = LLVOAvatar::VisualMuteSettings(LLRenderMuteList::getInstance()->getSavedVisualMuteSetting(getID())); sInstances.push_back(this); + + uuid_list_t::iterator it = sEarlyAppearanceList.find(id); + if (it != sEarlyAppearanceList.end()) + { + // Note: aside from LLVOAvatar::resetEarlyAppearanceList() (called on + // teleport), this is the only place where we remove from + // sEarlyAppearanceList, which means any agent who receives an + // AvatarAppearance message but is never actually instantiated will + // remain on the list until the next teleport. This is a resource leak + // but we expect it to be small enough per-session to not cause problems. + sEarlyAppearanceList.erase(it); + LL_INFOS("Avatar") << "Re-requesting AvatarAppearance for new avatar " << id << LL_ENDL; + LLAvatarPropertiesProcessor::getInstance()->sendAvatarTexturesRequest(getID()); + } } std::string LLVOAvatar::avString() const @@ -9937,7 +9953,7 @@ void LLVOAvatar::applyParsedAppearanceMessage(LLAppearanceMessageContents& conte if (visualParamWeightsAreDefault() && mRuthTimer.getElapsedTimeF32() > LOADING_TIMEOUT_SECONDS) { // re-request appearance, hoping that it comes back with a shape next time - LL_INFOS() << "Re-requesting AvatarAppearance for object: " << getID() << LL_ENDL; + LL_INFOS() << "Re-requesting AvatarAppearance for agent: " << getID() << LL_ENDL; LLAvatarPropertiesProcessor::getInstance()->sendAvatarTexturesRequest(getID()); mRuthTimer.reset(); } diff --git a/indra/newview/llvoavatar.h b/indra/newview/llvoavatar.h index 3f33dc54b8..3457e7bb6a 100644 --- a/indra/newview/llvoavatar.h +++ b/indra/newview/llvoavatar.h @@ -949,9 +949,26 @@ protected: ** APPEARANCE **/ +public: + // Used when an AvatarAppearance UDP message is received before the + // corresponding avatar could be created. + static void registerEarlyAppearance(const LLUUID& av_id) + { + sEarlyAppearanceList.emplace(av_id); + } + + // Entries left behind by agents who never get instantiated (e.g. an + // AvatarAppearance message arrives for an avatar we never rez) are a + // small resource leak. Teleporting to another region invalidates the + // whole list, since it was only ever relevant to avatars in the region + // we're leaving, so clear it out at that point to bound the leak. + static void resetEarlyAppearanceList() + { + sEarlyAppearanceList.clear(); + } + LLPointer mLastProcessedAppearance; -public: void parseAppearanceMessage(LLMessageSystem* mesgsys, LLAppearanceMessageContents& msg); void processAvatarAppearance(LLMessageSystem* mesgsys); void applyParsedAppearanceMessage(LLAppearanceMessageContents& contents, bool slam_params); @@ -983,6 +1000,8 @@ private: bool mIsEditingAppearance; // flag for if we're actively in appearance editing mode bool mUseLocalAppearance; // flag for if we're using a local composite + static uuid_list_t sEarlyAppearanceList; + //-------------------------------------------------------------------- // Visibility //-------------------------------------------------------------------- -- cgit v1.3 From 63eb2da05a6211ce4e038f1bf1aef5dda3c6ac77 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 19 Aug 2026 08:57:03 -0700 Subject: move zerocode logic to util, add unit-tests (#6152) --- indra/llmessage/CMakeLists.txt | 2 + indra/llmessage/lltemplatemessagebuilder.cpp | 93 ++-------- indra/llmessage/llzerocode.h | 211 ++++++++++++++++++++++ indra/llmessage/message.cpp | 152 +--------------- indra/llmessage/message.h | 1 - indra/llmessage/tests/llzerocode_test.cpp | 253 +++++++++++++++++++++++++++ 6 files changed, 490 insertions(+), 222 deletions(-) create mode 100644 indra/llmessage/llzerocode.h create mode 100644 indra/llmessage/tests/llzerocode_test.cpp (limited to 'indra/llmessage/message.cpp') diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index f143ee18cf..2306c377ba 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -165,6 +165,7 @@ set(llmessage_HEADER_FILES llxfer_mem.h llxfer_vfile.h llxorcipher.h + llzerocode.h machine.h mean_collision_data.h message.h @@ -228,6 +229,7 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(llservicebuilder "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lltemplatemessagebuilder "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llxfer_file "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(llzerocode "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llxorcipher "" "${test_libs}") LL_ADD_INTEGRATION_TEST(message "" "${test_libs}") endif (LL_TESTS) diff --git a/indra/llmessage/lltemplatemessagebuilder.cpp b/indra/llmessage/lltemplatemessagebuilder.cpp index 758d6d343f..c4ff63bea4 100644 --- a/indra/llmessage/lltemplatemessagebuilder.cpp +++ b/indra/llmessage/lltemplatemessagebuilder.cpp @@ -29,6 +29,7 @@ #include "lltemplatemessagebuilder.h" #include "llmessagetemplate.h" +#include "llzerocode.h" #include "llmath.h" #include "llquaternion.h" #include "u64.h" @@ -492,93 +493,29 @@ void LLTemplateMessageBuilder::addUUID(const char *varname, const LLUUID& uuid) addData(varname, uuid.mData, MVT_LLUUID, sizeof(uuid.mData)); } -static S32 zero_code(U8 **data, U32 *data_size) +void LLTemplateMessageBuilder::compressMessage(U8*& buf_ptr, U32& buffer_length) { - // Encoded send buffer needs to be slightly larger since the zero - // coding can potentially increase the size of the send data. - static U8 encodedSendBuffer[2 * MAX_BUFFER_SIZE]; - - S32 count = *data_size; - - S32 net_gain = 0; - U8 num_zeroes = 0; - - U8 *inptr = (U8 *)*data; - U8 *outptr = (U8 *)encodedSendBuffer; - -// skip the packet id field - - for (U32 ii = 0; ii < LL_PACKET_ID_SIZE ; ++ii) - { - count--; - *outptr++ = *inptr++; - } - -// build encoded packet, keeping track of net size gain - -// sequential zero bytes are encoded as 0 [U8 count] -// with 0 0 [count] representing wrap (>256 zeroes) - - while (count--) + if(ME_ZEROCODED != mCurrentSMessageTemplate->getEncoding()) { - if (!(*inptr)) // in a zero count - { - if (num_zeroes) - { - if (++num_zeroes > 254) - { - *outptr++ = num_zeroes; - num_zeroes = 0; - } - net_gain--; // subseqent zeroes save one - } - else - { - *outptr++ = 0; - net_gain++; // starting a zero count adds one - num_zeroes = 1; - } - inptr++; - } - else - { - if (num_zeroes) - { - *outptr++ = num_zeroes; - num_zeroes = 0; - } - *outptr++ = *inptr++; - } + return; } - if (num_zeroes) - { - *outptr++ = num_zeroes; - } + // Encoded send buffer needs to be slightly larger since the zero + // coding can potentially increase the size of the send data. + static U8 encodedSendBuffer[2 * MAX_BUFFER_SIZE]; - if (net_gain < 0) + S32 encoded_size = LLZeroCode::encode(buf_ptr, buffer_length, + encodedSendBuffer, sizeof(encodedSendBuffer), + LL_PACKET_ID_SIZE); + if (encoded_size >= 0) { // TODO: babbage: reinstate stat collecting... //mCompressedPacketsOut++; - //mUncompressedBytesOut += *data_size; - - *data = encodedSendBuffer; - *data_size += net_gain; - encodedSendBuffer[0] |= LL_ZERO_CODE_FLAG; // set the head bit to indicate zero coding - - //mCompressedBytesOut += *data_size; - - } - //mTotalBytesOut += *data_size; + //mUncompressedBytesOut += buffer_length; + //mCompressedBytesOut += encoded_size; - return(net_gain); -} - -void LLTemplateMessageBuilder::compressMessage(U8*& buf_ptr, U32& buffer_length) -{ - if(ME_ZEROCODED == mCurrentSMessageTemplate->getEncoding()) - { - zero_code(&buf_ptr, &buffer_length); + buf_ptr = encodedSendBuffer; + buffer_length = (U32)encoded_size; } } diff --git a/indra/llmessage/llzerocode.h b/indra/llmessage/llzerocode.h new file mode 100644 index 0000000000..7246c9d946 --- /dev/null +++ b/indra/llmessage/llzerocode.h @@ -0,0 +1,211 @@ +/** + * @file llzerocode.h + * @brief Zero-code run-length compression used by the LLMessageSystem UDP protocol. + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLZEROCODE_H +#define LL_LLZEROCODE_H + +#include + +#include "stdtypes.h" + +// Zero-coding compresses runs of zero bytes in a packet body, leaving the +// first header_size bytes of the buffer (the packet header - flags, +// sequence number, offset, etc.) untouched aside from the flag bit below. +// +// Runs of zero bytes in the body are replaced by a two-byte token: +// 0x00 N - represents N zero bytes, for N in 1..254 +// 0x00 0x00 N - represents (255 + N) zero bytes (wrap/overflow case, +// produced by decode()'s wire format but never emitted +// by encode(), which instead starts a fresh 0x00 token +// every 255 zero bytes) +namespace LLZeroCode +{ + // High bit of the first header byte: set by encode() and cleared by + // decode() to indicate whether the body that follows is zero-coded. + const U8 FLAG = 0x80; + + // Zero-codes src (src_size bytes, the first header_size of which are the + // packet header and are copied verbatim) into dst. + // + // dst_capacity must be at least 2 * src_size: a pathological body of + // isolated zero bytes can nearly double in size when encoded. + // + // Returns the encoded size (with FLAG set in dst[0]) if doing so made the + // packet smaller. Returns -1 if compression would not help (or the + // arguments are invalid), in which case dst is left untouched and the + // caller should keep using the original, uncompressed buffer. + inline S32 encode(const U8* src, U32 src_size, U8* dst, U32 dst_capacity, U32 header_size) + { + if (src_size < header_size || dst_capacity < 2 * src_size) + { + return -1; + } + + S32 count = (S32)(src_size - header_size); + S32 net_gain = 0; + U8 num_zeroes = 0; + + const U8* inptr = src; + U8* outptr = dst; + + // copy the header verbatim + for (U32 ii = 0; ii < header_size; ++ii) + { + *outptr++ = *inptr++; + } + + // sequential zero bytes are encoded as 0 [U8 count]; a run longer + // than 254 bytes is split into consecutive 0 [U8 count] tokens. + while (count--) + { + if (!(*inptr)) // in a zero count + { + if (num_zeroes) + { + if (++num_zeroes > 254) + { + *outptr++ = num_zeroes; + num_zeroes = 0; + } + net_gain--; // subsequent zeroes save one + } + else + { + *outptr++ = 0; + net_gain++; // starting a zero count adds one + num_zeroes = 1; + } + inptr++; + } + else + { + if (num_zeroes) + { + *outptr++ = num_zeroes; + num_zeroes = 0; + } + *outptr++ = *inptr++; + } + } + + if (num_zeroes) + { + *outptr++ = num_zeroes; + } + + if (net_gain >= 0) + { + // compression did not shrink the packet; caller should keep the original + return -1; + } + + dst[0] |= FLAG; + return (S32)src_size + net_gain; + } + + // Expands a zero-coded src (src_size bytes) into dst. + // + // If FLAG is not set in src[0], the body is not zero-coded: no work is + // done and the function returns 0. + // + // On success, returns the number of bytes written to dst (always includes + // the header_size header bytes, copied verbatim except for FLAG being + // cleared from dst[0]). + // + // If expansion would write past dst_capacity - which only a malformed or + // malicious packet should cause - decoding is aborted, *overflow is set + // true, and the returned size reflects however much (if anything) was + // salvaged; the caller should treat the packet as invalid. + inline U32 decode(const U8* src, U32 src_size, U8* dst, U32 dst_capacity, U32 header_size, bool& overflow) + { + overflow = false; + + if (src_size < header_size || !(src[0] & FLAG)) + { + return 0; + } + + S32 count = (S32)(src_size - header_size); + + const U8* inptr = src; + U8* outptr = dst; + + for (U32 ii = 0; ii < header_size; ++ii) + { + *outptr++ = *inptr++; + } + dst[0] &= ~FLAG; + + // reconstruct the body: a 0x00 byte starts a run; the byte(s) that + // follow give its length (see the wire format described above). + while (count--) + { + if (outptr > &dst[dst_capacity - 1]) + { + overflow = true; + outptr = dst; + break; + } + + if (!((*outptr++ = *inptr++))) + { + while ((count--) && (!(*inptr))) + { + if (outptr > &dst[dst_capacity - 256]) + { + overflow = true; + outptr = dst; + count = -1; + break; + } + *outptr++ = *inptr++; + memset(outptr, 0, 255); + outptr += 255; + } + + if (count < 0) + { + break; + } + else + { + if (outptr > &dst[dst_capacity - (*inptr)]) + { + overflow = true; + outptr = dst; + } + memset(outptr, 0, (*inptr) - 1); + outptr += ((*inptr) - 1); + inptr++; + } + } + } + + return (U32)(outptr - dst); + } +} + +#endif // LL_LLZEROCODE_H diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index db7405f93f..71210f7535 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -67,6 +67,7 @@ #include "llsdmessagereader.h" #include "llsdserialize.h" #include "llstring.h" +#include "llzerocode.h" #include "lltransfermanager.h" #include "lluuid.h" #include "llxfermanager.h" @@ -3060,85 +3061,6 @@ U32 LLMessageSystem::getListenPort( void ) const return mPort; } -// TODO: babbage: remove this horror! -S32 LLMessageSystem::zeroCodeAdjustCurrentSendTotal() -{ - if(mMessageBuilder == mLLSDMessageBuilder) - { - // babbage: don't compress LLSD messages, so delta is 0 - return 0; - } - - if (! mMessageBuilder->isBuilt()) - { - mSendSize = mMessageBuilder->buildMessage( - mSendBuffer, - MAX_BUFFER_SIZE, - 0); - } - // TODO: babbage: remove this horror - mMessageBuilder->setBuilt(false); - - S32 count = mSendSize; - - S32 net_gain = 0; - U8 num_zeroes = 0; - - U8 *inptr = (U8 *)mSendBuffer; - -// skip the packet id field - - for (U32 ii = 0; ii < LL_PACKET_ID_SIZE; ++ii) - { - count--; - inptr++; - } - -// don't actually build, just test - -// sequential zero bytes are encoded as 0 [U8 count] -// with 0 0 [count] representing wrap (>256 zeroes) - - while (count--) - { - if (!(*inptr)) // in a zero count - { - if (num_zeroes) - { - if (++num_zeroes > 254) - { - num_zeroes = 0; - } - net_gain--; // subseqent zeroes save one - } - else - { - net_gain++; // starting a zero count adds one - num_zeroes = 1; - } - inptr++; - } - else - { - if (num_zeroes) - { - num_zeroes = 0; - } - inptr++; - } - } - if (net_gain < 0) - { - return net_gain; - } - else - { - return 0; - } -} - - - S32 LLMessageSystem::zeroCodeExpand(U8** data, S32* data_size) { if ((*data_size ) < LL_MINIMUM_VALID_PACKET_SIZE) @@ -3159,74 +3081,18 @@ S32 LLMessageSystem::zeroCodeExpand(U8** data, S32* data_size) mCompressedPacketsIn++; mCompressedBytesIn += *data_size; - *data[0] &= (~LL_ZERO_CODE_FLAG); - - S32 count = (*data_size); - - U8 *inptr = (U8 *)*data; - U8 *outptr = (U8 *)mEncodedRecvBuffer; - -// skip the packet id field - - for (U32 ii = 0; ii < LL_PACKET_ID_SIZE; ++ii) - { - count--; - *outptr++ = *inptr++; - } - -// reconstruct encoded packet, keeping track of net size gain - -// sequential zero bytes are encoded as 0 [U8 count] -// with 0 0 [count] representing wrap (>256 zeroes) - - while (count--) + bool overflow = false; + U32 decoded_size = LLZeroCode::decode(*data, (U32)*data_size, + mEncodedRecvBuffer, sizeof(mEncodedRecvBuffer), + LL_PACKET_ID_SIZE, overflow); + if (overflow) { - if (outptr > (&mEncodedRecvBuffer[MAX_BUFFER_SIZE-1])) - { - LL_WARNS("Messaging") << "attempt to write past reasonable encoded buffer size 1" << LL_ENDL; - callExceptionFunc(MX_WROTE_PAST_BUFFER_SIZE); - outptr = mEncodedRecvBuffer; - break; - } - if (!((*outptr++ = *inptr++))) - { - while (((count--)) && (!(*inptr))) - { - *outptr++ = *inptr++; - if (outptr > (&mEncodedRecvBuffer[MAX_BUFFER_SIZE-256])) - { - LL_WARNS("Messaging") << "attempt to write past reasonable encoded buffer size 2" << LL_ENDL; - callExceptionFunc(MX_WROTE_PAST_BUFFER_SIZE); - outptr = mEncodedRecvBuffer; - count = -1; - break; - } - memset(outptr,0,255); - outptr += 255; - } - - if (count < 0) - { - break; - } - - else - { - if (outptr > (&mEncodedRecvBuffer[MAX_BUFFER_SIZE-(*inptr)])) - { - LL_WARNS("Messaging") << "attempt to write past reasonable encoded buffer size 3" << LL_ENDL; - callExceptionFunc(MX_WROTE_PAST_BUFFER_SIZE); - outptr = mEncodedRecvBuffer; - } - memset(outptr,0,(*inptr) - 1); - outptr += ((*inptr) - 1); - inptr++; - } - } + LL_WARNS("Messaging") << "attempt to write past reasonable encoded buffer size" << LL_ENDL; + callExceptionFunc(MX_WROTE_PAST_BUFFER_SIZE); } *data = mEncodedRecvBuffer; - *data_size = (S32)(outptr - mEncodedRecvBuffer); + *data_size = (S32)decoded_size; mUncompressedBytesIn += *data_size; return(in_size); diff --git a/indra/llmessage/message.h b/indra/llmessage/message.h index e23a67c04a..2e19e567e6 100644 --- a/indra/llmessage/message.h +++ b/indra/llmessage/message.h @@ -604,7 +604,6 @@ public: //void buildMessage(); S32 zeroCodeExpand(U8 **data, S32 *data_size); - S32 zeroCodeAdjustCurrentSendTotal(); // Uses ping-based retry S32 sendReliable(const LLHost &host); diff --git a/indra/llmessage/tests/llzerocode_test.cpp b/indra/llmessage/tests/llzerocode_test.cpp new file mode 100644 index 0000000000..1aba46149d --- /dev/null +++ b/indra/llmessage/tests/llzerocode_test.cpp @@ -0,0 +1,253 @@ +/** + * @file llzerocode_test.cpp + * @brief LLZeroCode test cases. + * + * $LicenseInfo:firstyear=2026&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "../llzerocode.h" + +#include "../test/lltut.h" + +#include + +namespace tut +{ + struct zerocode_data + { + }; + typedef test_group zerocode_test; + typedef zerocode_test::object zerocode_object; + tut::zerocode_test zerocode_testcase("LLZeroCode"); + + // Builds header_size bytes of header (with header[0] == header0) followed by body. + static std::vector makeBuffer(U32 header_size, U8 header0, const std::vector& body) + { + std::vector buf(header_size, 0); + if (header_size) + { + buf[0] = header0; + } + buf.insert(buf.end(), body.begin(), body.end()); + return buf; + } + + // Runs src through encode() then decode(), and (when compression is expected to help) + // asserts the round trip reproduces src exactly, including untouched header bits other + // than the zero-code flag. When compression is not expected to help, asserts encode() + // refuses it. + static void ensureRoundTrip(const char* msg, U32 header_size, U8 header0, + const std::vector& body, bool expect_compressed) + { + std::vector src = makeBuffer(header_size, header0, body); + std::vector enc(2 * src.size() + 16, 0xAA); + + S32 enc_size = LLZeroCode::encode(src.data(), (U32)src.size(), + enc.data(), (U32)enc.size(), header_size); + + if (!expect_compressed) + { + ensure(std::string(msg) + ": encode should refuse (no benefit)", enc_size < 0); + return; + } + + ensure(std::string(msg) + ": encode should succeed", enc_size >= 0); + ensure(std::string(msg) + ": encoded size should be smaller", (U32)enc_size < src.size()); + ensure(std::string(msg) + ": FLAG should be set on encoded output", (enc[0] & LLZeroCode::FLAG) != 0); + if (header_size > 0) + { + ensure_equals(std::string(msg) + ": non-flag header bits preserved on encode", + (U8)(enc[0] & ~LLZeroCode::FLAG), (U8)(header0 & ~LLZeroCode::FLAG)); + } + + std::vector dec(src.size() + 16, 0xBB); + bool overflow = false; + U32 dec_size = LLZeroCode::decode(enc.data(), (U32)enc_size, + dec.data(), (U32)dec.size(), header_size, overflow); + ensure(std::string(msg) + ": decode should not overflow", !overflow); + ensure_equals(std::string(msg) + ": decoded size should match original", dec_size, (U32)src.size()); + ensure(std::string(msg) + ": decoded bytes should match original", + memcmp(dec.data(), src.data(), src.size()) == 0); + } + + // Basic mixed body; header carries an unrelated flag bit (0x40) that must survive untouched. + template<> template<> + void zerocode_object::test<1>() + { + ensureRoundTrip("mixed body", 6, 0x40, + {0,0,0,5,0,0,0,0,0,0,7,8,9,0,0}, true); + } + + // A body with no zero bytes at all cannot benefit from zero-coding. + template<> template<> + void zerocode_object::test<2>() + { + ensureRoundTrip("no zero bytes", 1, 0x00, {1,2,3,4,5,6,7,8,9}, false); + } + + // Isolated zero-byte runs of length 1 or 2 cost as much or more than they save + // (marker + terminator == 2 bytes), so encode must refuse them. + template<> template<> + void zerocode_object::test<3>() + { + ensureRoundTrip("isolated single zero", 1, 0x00, {5, 0}, false); + ensureRoundTrip("isolated double zero", 1, 0x00, {5, 0, 0, 9}, false); + } + + // A long enough zero run followed by other data pays off. + template<> template<> + void zerocode_object::test<4>() + { + std::vector body(20, 0); + body.push_back(99); + ensureRoundTrip("long zero run", 1, 0x00, body, true); + } + + // Wrap boundary: runs are split into chunks of at most 255 zero bytes each + // (encode never emits the doubled-0x00 wire form; see test<7> for that). + // A run only shrinks the packet once it is longer than 2 bytes. + template<> template<> + void zerocode_object::test<5>() + { + static const U32 lengths[] = {1, 2, 253, 254, 255, 256, 257, 300, 509, 510, 511, 1000}; + for (U32 n : lengths) + { + std::vector body(n, 0); + body.push_back(42); // trailing nonzero byte + ensureRoundTrip("wrap boundary (with trailing byte)", 1, 0x00, body, n > 2); + } + } + + // Same boundary check, but with the zero run flushed at end-of-buffer + // (no trailing nonzero byte to force the terminator write mid-loop). + template<> template<> + void zerocode_object::test<6>() + { + static const U32 lengths[] = {1, 2, 254, 255, 256, 300}; + for (U32 n : lengths) + { + std::vector body(n, 0); + ensureRoundTrip("wrap boundary (end-of-buffer flush)", 1, 0x00, body, n > 2); + } + } + + // decode() must also accept the "0x00 0x00 N" doubled-marker wire format + // (worth +255 zero bytes per extra marker) for compatibility with any + // encoder other than this one's chunking strategy, even though encode() + // itself never emits it. + template<> template<> + void zerocode_object::test<7>() + { + // header (1 byte, FLAG set) + [0x00 marker][0x00 extra-wrap-marker][terminator=5] + // decoded body length = 1 (marker) + 1 (extra marker) + 255 (memset) + (5 - 1) = 261 + std::vector encoded = {(U8)LLZeroCode::FLAG, 0x00, 0x00, 0x05}; + std::vector dec(1024, 0xDD); + bool overflow = false; + U32 dec_size = LLZeroCode::decode(encoded.data(), (U32)encoded.size(), + dec.data(), (U32)dec.size(), 1, overflow); + ensure("wrap format: no overflow", !overflow); + ensure_equals("wrap format: decoded size", dec_size, (U32)262); + ensure_equals("wrap format: FLAG cleared on header", dec[0], (U8)0x00); + for (U32 i = 1; i < dec_size; ++i) + { + ensure_equals("wrap format: decoded byte is zero", dec[i], (U8)0); + } + } + + // decode() is a no-op (returns 0, does not touch dst) when FLAG is not set. + template<> template<> + void zerocode_object::test<8>() + { + std::vector src = makeBuffer(6, 0x00, {1,2,3,0,0,0}); + std::vector dec(64, 0xCC); + bool overflow = false; + U32 dec_size = LLZeroCode::decode(src.data(), (U32)src.size(), + dec.data(), (U32)dec.size(), 6, overflow); + ensure_equals("not zero-coded: decode returns 0", dec_size, (U32)0); + ensure("not zero-coded: no overflow", !overflow); + } + + // encode() refuses to write into an undersized destination buffer. + template<> template<> + void zerocode_object::test<9>() + { + std::vector body(50, 0); + std::vector src = makeBuffer(1, 0, body); + std::vector enc(src.size(), 0); // smaller than the required 2 * src_size + S32 r = LLZeroCode::encode(src.data(), (U32)src.size(), enc.data(), (U32)enc.size(), 1); + ensure("encode: capacity guard rejects undersized dst", r < 0); + } + + // decode() reports overflow (rather than writing out of bounds) when dst is too small, + // whether the destination is smaller than a single byte's worth of headroom... + template<> template<> + void zerocode_object::test<10>() + { + std::vector body(500, 0); + std::vector src = makeBuffer(1, 0, body); + std::vector enc(2 * src.size() + 16, 0); + S32 enc_size = LLZeroCode::encode(src.data(), (U32)src.size(), enc.data(), (U32)enc.size(), 1); + ensure("decode overflow setup: encode succeeded", enc_size >= 0); + + std::vector dec(1, 0); // capacity == header_size exactly + bool overflow = false; + U32 dec_size = LLZeroCode::decode(enc.data(), (U32)enc_size, + dec.data(), (U32)dec.size(), 1, overflow); + ensure("decode overflow (minimal capacity): overflow flagged", overflow); + ensure("decode overflow (minimal capacity): size bounded by capacity", dec_size <= dec.size()); + } + + // ...or comfortably larger than 256 bytes but still short of the true decoded size. + template<> template<> + void zerocode_object::test<11>() + { + std::vector body(500, 0); + std::vector src = makeBuffer(1, 0, body); + std::vector enc(2 * src.size() + 16, 0); + S32 enc_size = LLZeroCode::encode(src.data(), (U32)src.size(), enc.data(), (U32)enc.size(), 1); + ensure("decode overflow setup: encode succeeded", enc_size >= 0); + + std::vector dec(300, 0); // > 256, but less than the true decoded size (~501) + bool overflow = false; + U32 dec_size = LLZeroCode::decode(enc.data(), (U32)enc_size, + dec.data(), (U32)dec.size(), 1, overflow); + ensure("decode overflow (insufficient capacity): overflow flagged", overflow); + ensure("decode overflow (insufficient capacity): size bounded by capacity", dec_size <= dec.size()); + } + + // A pathological alternating zero/non-zero pattern grows under zero-coding + // (every isolated zero costs 2 output bytes for 1 input byte), so encode + // must refuse it and leave the original buffer in use. + template<> template<> + void zerocode_object::test<12>() + { + std::vector body; + for (int i = 0; i < 200; ++i) + { + body.push_back(0); + body.push_back((U8)(i + 1)); + } + ensureRoundTrip("alternating pattern", 1, 0x00, body, false); + } +} -- cgit v1.3