From b6841d75c2f259c84d5ab6b012bd2ae37d985451 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Fri, 15 Apr 2022 19:02:07 -0500 Subject: SL-17219 WIP - Texture pipeline overhaul --- indra/llcommon/llapr.cpp | 2 + indra/llcommon/llqueuedthread.cpp | 289 +++++++++++++++++++------------------- indra/llcommon/llqueuedthread.h | 51 ++----- indra/llcommon/lltimer.cpp | 3 +- indra/llcommon/llworkerthread.cpp | 27 ++-- indra/llcommon/llworkerthread.h | 11 +- indra/llcommon/workqueue.h | 8 +- 7 files changed, 177 insertions(+), 214 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llapr.cpp b/indra/llcommon/llapr.cpp index db94765871..b537102510 100644 --- a/indra/llcommon/llapr.cpp +++ b/indra/llcommon/llapr.cpp @@ -526,6 +526,7 @@ S32 LLAPRFile::seek(apr_file_t* file_handle, apr_seek_where_t where, S32 offset) //static S32 LLAPRFile::readEx(const std::string& filename, void *buf, S32 offset, S32 nbytes, LLVolatileAPRPool* pool) { + LL_PROFILE_ZONE_SCOPED; //***************************************** LLAPRFilePoolScope scope(pool); apr_file_t* file_handle = open(filename, scope.getVolatileAPRPool(), APR_READ|APR_BINARY); @@ -570,6 +571,7 @@ S32 LLAPRFile::readEx(const std::string& filename, void *buf, S32 offset, S32 nb //static S32 LLAPRFile::writeEx(const std::string& filename, void *buf, S32 offset, S32 nbytes, LLVolatileAPRPool* pool) { + LL_PROFILE_ZONE_SCOPED; apr_int32_t flags = APR_CREATE|APR_WRITE|APR_BINARY; if (offset < 0) { diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index 8cef4293cd..871c42f7ee 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -26,20 +26,25 @@ #include "linden_common.h" #include "llqueuedthread.h" +#include + #include "llstl.h" #include "lltimer.h" // ms_sleep() -#include "lltracethreadrecorder.h" +#include "llmutex.h" //============================================================================ // MAIN THREAD LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded, bool should_pause) : - LLThread(name), - mThreaded(threaded), - mIdleThread(TRUE), - mNextHandle(0), - mStarted(FALSE) + LLThread(name), + mThreaded(threaded), + mIdleThread(TRUE), + mNextHandle(0), + mStarted(FALSE), + mRequestQueue(name, 1024 * 1024) { + mMainQueue = LL::WorkQueue::getInstance("mainloop"); + if (mThreaded) { if(should_pause) @@ -104,6 +109,8 @@ void LLQueuedThread::shutdown() { LL_WARNS() << "~LLQueuedThread() called with active requests: " << active_count << LL_ENDL; } + + mRequestQueue.close(); } //---------------------------------------------------------------------------- @@ -112,6 +119,7 @@ void LLQueuedThread::shutdown() // virtual S32 LLQueuedThread::update(F32 max_time_ms) { + LL_PROFILE_ZONE_SCOPED; if (!mStarted) { if (!mThreaded) @@ -125,29 +133,30 @@ S32 LLQueuedThread::update(F32 max_time_ms) S32 LLQueuedThread::updateQueue(F32 max_time_ms) { - F64 max_time = (F64)max_time_ms * .001; - LLTimer timer; - S32 pending = 1; - + LL_PROFILE_ZONE_SCOPED; // Frame Update if (mThreaded) { - pending = getPending(); - if(pending > 0) + // schedule a call to threadedUpdate for every call to updateQueue + mRequestQueue.post([=]() + { + LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qt - update"); + mIdleThread = FALSE; + threadedUpdate(); + mIdleThread = TRUE; + } + ); + if(getPending() > 0) { - unpause(); - } + unpause(); + } } else { - while (pending > 0) - { - pending = processNextRequest(); - if (max_time && timer.getElapsedTimeF64() > max_time) - break; - } + mRequestQueue.runFor(std::chrono::microseconds((int) (max_time_ms*1000.f))); + threadedUpdate(); } - return pending; + return getPending(); } void LLQueuedThread::incQueue() @@ -166,11 +175,7 @@ void LLQueuedThread::incQueue() // May be called from any thread S32 LLQueuedThread::getPending() { - S32 res; - lockData(); - res = mRequestQueue.size(); - unlockData(); - return res; + return mRequestQueue.size(); } // MAIN thread @@ -195,35 +200,28 @@ void LLQueuedThread::waitOnPending() // MAIN thread void LLQueuedThread::printQueueStats() { - lockData(); - if (!mRequestQueue.empty()) + U32 size = mRequestQueue.size(); + if (size > 0) { - QueuedRequest *req = *mRequestQueue.begin(); - LL_INFOS() << llformat("Pending Requests:%d Current status:%d", mRequestQueue.size(), req->getStatus()) << LL_ENDL; + LL_INFOS() << llformat("Pending Requests:%d ", mRequestQueue.size()) << LL_ENDL; } else { LL_INFOS() << "Queued Thread Idle" << LL_ENDL; } - unlockData(); } // MAIN thread LLQueuedThread::handle_t LLQueuedThread::generateHandle() { - lockData(); - while ((mNextHandle == nullHandle()) || (mRequestHash.find(mNextHandle))) - { - mNextHandle++; - } - const LLQueuedThread::handle_t res = mNextHandle++; - unlockData(); + U32 res = ++mNextHandle; return res; } // MAIN thread bool LLQueuedThread::addRequest(QueuedRequest* req) { + LL_PROFILE_ZONE_SCOPED; if (mStatus == QUITTING) { return false; @@ -231,14 +229,14 @@ bool LLQueuedThread::addRequest(QueuedRequest* req) lockData(); req->setStatus(STATUS_QUEUED); - mRequestQueue.insert(req); - mRequestHash.insert(req); + mRequestHash.insert(req); #if _DEBUG // LL_INFOS() << llformat("LLQueuedThread::Added req [%08d]",handle) << LL_ENDL; #endif unlockData(); - incQueue(); + llassert(!mDataLock->isSelfLocked()); + mRequestQueue.post([this, req]() { processRequest(req); }); return true; } @@ -246,6 +244,7 @@ bool LLQueuedThread::addRequest(QueuedRequest* req) // MAIN thread bool LLQueuedThread::waitForResult(LLQueuedThread::handle_t handle, bool auto_complete) { + LL_PROFILE_ZONE_SCOPED; llassert (handle != nullHandle()); bool res = false; bool waspaused = isPaused(); @@ -312,6 +311,7 @@ LLQueuedThread::status_t LLQueuedThread::getRequestStatus(handle_t handle) void LLQueuedThread::abortRequest(handle_t handle, bool autocomplete) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; lockData(); QueuedRequest* req = (QueuedRequest*)mRequestHash.find(handle); if (req) @@ -333,30 +333,9 @@ void LLQueuedThread::setFlags(handle_t handle, U32 flags) unlockData(); } -void LLQueuedThread::setPriority(handle_t handle, U32 priority) -{ - lockData(); - QueuedRequest* req = (QueuedRequest*)mRequestHash.find(handle); - if (req) - { - if(req->getStatus() == STATUS_INPROGRESS) - { - // not in list - req->setPriority(priority); - } - else if(req->getStatus() == STATUS_QUEUED) - { - // remove from list then re-insert - llverify(mRequestQueue.erase(req) == 1); - req->setPriority(priority); - mRequestQueue.insert(req); - } - } - unlockData(); -} - bool LLQueuedThread::completeRequest(handle_t handle) { + LL_PROFILE_ZONE_SCOPED; bool res = false; lockData(); QueuedRequest* req = (QueuedRequest*)mRequestHash.find(handle); @@ -399,89 +378,115 @@ bool LLQueuedThread::check() //============================================================================ // Runs on its OWN thread -S32 LLQueuedThread::processNextRequest() +void LLQueuedThread::processRequest(LLQueuedThread::QueuedRequest* req) { - QueuedRequest *req; + LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; + + mIdleThread = FALSE; + //threadedUpdate(); + // Get next request from pool lockData(); - while(1) - { - req = NULL; - if (mRequestQueue.empty()) - { - break; - } - req = *mRequestQueue.begin(); - mRequestQueue.erase(mRequestQueue.begin()); - if ((req->getFlags() & FLAG_ABORT) || (mStatus == QUITTING)) - { - req->setStatus(STATUS_ABORTED); - req->finishRequest(false); - if (req->getFlags() & FLAG_AUTO_COMPLETE) - { - mRequestHash.erase(req); - req->deleteRequest(); -// check(); - } - continue; - } - llassert_always(req->getStatus() == STATUS_QUEUED); - break; - } - U32 start_priority = 0 ; - if (req) - { - req->setStatus(STATUS_INPROGRESS); - start_priority = req->getPriority(); - } - unlockData(); - - // This is the only place we will call req->setStatus() after - // it has initially been seet to STATUS_QUEUED, so it is - // safe to access req. - if (req) + if ((req->getFlags() & FLAG_ABORT) || (mStatus == QUITTING)) { - // process request - bool complete = req->processRequest(); - - if (complete) + LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qtpr - abort"); + req->setStatus(STATUS_ABORTED); + req->finishRequest(false); + if (req->getFlags() & FLAG_AUTO_COMPLETE) { - lockData(); - req->setStatus(STATUS_COMPLETE); - req->finishRequest(true); - if (req->getFlags() & FLAG_AUTO_COMPLETE) - { - mRequestHash.erase(req); - req->deleteRequest(); + mRequestHash.erase(req); + req->deleteRequest(); // check(); - } - unlockData(); } - else - { - lockData(); - req->setStatus(STATUS_QUEUED); - mRequestQueue.insert(req); - unlockData(); - if (mThreaded && start_priority < PRIORITY_NORMAL) - { - ms_sleep(1); // sleep the thread a little - } - } - - LLTrace::get_thread_recorder()->pushToParent(); + unlockData(); } + else + { + llassert_always(req->getStatus() == STATUS_QUEUED); + + if (req) + { + req->setStatus(STATUS_INPROGRESS); + } + unlockData(); + + // This is the only place we will call req->setStatus() after + // it has initially been seet to STATUS_QUEUED, so it is + // safe to access req. + if (req) + { + // process request + bool complete = req->processRequest(); + + if (complete) + { + LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qtpr - complete"); + lockData(); + req->setStatus(STATUS_COMPLETE); + req->finishRequest(true); + if (req->getFlags() & FLAG_AUTO_COMPLETE) + { + mRequestHash.erase(req); + req->deleteRequest(); + // check(); + } + unlockData(); + } + else + { + LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qtpr - retry"); + //put back on queue and try again in 0.1ms + lockData(); + req->setStatus(STATUS_QUEUED); + + unlockData(); + + llassert(!mDataLock->isSelfLocked()); + +#if 0 + // try again on next frame + // NOTE: tried using "post" with a time in the future, but this + // would invariably cause this thread to wait for a long time (10+ ms) + // while work is pending + bool ret = LL::WorkQueue::postMaybe( + mMainQueue, + [=]() + { + LL_PROFILE_ZONE_NAMED("processRequest - retry"); + mRequestQueue.post([=]() + { + LL_PROFILE_ZONE_NAMED("processRequest - retry"); // <-- not redundant, track retry on both queues + processRequest(req); + }); + }); + llassert(ret); +#else + using namespace std::chrono_literals; + auto retry_time = LL::WorkQueue::TimePoint::clock::now() + 16ms; + mRequestQueue.post([=] + { + LL_PROFILE_ZONE_NAMED("processRequest - retry"); + while (LL::WorkQueue::TimePoint::clock::now() < retry_time) + { + std::this_thread::yield(); //note: don't use LLThread::yield here to avoid + } + processRequest(req); + }); +#endif + + } + } + } - S32 pending = getPending(); - return pending; + mIdleThread = TRUE; } // virtual bool LLQueuedThread::runCondition() { // mRunCondition must be locked here - if (mRequestQueue.empty() && mIdleThread) + if (mRequestQueue.size() == 0 && mIdleThread) return false; else return true; @@ -495,18 +500,13 @@ void LLQueuedThread::run() startThread(); mStarted = TRUE; - while (1) + + /*while (1) { + LL_PROFILE_ZONE_SCOPED; // this will block on the condition until runCondition() returns true, the thread is unpaused, or the thread leaves the RUNNING state. checkPause(); - if (isQuitting()) - { - LLTrace::get_thread_recorder()->pushToParent(); - endThread(); - break; - } - mIdleThread = FALSE; threadedUpdate(); @@ -515,12 +515,18 @@ void LLQueuedThread::run() if (pending_work == 0) { + //LL_PROFILE_ZONE_NAMED("LLQueuedThread - sleep"); mIdleThread = TRUE; - ms_sleep(1); + //ms_sleep(1); } //LLThread::yield(); // thread should yield after each request - } + }*/ + mRequestQueue.runUntilClose(); + + endThread(); LL_INFOS() << "LLQueuedThread " << mName << " EXITING." << LL_ENDL; + + } // virtual @@ -540,10 +546,9 @@ void LLQueuedThread::threadedUpdate() //============================================================================ -LLQueuedThread::QueuedRequest::QueuedRequest(LLQueuedThread::handle_t handle, U32 priority, U32 flags) : +LLQueuedThread::QueuedRequest::QueuedRequest(LLQueuedThread::handle_t handle, U32 flags) : LLSimpleHashEntry(handle), mStatus(STATUS_UNKNOWN), - mPriority(priority), mFlags(flags) { } diff --git a/indra/llcommon/llqueuedthread.h b/indra/llcommon/llqueuedthread.h index 5d3f873646..6b82ccc434 100644 --- a/indra/llcommon/llqueuedthread.h +++ b/indra/llcommon/llqueuedthread.h @@ -36,6 +36,7 @@ #include "llthread.h" #include "llsimplehash.h" +#include "workqueue.h" //============================================================================ // Note: ~LLQueuedThread is O(N) N=# of queued threads, assumed to be small @@ -45,15 +46,6 @@ class LL_COMMON_API LLQueuedThread : public LLThread { //------------------------------------------------------------------------ public: - enum priority_t { - PRIORITY_IMMEDIATE = 0x7FFFFFFF, - PRIORITY_URGENT = 0x40000000, - PRIORITY_HIGH = 0x30000000, - PRIORITY_NORMAL = 0x20000000, - PRIORITY_LOW = 0x10000000, - PRIORITY_LOWBITS = 0x0FFFFFFF, - PRIORITY_HIGHBITS = 0x70000000 - }; enum status_t { STATUS_EXPIRED = -1, STATUS_UNKNOWN = 0, @@ -82,28 +74,17 @@ public: virtual ~QueuedRequest(); // use deleteRequest() public: - QueuedRequest(handle_t handle, U32 priority, U32 flags = 0); + QueuedRequest(handle_t handle, U32 flags = 0); status_t getStatus() { return mStatus; } - U32 getPriority() const - { - return mPriority; - } U32 getFlags() const { return mFlags; } - bool higherPriority(const QueuedRequest& second) const - { - if ( mPriority == second.mPriority) - return mHashKey < second.mHashKey; - else - return mPriority > second.mPriority; - } - + protected: status_t setStatus(status_t newstatus) { @@ -121,28 +102,11 @@ public: virtual void finishRequest(bool completed); // Always called from thread after request has completed or aborted virtual void deleteRequest(); // Only method to delete a request - void setPriority(U32 pri) - { - // Only do this on a request that is not in a queued list! - mPriority = pri; - }; - protected: LLAtomicBase mStatus; - U32 mPriority; U32 mFlags; }; -protected: - struct queued_request_less - { - bool operator()(const QueuedRequest* lhs, const QueuedRequest* rhs) const - { - return lhs->higherPriority(*rhs); // higher priority in front of queue (set) - } - }; - - //------------------------------------------------------------------------ public: @@ -167,7 +131,7 @@ private: protected: handle_t generateHandle(); bool addRequest(QueuedRequest* req); - S32 processNextRequest(void); + void processRequest(QueuedRequest* req); void incQueue(); public: @@ -186,7 +150,6 @@ public: status_t getRequestStatus(handle_t handle); void abortRequest(handle_t handle, bool autocomplete); void setFlags(handle_t handle, U32 flags); - void setPriority(handle_t handle, U32 priority); bool completeRequest(handle_t handle); // This is public for support classes like LLWorkerThread, // but generally the methods above should be used. @@ -200,8 +163,10 @@ protected: BOOL mStarted; // required when mThreaded is false to call startThread() from update() LLAtomicBool mIdleThread; // request queue is empty (or we are quitting) and the thread is idle - typedef std::set request_queue_t; - request_queue_t mRequestQueue; + //typedef std::set request_queue_t; + //request_queue_t mRequestQueue; + LL::WorkQueue mRequestQueue; + LL::WorkQueue::weak_t mMainQueue; enum { REQUEST_HASH_SIZE = 512 }; // must be power of 2 typedef LLSimpleHash request_hash_t; diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index aaa6df325c..b250bc3e1c 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -64,7 +64,8 @@ LLTimer* LLTimer::sTimer = NULL; #if LL_WINDOWS void ms_sleep(U32 ms) { - Sleep(ms); + LL_PROFILE_ZONE_SCOPED; + std::this_thread::sleep_for(std::chrono::microseconds(ms)); } U32 micro_sleep(U64 us, U32 max_yields) diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index 4b91b2caca..02ce4823b8 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -133,11 +133,11 @@ S32 LLWorkerThread::update(F32 max_time_ms) //---------------------------------------------------------------------------- -LLWorkerThread::handle_t LLWorkerThread::addWorkRequest(LLWorkerClass* workerclass, S32 param, U32 priority) +LLWorkerThread::handle_t LLWorkerThread::addWorkRequest(LLWorkerClass* workerclass, S32 param) { handle_t handle = generateHandle(); - WorkRequest* req = new WorkRequest(handle, priority, workerclass, param); + WorkRequest* req = new WorkRequest(handle, workerclass, param); bool res = addRequest(req); if (!res) @@ -160,8 +160,8 @@ void LLWorkerThread::deleteWorker(LLWorkerClass* workerclass) //============================================================================ // Runs on its OWN thread -LLWorkerThread::WorkRequest::WorkRequest(handle_t handle, U32 priority, LLWorkerClass* workerclass, S32 param) : - LLQueuedThread::QueuedRequest(handle, priority), +LLWorkerThread::WorkRequest::WorkRequest(handle_t handle, LLWorkerClass* workerclass, S32 param) : + LLQueuedThread::QueuedRequest(handle), mWorkerClass(workerclass), mParam(param) { @@ -180,6 +180,7 @@ void LLWorkerThread::WorkRequest::deleteRequest() // virtual bool LLWorkerThread::WorkRequest::processRequest() { + LL_PROFILE_ZONE_SCOPED; LLWorkerClass* workerclass = getWorkerClass(); workerclass->setWorking(true); bool complete = workerclass->doWork(getParam()); @@ -190,6 +191,7 @@ bool LLWorkerThread::WorkRequest::processRequest() // virtual void LLWorkerThread::WorkRequest::finishRequest(bool completed) { + LL_PROFILE_ZONE_SCOPED; LLWorkerClass* workerclass = getWorkerClass(); workerclass->finishWork(getParam(), completed); U32 flags = LLWorkerClass::WCF_WORK_FINISHED | (completed ? 0 : LLWorkerClass::WCF_WORK_ABORTED); @@ -203,7 +205,6 @@ LLWorkerClass::LLWorkerClass(LLWorkerThread* workerthread, const std::string& na : mWorkerThread(workerthread), mWorkerClassName(name), mRequestHandle(LLWorkerThread::nullHandle()), - mRequestPriority(LLWorkerThread::PRIORITY_NORMAL), mMutex(), mWorkFlags(0) { @@ -292,7 +293,7 @@ bool LLWorkerClass::yield() //---------------------------------------------------------------------------- // calls startWork, adds doWork() to queue -void LLWorkerClass::addWork(S32 param, U32 priority) +void LLWorkerClass::addWork(S32 param) { mMutex.lock(); llassert_always(!(mWorkFlags & (WCF_WORKING|WCF_HAVE_WORK))); @@ -306,7 +307,7 @@ void LLWorkerClass::addWork(S32 param, U32 priority) startWork(param); clearFlags(WCF_WORK_FINISHED|WCF_WORK_ABORTED); setFlags(WCF_HAVE_WORK); - mRequestHandle = mWorkerThread->addWorkRequest(this, param, priority); + mRequestHandle = mWorkerThread->addWorkRequest(this, param); mMutex.unlock(); } @@ -321,7 +322,6 @@ void LLWorkerClass::abortWork(bool autocomplete) if (mRequestHandle != LLWorkerThread::nullHandle()) { mWorkerThread->abortRequest(mRequestHandle, autocomplete); - mWorkerThread->setPriority(mRequestHandle, LLQueuedThread::PRIORITY_IMMEDIATE); setFlags(WCF_ABORT_REQUESTED); } mMutex.unlock(); @@ -395,16 +395,5 @@ void LLWorkerClass::scheduleDelete() } } -void LLWorkerClass::setPriority(U32 priority) -{ - mMutex.lock(); - if (mRequestHandle != LLWorkerThread::nullHandle() && mRequestPriority != priority) - { - mRequestPriority = priority; - mWorkerThread->setPriority(mRequestHandle, priority); - } - mMutex.unlock(); -} - //============================================================================ diff --git a/indra/llcommon/llworkerthread.h b/indra/llcommon/llworkerthread.h index 0387e75c65..06bbb7369e 100644 --- a/indra/llcommon/llworkerthread.h +++ b/indra/llcommon/llworkerthread.h @@ -56,7 +56,7 @@ public: virtual ~WorkRequest(); // use deleteRequest() public: - WorkRequest(handle_t handle, U32 priority, LLWorkerClass* workerclass, S32 param); + WorkRequest(handle_t handle, LLWorkerClass* workerclass, S32 param); S32 getParam() { @@ -90,7 +90,7 @@ public: /*virtual*/ S32 update(F32 max_time_ms); - handle_t addWorkRequest(LLWorkerClass* workerclass, S32 param, U32 priority = PRIORITY_NORMAL); + handle_t addWorkRequest(LLWorkerClass* workerclass, S32 param); S32 getNumDeletes() { return (S32)mDeleteList.size(); } // debug @@ -151,10 +151,6 @@ public: bool isWorking() { return getFlags(WCF_WORKING); } bool wasAborted() { return getFlags(WCF_ABORT_REQUESTED); } - // setPriority(): changes the priority of a request - void setPriority(U32 priority); - U32 getPriority() { return mRequestPriority; } - const std::string& getName() const { return mWorkerClassName; } protected: @@ -169,7 +165,7 @@ protected: void setWorkerThread(LLWorkerThread* workerthread); // addWork(): calls startWork, adds doWork() to queue - void addWork(S32 param, U32 priority = LLWorkerThread::PRIORITY_NORMAL); + void addWork(S32 param); // abortWork(): requests that work be aborted void abortWork(bool autocomplete); @@ -193,7 +189,6 @@ protected: LLWorkerThread* mWorkerThread; std::string mWorkerClassName; handle_t mRequestHandle; - U32 mRequestPriority; // last priority set private: LLMutex mMutex; diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 96574a18b9..46f7363830 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -161,10 +161,16 @@ namespace LL void postEvery(const std::chrono::duration& interval, CALLABLE&& callable); + template + bool tryPost(const TimePoint& time, CALLABLE&& callable) + { + return mQueue.tryPush(TimedWork(time, std::move(callable))); + } + template bool tryPost(CALLABLE&& callable) { - return mQueue.tryPush(TimedWork(TimePoint::clock::now(), std::move(callable))); + return mQueue.tryPost(TimePoint::clock::now(), std::move(callable)); } /*------------------------- handshake API --------------------------*/ -- cgit v1.2.3 From b08742d0b3e0000430b8ae772c7c4d25cfe084fa Mon Sep 17 00:00:00 2001 From: Dave Houlton Date: Thu, 28 Apr 2022 11:53:43 -0600 Subject: Add a (broken) material upload handler fxn --- indra/llcommon/llassettype.cpp | 3 ++- indra/llcommon/llassettype.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index e6cc06e8d0..7bf3c6553e 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -96,7 +96,8 @@ LLAssetDictionary::LLAssetDictionary() addEntry(LLAssetType::AT_WIDGET, new AssetEntry("WIDGET", "widget", "widget", false, false, false)); addEntry(LLAssetType::AT_PERSON, new AssetEntry("PERSON", "person", "person", false, false, false)); addEntry(LLAssetType::AT_SETTINGS, new AssetEntry("SETTINGS", "settings", "settings blob", true, true, true)); - addEntry(LLAssetType::AT_UNKNOWN, new AssetEntry("UNKNOWN", "invalid", NULL, false, false, false)); + addEntry(LLAssetType::AT_MATERIAL, new AssetEntry("MATERIAL", "material", "material", true, false, true)); + addEntry(LLAssetType::AT_UNKNOWN, new AssetEntry("UNKNOWN", "invalid", NULL, false, false, false)); addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, FALSE, FALSE, FALSE)); }; diff --git a/indra/llcommon/llassettype.h b/indra/llcommon/llassettype.h index 652c548d59..1027957863 100644 --- a/indra/llcommon/llassettype.h +++ b/indra/llcommon/llassettype.h @@ -127,8 +127,10 @@ public: AT_RESERVED_6 = 55, AT_SETTINGS = 56, // Collection of settings + + AT_MATERIAL = 57, - AT_COUNT = 57, + AT_COUNT = 58, // +*********************************************************+ // | TO ADD AN ELEMENT TO THIS ENUM: | -- cgit v1.2.3 From c9ef206e39063c46c1fbab355c1a015e3e8b022e Mon Sep 17 00:00:00 2001 From: Brad Kittenbrink Date: Thu, 28 Apr 2022 13:08:37 -0700 Subject: Beginning viewer side work for SL-17198 new asset and inventory types for Materials --- indra/llcommon/llassettype.cpp | 1 + indra/llcommon/llassettype.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index e6cc06e8d0..0bb1f1a0fd 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -96,6 +96,7 @@ LLAssetDictionary::LLAssetDictionary() addEntry(LLAssetType::AT_WIDGET, new AssetEntry("WIDGET", "widget", "widget", false, false, false)); addEntry(LLAssetType::AT_PERSON, new AssetEntry("PERSON", "person", "person", false, false, false)); addEntry(LLAssetType::AT_SETTINGS, new AssetEntry("SETTINGS", "settings", "settings blob", true, true, true)); + addEntry(LLAssetType::AT_MATERIAL, new AssetEntry("MATERIAL", "material", "render material", true, true, true)); addEntry(LLAssetType::AT_UNKNOWN, new AssetEntry("UNKNOWN", "invalid", NULL, false, false, false)); addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, FALSE, FALSE, FALSE)); diff --git a/indra/llcommon/llassettype.h b/indra/llcommon/llassettype.h index 652c548d59..9598f0f707 100644 --- a/indra/llcommon/llassettype.h +++ b/indra/llcommon/llassettype.h @@ -127,8 +127,9 @@ public: AT_RESERVED_6 = 55, AT_SETTINGS = 56, // Collection of settings + AT_MATERIAL = 57, // Render Material - AT_COUNT = 57, + AT_COUNT = 58, // +*********************************************************+ // | TO ADD AN ELEMENT TO THIS ENUM: | -- cgit v1.2.3 From 72ddfbd76ef3152c86e9b0b4331919d15d6a3d2a Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 10 May 2022 12:18:27 -0400 Subject: SL-17219: WorkQueue::tryPost() must call ThreadSafeSchedule::tryPush(). We inadvertently changed tryPost() to call ThreadSafeSchedule::tryPost(), which doesn't exist. --- indra/llcommon/workqueue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 46f7363830..784327f929 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -170,7 +170,7 @@ namespace LL template bool tryPost(CALLABLE&& callable) { - return mQueue.tryPost(TimePoint::clock::now(), std::move(callable)); + return mQueue.tryPush(TimePoint::clock::now(), std::move(callable)); } /*------------------------- handshake API --------------------------*/ -- cgit v1.2.3 From 6c6d9a10f830e264cf75603949b54a12256cab78 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Fri, 20 May 2022 13:31:18 -0500 Subject: SL-17287 Update Tracy to 0.8.1. Clean up GPU instrumentation. --- indra/llcommon/llframetimer.cpp | 6 +++--- indra/llcommon/llprofiler.h | 35 +++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 11 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llframetimer.cpp b/indra/llcommon/llframetimer.cpp index c54029e8b4..58af32f1af 100644 --- a/indra/llcommon/llframetimer.cpp +++ b/indra/llcommon/llframetimer.cpp @@ -30,9 +30,9 @@ #include "llframetimer.h" // We don't bother building a stand alone lib; we just need to include the one source file for Tracy support -#if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY || LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY_FAST_TIMER - #include "TracyClient.cpp" -#endif // LL_PROFILER_CONFIGURATION +//#if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY || LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY_FAST_TIMER +// #include "TracyClient.cpp" +//#endif // LL_PROFILER_CONFIGURATION // Static members //LLTimer LLFrameTimer::sInternalTimer; diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index f9d7ae7ce4..bc46128d21 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -86,8 +86,8 @@ extern thread_local bool gProfilerEnabled; #define TRACY_ONLY_IPV4 1 #include "Tracy.hpp" - // Mutually exclusive with detailed memory tracing - #define LL_PROFILER_ENABLE_TRACY_OPENGL 0 + // Disable memory tracing when enabled, but enabled + #define LL_PROFILER_ENABLE_TRACY_OPENGL 1 #endif #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY @@ -104,8 +104,6 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILE_ZONE_ERR(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0XFF0000 ) // RGB yellow #define LL_PROFILE_ZONE_INFO(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0X00FFFF ) // RGB cyan #define LL_PROFILE_ZONE_WARN(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0x0FFFF00 ) // RGB red - #define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) - #define LL_PROFILE_FREE(ptr) TracyFree(ptr) #endif #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_FAST_TIMER #define LL_PROFILER_FRAME_END @@ -121,8 +119,6 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILE_ZONE_ERR(name) (void)(name); // Not supported #define LL_PROFILE_ZONE_INFO(name) (void)(name); // Not supported #define LL_PROFILE_ZONE_WARN(name) (void)(name); // Not supported - #define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); - #define LL_PROFILE_FREE(ptr) (void)(ptr); #endif #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY_FAST_TIMER #define LL_PROFILER_FRAME_END FrameMark @@ -138,14 +134,37 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILE_ZONE_ERR(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0XFF0000 ) // RGB yellow #define LL_PROFILE_ZONE_INFO(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0X00FFFF ) // RGB cyan #define LL_PROFILE_ZONE_WARN(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0x0FFFF00 ) // RGB red - #define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) - #define LL_PROFILE_FREE(ptr) TracyFree(ptr) #endif #else #define LL_PROFILER_FRAME_END #define LL_PROFILER_SET_THREAD_NAME( name ) (void)(name) #endif // LL_PROFILER +#if LL_PROFILER_ENABLE_TRACY_OPENGL +#define LL_PROFILE_GPU_ZONE(name) TracyGpuZone(name) +#define LL_PROFILE_GPU_ZONEC(name,color) TracyGpuZoneC(name,color) +#define LL_PROFILER_GPU_COLLECT TracyGpuCollect +#define LL_PROFILER_GPU_CONTEXT TracyGpuContext + +// disable memory tracking (incompatible with GPU tracing +#define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); +#define LL_PROFILE_FREE(ptr) (void)(ptr); +#else +#define LL_PROFILE_GPU_ZONE(name) (void)name; +#define LL_PROFILE_GPU_ZONEC(name,color) (void)name;(void)color; +#define LL_PROFILER_GPU_COLLECT +#define LL_PROFILER_GPU_CONTEXT + +#if LL_PROFILER_CONFIG > 1 +#define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) +#define LL_PROFILE_FREE(ptr) TracyFree(ptr) +#else +#define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); +#define LL_PROFILE_FREE(ptr) (void)(ptr); +#endif + +#endif + #include "llprofilercategories.h" #endif // LL_PROFILER_H -- cgit v1.2.3 From 9bcc01e3e33c9152a0fb71de2b33e3e41b5a5534 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Tue, 24 May 2022 14:30:30 -0500 Subject: SL-17484 Build fix. --- indra/llcommon/lltimer.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index b250bc3e1c..466f98f9b2 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -30,6 +30,9 @@ #include "u64.h" +#include +#include + #if LL_WINDOWS # include "llwin32headerslean.h" #elif LL_LINUX || LL_DARWIN -- cgit v1.2.3 From 8c5697a2d6b3976979d22da5c16569ef1c5aa9fd Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Tue, 31 May 2022 08:33:18 -0500 Subject: SL-17490 Update Tracy --- indra/llcommon/llframetimer.cpp | 5 ----- indra/llcommon/llprofiler.h | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llframetimer.cpp b/indra/llcommon/llframetimer.cpp index c54029e8b4..1e9920746b 100644 --- a/indra/llcommon/llframetimer.cpp +++ b/indra/llcommon/llframetimer.cpp @@ -29,11 +29,6 @@ #include "llframetimer.h" -// We don't bother building a stand alone lib; we just need to include the one source file for Tracy support -#if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY || LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY_FAST_TIMER - #include "TracyClient.cpp" -#endif // LL_PROFILER_CONFIGURATION - // Static members //LLTimer LLFrameTimer::sInternalTimer; U64 LLFrameTimer::sStartTotalTime = totalTime(); diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index f9d7ae7ce4..7a593092da 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -138,8 +138,10 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILE_ZONE_ERR(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0XFF0000 ) // RGB yellow #define LL_PROFILE_ZONE_INFO(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0X00FFFF ) // RGB cyan #define LL_PROFILE_ZONE_WARN(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0x0FFFF00 ) // RGB red - #define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) - #define LL_PROFILE_FREE(ptr) TracyFree(ptr) + //#define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) // memory allocation tracking currently not working + //#define LL_PROFILE_FREE(ptr) TracyFree(ptr) + #define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); + #define LL_PROFILE_FREE(ptr) (void)(ptr); #endif #else #define LL_PROFILER_FRAME_END -- cgit v1.2.3 From fc7b5549cb6092194d11b8d87600f21992305c1c Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Tue, 31 May 2022 16:54:05 -0500 Subject: SL-17484 Fix for unit tests. Deprecate non-threaded LLQueuedThread and make lllfsthread threaded. --- indra/llcommon/llqueuedthread.cpp | 23 ++++++++++++++--------- indra/llcommon/lltimer.cpp | 7 ++++++- 2 files changed, 20 insertions(+), 10 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index 871c42f7ee..60304fff75 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -37,12 +37,13 @@ // MAIN THREAD LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded, bool should_pause) : LLThread(name), - mThreaded(threaded), mIdleThread(TRUE), mNextHandle(0), mStarted(FALSE), + mThreaded(threaded), mRequestQueue(name, 1024 * 1024) { + llassert(threaded); // not threaded implementation is deprecated mMainQueue = LL::WorkQueue::getInstance("mainloop"); if (mThreaded) @@ -138,14 +139,18 @@ S32 LLQueuedThread::updateQueue(F32 max_time_ms) if (mThreaded) { // schedule a call to threadedUpdate for every call to updateQueue - mRequestQueue.post([=]() - { - LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qt - update"); - mIdleThread = FALSE; - threadedUpdate(); - mIdleThread = TRUE; - } - ); + if (!isQuitting()) + { + mRequestQueue.post([=]() + { + LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qt - update"); + mIdleThread = FALSE; + threadedUpdate(); + mIdleThread = TRUE; + } + ); + } + if(getPending() > 0) { unpause(); diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index 466f98f9b2..39dfee3755 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -68,7 +68,12 @@ LLTimer* LLTimer::sTimer = NULL; void ms_sleep(U32 ms) { LL_PROFILE_ZONE_SCOPED; - std::this_thread::sleep_for(std::chrono::microseconds(ms)); + using TimePoint = std::chrono::steady_clock::time_point; + auto resume_time = TimePoint::clock::now() + std::chrono::milliseconds(ms); + while (TimePoint::clock::now() < resume_time) + { + std::this_thread::yield(); //note: don't use LLThread::yield here to avoid yielding for too long + } } U32 micro_sleep(U64 us, U32 max_yields) -- cgit v1.2.3 From 609476e607b18d303afa5f5b5eeabeca84f95d16 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Wed, 1 Jun 2022 09:25:16 -0500 Subject: SL-17484 More unit test pruning. Fix for crash when deleting textures. --- indra/llcommon/llqueuedthread.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index 60304fff75..155e32ebae 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -75,6 +75,11 @@ void LLQueuedThread::shutdown() unpause(); // MAIN THREAD if (mThreaded) { + if (mRequestQueue.size() == 0) + { + mRequestQueue.close(); + } + S32 timeout = 100; for ( ; timeout>0; timeout--) { -- cgit v1.2.3 From ef87eb7fa80a72b94d67d5ab680f60a837dd1ddd Mon Sep 17 00:00:00 2001 From: Cosmic Linden Date: Tue, 31 May 2022 12:49:53 -0700 Subject: SL-17483: Make ThreadPool inherit LLInstanceTracker (cherry picked from commit 41d6a0e222241606c317281e2f0b211e16813dd5) --- indra/llcommon/threadpool.cpp | 1 + indra/llcommon/threadpool.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index ba914035e2..d5adf11264 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -22,6 +22,7 @@ #include "stringize.h" LL::ThreadPool::ThreadPool(const std::string& name, size_t threads, size_t capacity): + super(name), mQueue(name, capacity), mName("ThreadPool:" + name), mThreadCount(threads) diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h index b79c9b9090..f8eec3b457 100644 --- a/indra/llcommon/threadpool.h +++ b/indra/llcommon/threadpool.h @@ -22,8 +22,10 @@ namespace LL { - class ThreadPool + class ThreadPool: public LLInstanceTracker { + private: + using super = LLInstanceTracker; public: /** * Pass ThreadPool a string name. This can be used to look up the -- cgit v1.2.3 From ac99e979f43d49402a24b2f58a154c4ef1583efd Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 9 Jun 2022 10:18:29 -0400 Subject: SL-17483: Make it possible to override width of any ThreadPool. Introduce CommonControl, which in a running viewer (or any program containing an LLViewerControlListener instance) gives access to LLViewerControl functionality, e.g. getting, setting or enumerating control variables -- without introducing a link dependency on newview. Make ThreadPool's constructor consult CommonControl to check for an override for the width of the new ThreadPool in the Global (i.e. gSavedSettings) setting ThreadPoolSizes, and honor that if found. Introduce static ThreadPool methods getConfiguredWidth(), to query for such an override on any particular ThreadPool name; and getWidth(), to ask for the width of an instance if that instance already exists, else the width with which it *would* be instantiated. --- indra/llcommon/CMakeLists.txt | 2 + indra/llcommon/commoncontrol.cpp | 104 +++++++++++++++++++++++++++++++++++++++ indra/llcommon/commoncontrol.h | 75 ++++++++++++++++++++++++++++ indra/llcommon/threadpool.cpp | 51 ++++++++++++++++++- indra/llcommon/threadpool.h | 30 ++++++++++- 5 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 indra/llcommon/commoncontrol.cpp create mode 100644 indra/llcommon/commoncontrol.h (limited to 'indra/llcommon') diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index ca8b5e946f..4dbf1282c4 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -29,6 +29,7 @@ include_directories( # ${LLCOMMON_LIBRARIES}) set(llcommon_SOURCE_FILES + commoncontrol.cpp indra_constants.cpp llallocator.cpp llallocator_heap_profile.cpp @@ -129,6 +130,7 @@ set(llcommon_HEADER_FILES CMakeLists.txt chrono.h + commoncontrol.h ctype_workaround.h fix_macros.h indra_constants.h diff --git a/indra/llcommon/commoncontrol.cpp b/indra/llcommon/commoncontrol.cpp new file mode 100644 index 0000000000..2c2a2abeb0 --- /dev/null +++ b/indra/llcommon/commoncontrol.cpp @@ -0,0 +1,104 @@ +/** + * @file commoncontrol.cpp + * @author Nat Goodspeed + * @date 2022-06-08 + * @brief Implementation for commoncontrol. + * + * $LicenseInfo:firstyear=2022&license=viewerlgpl$ + * Copyright (c) 2022, Linden Research, Inc. + * $/LicenseInfo$ + */ + +// Precompiled header +#include "linden_common.h" +// associated header +#include "commoncontrol.h" +// STL headers +// std headers +// external library headers +// other Linden headers +#include "llevents.h" +#include "llsdutil.h" + +LLSD LL::CommonControl::access(const LLSD& params) +{ + // We can't actually introduce a link-time dependency on llxml, or on any + // global LLControlGroup (*koff* gSavedSettings *koff*) but we can issue a + // runtime query. If we're running as part of a viewer with + // LLViewerControlListener, we can use that to interact with any + // instantiated LLControGroup. + LLSD response; + { + LLEventStream reply("reply"); + LLTempBoundListener connection = reply.listen("listener", + [&response] (const LLSD& event) + { + response = event; + return false; + }); + LLSD rparams{ params }; + rparams["reply"] = reply.getName(); + LLEventPumps::instance().obtain("LLViewerControl").post(rparams); + } + // LLViewerControlListener responds immediately. If it's listening at all, + // it will already have set response. + if (! response.isDefined()) + { + LLTHROW(NoListener("No LLViewerControl listener instantiated")); + } + LLSD error{ response["error"] }; + if (error.isDefined()) + { + LLTHROW(ParamError(error)); + } + return response; +} + +/// set control group.key to defined default value +LLSD LL::CommonControl::set_default(const std::string& group, const std::string& key) +{ + return access(llsd::map("op", "set", + "group", group, "key", key))["value"]; +} + +/// set control group.key to specified value +LLSD LL::CommonControl::set(const std::string& group, const std::string& key, const LLSD& value) +{ + return access(llsd::map("op", "set", + "group", group, "key", key, "value", value))["value"]; +} + +/// toggle boolean control group.key +LLSD LL::CommonControl::toggle(const std::string& group, const std::string& key) +{ + return access(llsd::map("op", "toggle", + "group", group, "key", key))["value"]; +} + +/// get the definition for control group.key, (! isDefined()) if bad +/// ["name"], ["type"], ["value"], ["comment"] +LLSD LL::CommonControl::get_def(const std::string& group, const std::string& key) +{ + return access(llsd::map("op", "get", + "group", group, "key", key)); +} + +/// get the value of control group.key +LLSD LL::CommonControl::get(const std::string& group, const std::string& key) +{ + return access(llsd::map("op", "get", + "group", group, "key", key))["value"]; +} + +/// get defined groups +std::vector LL::CommonControl::get_groups() +{ + auto groups{ access(llsd::map("op", "groups"))["groups"] }; + return { groups.beginArray(), groups.endArray() }; +} + +/// get definitions for all variables in group +LLSD LL::CommonControl::get_vars(const std::string& group) +{ + return access(llsd::map("op", "vars", "group", group))["vars"]; +} diff --git a/indra/llcommon/commoncontrol.h b/indra/llcommon/commoncontrol.h new file mode 100644 index 0000000000..07d4a45ac5 --- /dev/null +++ b/indra/llcommon/commoncontrol.h @@ -0,0 +1,75 @@ +/** + * @file commoncontrol.h + * @author Nat Goodspeed + * @date 2022-06-08 + * @brief Access LLViewerControl LLEventAPI, if process has one. + * + * $LicenseInfo:firstyear=2022&license=viewerlgpl$ + * Copyright (c) 2022, Linden Research, Inc. + * $/LicenseInfo$ + */ + +#if ! defined(LL_COMMONCONTROL_H) +#define LL_COMMONCONTROL_H + +#include +#include "llexception.h" +#include "llsd.h" + +namespace LL +{ + class CommonControl + { + public: + struct Error: public LLException + { + Error(const std::string& what): LLException(what) {} + }; + + /// Exception thrown if there's no LLViewerControl LLEventAPI + struct NoListener: public Error + { + NoListener(const std::string& what): Error(what) {} + }; + + struct ParamError: public Error + { + ParamError(const std::string& what): Error(what) {} + }; + + /// set control group.key to defined default value + static + LLSD set_default(const std::string& group, const std::string& key); + + /// set control group.key to specified value + static + LLSD set(const std::string& group, const std::string& key, const LLSD& value); + + /// toggle boolean control group.key + static + LLSD toggle(const std::string& group, const std::string& key); + + /// get the definition for control group.key, (! isDefined()) if bad + /// ["name"], ["type"], ["value"], ["comment"] + static + LLSD get_def(const std::string& group, const std::string& key); + + /// get the value of control group.key + static + LLSD get(const std::string& group, const std::string& key); + + /// get defined groups + static + std::vector get_groups(); + + /// get definitions for all variables in group + static + LLSD get_vars(const std::string& group); + + private: + static + LLSD access(const LLSD& params); + }; +} // namespace LL + +#endif /* ! defined(LL_COMMONCONTROL_H) */ diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index d5adf11264..10d67abf37 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -17,15 +17,17 @@ // std headers // external library headers // other Linden headers +#include "commoncontrol.h" #include "llerror.h" #include "llevents.h" +#include "llsd.h" #include "stringize.h" LL::ThreadPool::ThreadPool(const std::string& name, size_t threads, size_t capacity): super(name), mQueue(name, capacity), mName("ThreadPool:" + name), - mThreadCount(threads) + mThreadCount(getConfiguredWidth(name, threads)) {} void LL::ThreadPool::start() @@ -87,3 +89,50 @@ void LL::ThreadPool::run() { mQueue.runUntilClose(); } + +//static +size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft=0) +{ + LLSD poolSizes{ LL::CommonControl::get("Global", "ThreadPoolSizes") }; + // "ThreadPoolSizes" is actually a map containing the sizes of interest -- + // or should be, if this process has an "LLViewerControl" LLEventAPI + // instance and its settings include "ThreadPoolSizes". If we failed to + // retrieve it, perhaps we're in a program that doesn't define that, or + // perhaps there's no such setting, or perhaps we're asking too early, + // before the LLEventAPI itself has been instantiated. In any of those + // cases, it seems worth warning. + if (! poolSizes.isDefined()) + { + // Note: we don't warn about absence of an override key for a + // particular ThreadPool name, that's fine. This warning is about + // complete absence of a ThreadPoolSizes setting, which we expect in a + // normal viewer session. + LL_WARNS("ThreadPool") << "No 'ThreadPoolSizes' setting for ThreadPool '" + << name << "'" << LL_ENDL; + } + else + { + //LL_DEBUGS + LL_INFOS("ThreadPool") << "ThreadPoolSizes = " << poolSizes << LL_ENDL; + } + // LLSD treats an undefined value as an empty map when asked to retrieve a + // key, so we don't need this to be conditional. + LLSD sizeSpec{ poolSizes[name] }; + // We retrieve sizeSpec as LLSD, rather than immediately as LLSD::Integer, + // so we can distinguish the case when it's undefined. + return sizeSpec.isInteger() ? sizeSpec.asInteger() : dft; +} + +//static +size_t LL::ThreadPool::getWidth(const std::string& name, size_t dft) +{ + auto instance{ getInstance(name) }; + if (instance) + { + return instance->getWidth(); + } + else + { + return getConfiguredWidth(name, dft); + } +} diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h index f8eec3b457..b49d511257 100644 --- a/indra/llcommon/threadpool.h +++ b/indra/llcommon/threadpool.h @@ -30,8 +30,17 @@ namespace LL /** * Pass ThreadPool a string name. This can be used to look up the * relevant WorkQueue. + * + * The number of threads you pass sets the compile-time default. But + * if the user has overridden the LLSD map in the "ThreadPoolSizes" + * setting with a key matching this ThreadPool name, that setting + * overrides this parameter. + * + * Pass an explicit capacity to limit the size of the queue. + * Constraining the queue can cause a submitter to block. Do not + * constrain any ThreadPool accepting work from the main thread. */ - ThreadPool(const std::string& name, size_t threads=1, size_t capacity=1024); + ThreadPool(const std::string& name, size_t threads=1, size_t capacity=1024*1024); virtual ~ThreadPool(); /** @@ -59,6 +68,25 @@ namespace LL */ virtual void run(); + /** + * getConfiguredWidth() returns the setting, if any, for the specified + * ThreadPool name. Returns dft if the "ThreadPoolSizes" map does not + * contain the specified name. + */ + static + size_t getConfiguredWidth(const std::string& name, size_t dft=0); + + /** + * This getWidth() returns the width of the instantiated ThreadPool + * with the specified name, if any. If no instance exists, returns its + * getConfiguredWidth() if any. If there's no instance and no relevant + * override, return dft. Presumably dft should match the threads + * parameter passed to the ThreadPool constructor call that will + * eventually instantiate the ThreadPool with that name. + */ + static + size_t getWidth(const std::string& name, size_t dft); + private: void run(const std::string& name); -- cgit v1.2.3 From 1ff79a430d9d55cbb4b4ba55018c791a8a933916 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 9 Jun 2022 11:19:21 -0400 Subject: SL-17483: Fix ThreadPool::getConfiguredWidth() compile error. Log ThreadPoolSizes at DEBUG level, not INFO. --- indra/llcommon/threadpool.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index 10d67abf37..e8daf549ef 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -91,7 +91,7 @@ void LL::ThreadPool::run() } //static -size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft=0) +size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft) { LLSD poolSizes{ LL::CommonControl::get("Global", "ThreadPoolSizes") }; // "ThreadPoolSizes" is actually a map containing the sizes of interest -- @@ -112,8 +112,7 @@ size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft=0) } else { - //LL_DEBUGS - LL_INFOS("ThreadPool") << "ThreadPoolSizes = " << poolSizes << LL_ENDL; + LL_DEBUGS("ThreadPool") << "ThreadPoolSizes = " << poolSizes << LL_ENDL; } // LLSD treats an undefined value as an empty map when asked to retrieve a // key, so we don't need this to be conditional. -- cgit v1.2.3 From 64209ddeeafd944f82da6f13a6e790f9b542b3ff Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 9 Jun 2022 17:04:26 -0400 Subject: SL-17483: Add integration test for CommonControl and for LLViewerControlListener, to which it talks. Fix glitches detected by the tests. --- indra/llcommon/commoncontrol.cpp | 2 ++ indra/llcommon/threadpool.cpp | 43 ++++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 17 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/commoncontrol.cpp b/indra/llcommon/commoncontrol.cpp index 2c2a2abeb0..81e66baf8c 100644 --- a/indra/llcommon/commoncontrol.cpp +++ b/indra/llcommon/commoncontrol.cpp @@ -51,6 +51,8 @@ LLSD LL::CommonControl::access(const LLSD& params) { LLTHROW(ParamError(error)); } + response.erase("error"); + response.erase("reqid"); return response; } diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index e8daf549ef..f49dd40a8b 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -93,27 +93,36 @@ void LL::ThreadPool::run() //static size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft) { - LLSD poolSizes{ LL::CommonControl::get("Global", "ThreadPoolSizes") }; - // "ThreadPoolSizes" is actually a map containing the sizes of interest -- - // or should be, if this process has an "LLViewerControl" LLEventAPI - // instance and its settings include "ThreadPoolSizes". If we failed to - // retrieve it, perhaps we're in a program that doesn't define that, or - // perhaps there's no such setting, or perhaps we're asking too early, - // before the LLEventAPI itself has been instantiated. In any of those - // cases, it seems worth warning. - if (! poolSizes.isDefined()) + LLSD poolSizes; + try { - // Note: we don't warn about absence of an override key for a - // particular ThreadPool name, that's fine. This warning is about - // complete absence of a ThreadPoolSizes setting, which we expect in a - // normal viewer session. - LL_WARNS("ThreadPool") << "No 'ThreadPoolSizes' setting for ThreadPool '" - << name << "'" << LL_ENDL; + poolSizes = LL::CommonControl::get("Global", "ThreadPoolSizes"); + // "ThreadPoolSizes" is actually a map containing the sizes of + // interest -- or should be, if this process has an + // LLViewerControlListener instance and its settings include + // "ThreadPoolSizes". If we failed to retrieve it, perhaps we're in a + // program that doesn't define that, or perhaps there's no such + // setting, or perhaps we're asking too early, before the LLEventAPI + // itself has been instantiated. In any of those cases, it seems worth + // warning. + if (! poolSizes.isDefined()) + { + // Note: we don't warn about absence of an override key for a + // particular ThreadPool name, that's fine. This warning is about + // complete absence of a ThreadPoolSizes setting, which we expect + // in a normal viewer session. + LL_WARNS("ThreadPool") << "No 'ThreadPoolSizes' setting for ThreadPool '" + << name << "'" << LL_ENDL; + } } - else + catch (const LL::CommonControl::Error& exc) { - LL_DEBUGS("ThreadPool") << "ThreadPoolSizes = " << poolSizes << LL_ENDL; + // We don't want ThreadPool to *require* LLViewerControlListener. + // Just log it and carry on. + LL_WARNS("ThreadPool") << "Can't check 'ThreadPoolSizes': " << exc.what() << LL_ENDL; } + + LL_DEBUGS("ThreadPool") << "ThreadPoolSizes = " << poolSizes << LL_ENDL; // LLSD treats an undefined value as an empty map when asked to retrieve a // key, so we don't need this to be conditional. LLSD sizeSpec{ poolSizes[name] }; -- cgit v1.2.3 From bdd76a6a1d0ee0fa5a8da76f1e8c8583822eeabe Mon Sep 17 00:00:00 2001 From: Howard Stearns Date: Mon, 13 Jun 2022 13:59:07 -0700 Subject: SL-17485 - Provide corrected missing memory data on mac: sAllocatedPageSizeInKB, sAllocatedMemInKB, sAvailPhysicalMemInKB, sMaxPhysicalMemInKB --- indra/llcommon/llmemory.cpp | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 849867586a..267159e7cc 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -35,6 +35,7 @@ # include # include # include +#include #elif LL_LINUX # include #endif @@ -46,6 +47,9 @@ #include "lltrace.h" #include "llerror.h" //---------------------------------------------------------------------------- +#if defined(LL_DARWIN) +extern LLMemoryInfo gSysMemory; +#endif //static U32Kilobytes LLMemory::sAvailPhysicalMemInKB(U32_MAX); @@ -109,6 +113,51 @@ void LLMemory::updateMemoryInfo() { sAvailPhysicalMemInKB = U32Kilobytes(0); } + +#elif defined(LL_DARWIN) + task_vm_info info; + mach_msg_type_number_t infoCount = TASK_VM_INFO_COUNT; + // MACH_TASK_BASIC_INFO reports the same resident_size, but does not tell us the reusable bytes or phys_footprint. + if (task_info(mach_task_self(), TASK_VM_INFO, reinterpret_cast(&info), &infoCount) == KERN_SUCCESS) + { + // Our Windows definition of PagefileUsage is documented by Microsoft as "the total amount of + // memory that the memory manager has committed for a running process", which is rss. + sAllocatedPageSizeInKB = U32Bytes(info.resident_size); + + // Activity Monitor => Inspect Process => Real Memory Size appears to report resident_size + // Activity monitor => main window memory column appears to report phys_footprint, which spot checks as at least 30% less. + // I think that is because of compression, which isn't going to give us a consistent measurement. We want uncompressed totals. + // + // In between is resident_size - reusable. This is what Chrome source code uses, with source comments saying it is 'the "Real Memory" value + // reported for the app by the Memory Monitor in Instruments.' It is still about 8% bigger than phys_footprint. + // + // (On Windows, we use WorkingSetSize.) + sAllocatedMemInKB = U32Bytes(info.resident_size - info.reusable); + } + else + { + LL_WARNS() << "task_info failed" << LL_ENDL; + } + + // Total installed and available physical memory are properties of the host, not just our process. + vm_statistics64_data_t vmstat; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + mach_port_t host = mach_host_self(); + vm_size_t page_size; + host_page_size(host, &page_size); + kern_return_t result = host_statistics64(host, HOST_VM_INFO64, reinterpret_cast(&vmstat), &count); + if (result == KERN_SUCCESS) { + // This is what Chrome reports as 'the "Physical Memory Free" value reported by the Memory Monitor in Instruments.' + // Note though that inactive pages are not included here and not yet free, but could become so under memory pressure. + sAvailPhysicalMemInKB = U32Bytes(vmstat.free_count * page_size); + + sMaxPhysicalMemInKB = gSysMemory.getPhysicalMemoryKB(); + } + else + { + LL_WARNS() << "task_info failed" << LL_ENDL; + } + #else //not valid for other systems for now. sAllocatedMemInKB = U64Bytes(LLMemory::getCurrentRSS()); -- cgit v1.2.3 From 33fe18c335375ecdaf57ddd0138fc7a6cb227843 Mon Sep 17 00:00:00 2001 From: Howard Stearns Date: Fri, 17 Jun 2022 16:29:39 -0700 Subject: SL-17485-b - Attempt to make teamcity builds happy by not referencing a newview global from an llcommon file. --- indra/llcommon/llmemory.cpp | 6 +----- indra/llcommon/llsys.cpp | 24 ++++++++++++++++-------- indra/llcommon/llsys.h | 5 ++++- 3 files changed, 21 insertions(+), 14 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 267159e7cc..749b66b472 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -47,9 +47,6 @@ #include "lltrace.h" #include "llerror.h" //---------------------------------------------------------------------------- -#if defined(LL_DARWIN) -extern LLMemoryInfo gSysMemory; -#endif //static U32Kilobytes LLMemory::sAvailPhysicalMemInKB(U32_MAX); @@ -150,8 +147,7 @@ void LLMemory::updateMemoryInfo() // This is what Chrome reports as 'the "Physical Memory Free" value reported by the Memory Monitor in Instruments.' // Note though that inactive pages are not included here and not yet free, but could become so under memory pressure. sAvailPhysicalMemInKB = U32Bytes(vmstat.free_count * page_size); - - sMaxPhysicalMemInKB = gSysMemory.getPhysicalMemoryKB(); + sMaxPhysicalMemInKB = LLMemoryInfo::getHardwareMemSize(); } else { diff --git a/indra/llcommon/llsys.cpp b/indra/llcommon/llsys.cpp index f717b2cf34..aeb6ae1b0a 100644 --- a/indra/llcommon/llsys.cpp +++ b/indra/llcommon/llsys.cpp @@ -684,20 +684,28 @@ static U32Kilobytes LLMemoryAdjustKBResult(U32Kilobytes inKB) } #endif +#if LL_DARWIN +// static +U32Kilobytes LLMemoryInfo::getHardwareMemSize() +{ + // This might work on Linux as well. Someone check... + uint64_t phys = 0; + int mib[2] = { CTL_HW, HW_MEMSIZE }; + + size_t len = sizeof(phys); + sysctl(mib, 2, &phys, &len, NULL, 0); + + return U64Bytes(phys); +} +#endif + U32Kilobytes LLMemoryInfo::getPhysicalMemoryKB() const { #if LL_WINDOWS return LLMemoryAdjustKBResult(U32Kilobytes(mStatsMap["Total Physical KB"].asInteger())); #elif LL_DARWIN - // This might work on Linux as well. Someone check... - uint64_t phys = 0; - int mib[2] = { CTL_HW, HW_MEMSIZE }; - - size_t len = sizeof(phys); - sysctl(mib, 2, &phys, &len, NULL, 0); - - return U64Bytes(phys); + return getHardwareMemSize(); #elif LL_LINUX U64 phys = 0; diff --git a/indra/llcommon/llsys.h b/indra/llcommon/llsys.h index 5ab97939b9..b2f9b6a4ab 100644 --- a/indra/llcommon/llsys.h +++ b/indra/llcommon/llsys.h @@ -113,7 +113,10 @@ public: LLMemoryInfo(); ///< Default constructor void stream(std::ostream& s) const; ///< output text info to s - U32Kilobytes getPhysicalMemoryKB() const; + U32Kilobytes getPhysicalMemoryKB() const; +#if LL_DARWIN + static U32Kilobytes getHardwareMemSize(); // Because some Mac linkers won't let us reference extern gSysMemory from a different lib. +#endif //get the available memory infomation in KiloBytes. static void getAvailableMemoryKB(U32Kilobytes& avail_physical_mem_kb, U32Kilobytes& avail_virtual_mem_kb); -- cgit v1.2.3 From 1559ad47924711f64ecd540bc605d9aa1f7221e6 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Fri, 24 Jun 2022 13:09:24 +0300 Subject: SL-17649 Icon for Material type in Inventory --- indra/llcommon/llassettype.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index 0bb1f1a0fd..f08cc18036 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -96,7 +96,7 @@ LLAssetDictionary::LLAssetDictionary() addEntry(LLAssetType::AT_WIDGET, new AssetEntry("WIDGET", "widget", "widget", false, false, false)); addEntry(LLAssetType::AT_PERSON, new AssetEntry("PERSON", "person", "person", false, false, false)); addEntry(LLAssetType::AT_SETTINGS, new AssetEntry("SETTINGS", "settings", "settings blob", true, true, true)); - addEntry(LLAssetType::AT_MATERIAL, new AssetEntry("MATERIAL", "material", "render material", true, true, true)); + addEntry(LLAssetType::AT_MATERIAL, new AssetEntry("MATERIAL", "material", "render material", true, true, true)); addEntry(LLAssetType::AT_UNKNOWN, new AssetEntry("UNKNOWN", "invalid", NULL, false, false, false)); addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, FALSE, FALSE, FALSE)); -- cgit v1.2.3 From 2c692f635da67990f842f20adf3b42d870d42fdf Mon Sep 17 00:00:00 2001 From: "Brad Payne (Vir Linden)" Date: Thu, 1 Sep 2022 15:11:56 +0100 Subject: DRTVWR-559 refcount max change --- indra/llcommon/llrefcount.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llrefcount.cpp b/indra/llcommon/llrefcount.cpp index 5cbd346411..6852b5536a 100644 --- a/indra/llcommon/llrefcount.cpp +++ b/indra/llcommon/llrefcount.cpp @@ -30,7 +30,7 @@ #include "llerror.h" // maximum reference count before sounding memory leak alarm -const S32 gMaxRefCount = 65536; +const S32 gMaxRefCount = S32_MAX; LLRefCount::LLRefCount(const LLRefCount& other) : mRef(0) -- cgit v1.2.3 From e0c226b04d6f2ac566a9ea5841509f7cdfa98c11 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Thu, 1 Sep 2022 18:06:15 -0500 Subject: SL-18078, SL-18065 -- Experimentally allow uploading of lossless normal maps, fix for crash on shutdown. --- indra/llcommon/llqueuedthread.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index 155e32ebae..e5060a1076 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -146,7 +146,7 @@ S32 LLQueuedThread::updateQueue(F32 max_time_ms) // schedule a call to threadedUpdate for every call to updateQueue if (!isQuitting()) { - mRequestQueue.post([=]() + mRequestQueue.postIfOpen([=]() { LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qt - update"); mIdleThread = FALSE; -- cgit v1.2.3 From abf788175c451046f82cc9aeaddc894c47863ffa Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Tue, 6 Sep 2022 15:07:22 -0500 Subject: SL-18096 WIP -- partial support for double sided rendering. Shadow map and picking support TBD. --- indra/llcommon/llworkerthread.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index 02ce4823b8..bd2eb2089c 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -97,6 +97,7 @@ S32 LLWorkerThread::update(F32 max_time_ms) { if (worker->getFlags(LLWorkerClass::WCF_WORK_FINISHED)) { + worker->setFlags(LLWorkerClass::WCF_DELETE_REQUESTED); delete_list.push_back(worker); mDeleteList.erase(curiter); } -- cgit v1.2.3 From 7d426815019ac5394e5821f1c6ba374e12606aaa Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Fri, 30 Sep 2022 10:57:01 -0500 Subject: SL-18239 Unify PBR and non-PBR treatment of ambient/SSAO/irradiance. Restore SSAO to release version. --- indra/llcommon/llworkerthread.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index bd2eb2089c..cd4d587752 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -73,6 +73,7 @@ void LLWorkerThread::clearDeleteList() { (*iter)->mRequestHandle = LLWorkerThread::nullHandle(); (*iter)->clearFlags(LLWorkerClass::WCF_HAVE_WORK); + (*iter)->clearFlags(LLWorkerClass::WCF_WORKING); delete *iter ; } mDeleteList.clear() ; -- cgit v1.2.3 From be1cdc1aaa67eca71fee8cbbc16b4c85bcbdb258 Mon Sep 17 00:00:00 2001 From: Geenz Date: Wed, 5 Oct 2022 09:48:18 -0700 Subject: Initial pass at adding KHR_debug support This still needs some work - I'm not super satisfied with the overall structure of the code. Will continue to iterate as I add in proper RenderDoc support. --- indra/llcommon/llprofiler.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index c0f5868db3..ac20209062 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -153,12 +153,16 @@ extern thread_local bool gProfilerEnabled; // disable memory tracking (incompatible with GPU tracing #define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); #define LL_PROFILE_FREE(ptr) (void)(ptr); + +#define LL_LABEL_OBJECT_GL(type, name, length, label) glObjectLabel(type, name, length, label) #else #define LL_PROFILE_GPU_ZONE(name) (void)name; #define LL_PROFILE_GPU_ZONEC(name,color) (void)name;(void)color; #define LL_PROFILER_GPU_COLLECT #define LL_PROFILER_GPU_CONTEXT +#define LL_LABEL_OBJECT_GL(type, name, length, label) + #if LL_PROFILER_CONFIG > 1 #define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) #define LL_PROFILE_FREE(ptr) TracyFree(ptr) -- cgit v1.2.3 From 8d7cde22c31a59d0503230334b1d68e858364c9a Mon Sep 17 00:00:00 2001 From: Signal Linden Date: Tue, 11 Oct 2022 15:10:04 -0700 Subject: Replace llbase with llsd module --- indra/llcommon/tests/llleap_test.cpp | 2 +- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 7ee36a9ea6..60005fc6a9 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -109,7 +109,7 @@ namespace tut "import os\n" "import sys\n" "\n" - "from llbase import llsd\n" + "import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..be7ec12094 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -1706,7 +1706,7 @@ namespace tut // scanner. import_llsd("import os.path\n" "import sys\n" - "from llbase import llsd\n") + "import llsd\n") {} ~TestPythonCompatible() {} -- cgit v1.2.3 From ec5009d70eb0fd729126b2321b9d7d6118741573 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Thu, 13 Oct 2022 19:51:40 -0500 Subject: SL-18190 Potential fix for sapping CPU when "sleeping" --- indra/llcommon/lltimer.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index 39dfee3755..8739eeef00 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -65,6 +65,9 @@ LLTimer* LLTimer::sTimer = NULL; //--------------------------------------------------------------------------- #if LL_WINDOWS + + +#if 0 void ms_sleep(U32 ms) { LL_PROFILE_ZONE_SCOPED; @@ -83,6 +86,31 @@ U32 micro_sleep(U64 us, U32 max_yields) ms_sleep((U32)(us / 1000)); return 0; } + +#else + +U32 micro_sleep(U64 us, U32 max_yields) +{ + LL_PROFILE_ZONE_SCOPED + LARGE_INTEGER ft; + ft.QuadPart = -static_cast(us * 10); // '-' using relative time + + HANDLE timer = CreateWaitableTimer(NULL, TRUE, NULL); + SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0); + WaitForSingleObject(timer, INFINITE); + CloseHandle(timer); + + return 0; +} + +void ms_sleep(U32 ms) +{ + LL_PROFILE_ZONE_SCOPED + micro_sleep(ms * 1000, 0); +} + +#endif + #elif LL_LINUX || LL_DARWIN static void _sleep_loop(struct timespec& thiswait) { -- cgit v1.2.3 From 0451d51f4da9780e2f650b67672a29b3d07cb386 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Thu, 27 Oct 2022 11:27:21 -0500 Subject: SL-18459 WIP -- fix for assert in setGLTFMaterial --- indra/llcommon/llpointer.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llpointer.h b/indra/llcommon/llpointer.h index 9a6453ea48..f9de0c7929 100644 --- a/indra/llcommon/llpointer.h +++ b/indra/llcommon/llpointer.h @@ -340,4 +340,28 @@ private: bool mStayUnique; }; + +// boost hash adapter +template +struct boost::hash> +{ + typedef LLPointer argument_type; + typedef std::size_t result_type; + result_type operator()(argument_type const& s) const + { + return (std::size_t) s.get(); + } +}; + +// Adapt boost hash to std hash +namespace std +{ + template struct hash> + { + std::size_t operator()(LLPointer const& s) const noexcept + { + return boost::hash>()(s); + } + }; +} #endif -- cgit v1.2.3 From 8d2ac419b22c8c9475f2efb312dd198ac8eb9fb7 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Wed, 16 Nov 2022 14:49:17 -0600 Subject: SL-18154 Profile guided optimizations vs release viewer. Trim some unused abilities and remove some more fast timers. --- indra/llcommon/llprofiler.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index ac20209062..c03b2e93f4 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -86,8 +86,12 @@ extern thread_local bool gProfilerEnabled; #define TRACY_ONLY_IPV4 1 #include "Tracy.hpp" - // Disable memory tracing when enabled, but enabled + // Enable OpenGL profiling #define LL_PROFILER_ENABLE_TRACY_OPENGL 1 + + // Enable RenderDoc labeling + #define LL_PROFILER_ENABLE_RENDER_DOC 0 + #endif #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY @@ -153,8 +157,6 @@ extern thread_local bool gProfilerEnabled; // disable memory tracking (incompatible with GPU tracing #define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); #define LL_PROFILE_FREE(ptr) (void)(ptr); - -#define LL_LABEL_OBJECT_GL(type, name, length, label) glObjectLabel(type, name, length, label) #else #define LL_PROFILE_GPU_ZONE(name) (void)name; #define LL_PROFILE_GPU_ZONEC(name,color) (void)name;(void)color; @@ -173,6 +175,12 @@ extern thread_local bool gProfilerEnabled; #endif +#if LL_PROFILER_ENABLE_RENDER_DOC +#define LL_LABEL_OBJECT_GL(type, name, length, label) glObjectLabel(type, name, length, label) +#else +#define LL_LABEL_OBJECT_GL(type, name, length, label) +#endif + #include "llprofilercategories.h" #endif // LL_PROFILER_H -- cgit v1.2.3 From 87bb72a47a1fc64b98e498120e332de76e6a9211 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Wed, 30 Nov 2022 13:25:00 -0600 Subject: SL-18154 WIP -- CPU sampling (AMD uProf) profile guided optimizations to reduce CPU usage of background threads. --- indra/llcommon/llqueuedthread.cpp | 9 +++++++-- indra/llcommon/llthread.cpp | 10 ++++++++++ indra/llcommon/lltimer.cpp | 4 ++++ 3 files changed, 21 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index e5060a1076..9b1de2e9a5 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -477,9 +477,14 @@ void LLQueuedThread::processRequest(LLQueuedThread::QueuedRequest* req) mRequestQueue.post([=] { LL_PROFILE_ZONE_NAMED("processRequest - retry"); - while (LL::WorkQueue::TimePoint::clock::now() < retry_time) + if (LL::WorkQueue::TimePoint::clock::now() < retry_time) { - std::this_thread::yield(); //note: don't use LLThread::yield here to avoid + auto sleep_time = std::chrono::duration_cast(retry_time - LL::WorkQueue::TimePoint::clock::now()); + + if (sleep_time.count() > 0) + { + ms_sleep(sleep_time.count()); + } } processRequest(req); }); diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index a807acc56e..4eaa05c335 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -42,6 +42,7 @@ #ifdef LL_WINDOWS + const DWORD MS_VC_EXCEPTION=0x406D1388; #pragma pack(push,8) @@ -133,6 +134,15 @@ void LLThread::threadRun() { #ifdef LL_WINDOWS set_thread_name(-1, mName.c_str()); + +#if 0 // probably a bad idea, see usage of SetThreadIdealProcessor in LLWindowWin32) + HANDLE hThread = GetCurrentThread(); + if (hThread) + { + SetThreadAffinityMask(hThread, (DWORD_PTR) 0xFFFFFFFFFFFFFFFE); + } +#endif + #endif LL_PROFILER_SET_THREAD_NAME( mName.c_str() ); diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index 8739eeef00..24eaa4c1a9 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -92,6 +92,7 @@ U32 micro_sleep(U64 us, U32 max_yields) U32 micro_sleep(U64 us, U32 max_yields) { LL_PROFILE_ZONE_SCOPED +#if 0 LARGE_INTEGER ft; ft.QuadPart = -static_cast(us * 10); // '-' using relative time @@ -99,6 +100,9 @@ U32 micro_sleep(U64 us, U32 max_yields) SetWaitableTimer(timer, &ft, 0, NULL, NULL, 0); WaitForSingleObject(timer, INFINITE); CloseHandle(timer); +#else + Sleep(us / 1000); +#endif return 0; } -- cgit v1.2.3 From e98091e29859ee6a784ee6920048837f2b4536c2 Mon Sep 17 00:00:00 2001 From: RunitaiLinden Date: Wed, 7 Dec 2022 12:53:59 -0600 Subject: DRTVWR-559: Try using custom fiber scheduler for ThreadPool threads. (#30) Co-authored-by: Nat Goodspeed --- indra/llcommon/threadpool.cpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index f49dd40a8b..b0d2016be0 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -23,6 +23,37 @@ #include "llsd.h" #include "stringize.h" +#include + +/***************************************************************************** +* Custom fiber scheduler for worker threads +*****************************************************************************/ +// As of 2022-12-06, each of our worker threads only runs a single (default) +// fiber: we don't launch explicit fibers within worker threads, nor do we +// anticipate doing so. So a worker thread that's simply waiting for incoming +// tasks should really sleep a little. Override the default fiber scheduler to +// implement that. +struct sleepy_robin: public boost::fibers::algo::round_robin +{ + virtual void suspend_until( std::chrono::steady_clock::time_point const&) noexcept + { + // round_robin holds a std::condition_variable, and + // round_robin::suspend_until() calls + // std::condition_variable::wait_until(). On Windows, that call seems + // busier than it ought to be. Try just sleeping. + Sleep(1); + } + + virtual void notify() noexcept + { + // Since our Sleep() call above will wake up on its own, we need not + // take any special action to wake it. + } +}; + +/***************************************************************************** +* ThreadPool +*****************************************************************************/ LL::ThreadPool::ThreadPool(const std::string& name, size_t threads, size_t capacity): super(name), mQueue(name, capacity), @@ -80,6 +111,11 @@ void LL::ThreadPool::close() void LL::ThreadPool::run(const std::string& name) { +#if LL_WINDOWS + // Try using sleepy_robin fiber scheduler. + boost::fibers::use_scheduling_algorithm(); +#endif // LL_WINDOWS + LL_DEBUGS("ThreadPool") << name << " starting" << LL_ENDL; run(); LL_DEBUGS("ThreadPool") << name << " stopping" << LL_ENDL; -- cgit v1.2.3 From e3b34fec6962e6deda3dd9dd83bf9fa20ab594af Mon Sep 17 00:00:00 2001 From: Brad Kittenbrink Date: Thu, 8 Dec 2022 12:53:51 -0800 Subject: Fix for non-windows build of DRTVWR-559 use usleep() for sleepy_robin scheduler --- indra/llcommon/threadpool.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index b0d2016be0..1b29b8b69f 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -37,11 +37,17 @@ struct sleepy_robin: public boost::fibers::algo::round_robin { virtual void suspend_until( std::chrono::steady_clock::time_point const&) noexcept { +#if LL_WINDOWS // round_robin holds a std::condition_variable, and // round_robin::suspend_until() calls // std::condition_variable::wait_until(). On Windows, that call seems // busier than it ought to be. Try just sleeping. Sleep(1); +#else + // currently unused other than windows, but might as well have something here + // different units than Sleep(), but we actually just want to sleep for any de-minimis duration + usleep(1); +#endif } virtual void notify() noexcept -- cgit v1.2.3 From 00478b1e7671cb109771a1ad4fb40d47d15ab756 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 9 Dec 2022 13:16:39 -0500 Subject: DRTVWR-559: Introduce LLInstanceTrackerSubclass mediator class. Deriving your tracked class T from LLInstanceTracker gives you T::getInstance() et al. But what about a subclass S derived from T? S::getInstance() still delivers a pointer to T, requiring explicit downcast. And so on for other LLInstanceTracker methods. Instead, derive S from LLInstanceTrackerSubclass. This implies that S is a grandchild class of T, but it also recasts the LLInstanceTracker methods to deliver results for S rather than for T. --- indra/llcommon/CMakeLists.txt | 1 + indra/llcommon/llinstancetracker.h | 97 ++++++++++++++++++----------- indra/llcommon/llinstancetrackersubclass.h | 98 ++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 36 deletions(-) create mode 100644 indra/llcommon/llinstancetrackersubclass.h (limited to 'indra/llcommon') diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 21998f0b78..0fe61108ff 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -189,6 +189,7 @@ set(llcommon_HEADER_FILES llinitdestroyclass.h llinitparam.h llinstancetracker.h + llinstancetrackersubclass.h llkeybind.h llkeythrottle.h llleap.h diff --git a/indra/llcommon/llinstancetracker.h b/indra/llcommon/llinstancetracker.h index 02535a59e7..97f7817e74 100644 --- a/indra/llcommon/llinstancetracker.h +++ b/indra/llcommon/llinstancetracker.h @@ -104,22 +104,26 @@ public: return LockStatic()->mMap.size(); } - // snapshot of std::pair> pairs - class snapshot + // snapshot of std::pair> pairs, for + // some SUBCLASS derived from T + template + class snapshot_of { // It's very important that what we store in this snapshot are // weak_ptrs, NOT shared_ptrs. That's how we discover whether any // instance has been deleted during the lifespan of a snapshot. typedef std::vector> VectorType; - // Dereferencing our iterator produces a std::shared_ptr for each - // instance that still exists. Since we store weak_ptrs, that involves - // two chained transformations: + // Dereferencing the iterator we publish produces a + // std::shared_ptr for each instance that still exists. + // Since we store weak_ptr, that involves two chained + // transformations: // - a transform_iterator to lock the weak_ptr and return a shared_ptr - // - a filter_iterator to skip any shared_ptr that has become invalid. + // - a filter_iterator to skip any shared_ptr that has become + // invalid or references any T instance that isn't SUBCLASS. // It is very important that we filter lazily, that is, during // traversal. Any one of our stored weak_ptrs might expire during // traversal. - typedef std::pair strong_pair; + typedef std::pair> strong_pair; // Note for future reference: nat has not yet had any luck (up to // Boost 1.67) trying to use boost::transform_iterator with a hand- // coded functor, only with actual functions. In my experience, an @@ -127,7 +131,7 @@ public: // result_type typedef. But this works. static strong_pair strengthen(typename VectorType::value_type& pair) { - return { pair.first, pair.second.lock() }; + return { pair.first, std::dynamic_pointer_cast(pair.second.lock()) }; } static bool dead_skipper(const strong_pair& pair) { @@ -135,7 +139,7 @@ public: } public: - snapshot(): + snapshot_of(): // populate our vector with a snapshot of (locked!) InstanceMap // note, this assigns pair to pair mData(mLock->mMap.begin(), mLock->mMap.end()) @@ -184,44 +188,51 @@ public: #endif // LL_WINDOWS VectorType mData; }; + using snapshot = snapshot_of; - // iterate over this for references to each instance - class instance_snapshot: public snapshot + // iterate over this for references to each SUBCLASS instance + template + class instance_snapshot_of: public snapshot_of { private: - static T& instance_getter(typename snapshot::iterator::reference pair) + using super = snapshot_of; + static T& instance_getter(typename super::iterator::reference pair) { return *pair.second; } public: typedef boost::transform_iterator iterator; - iterator begin() { return iterator(snapshot::begin(), instance_getter); } - iterator end() { return iterator(snapshot::end(), instance_getter); } + typename super::iterator> iterator; + iterator begin() { return iterator(super::begin(), instance_getter); } + iterator end() { return iterator(super::end(), instance_getter); } void deleteAll() { - for (auto it(snapshot::begin()), end(snapshot::end()); it != end; ++it) + for (auto it(super::begin()), end(super::end()); it != end; ++it) { delete it->second.get(); } } - }; + }; + using instance_snapshot = instance_snapshot_of; // iterate over this for each key - class key_snapshot: public snapshot + template + class key_snapshot_of: public snapshot_of { private: - static KEY key_getter(typename snapshot::iterator::reference pair) + using super = snapshot_of; + static KEY key_getter(typename super::iterator::reference pair) { return pair.first; } public: typedef boost::transform_iterator iterator; - iterator begin() { return iterator(snapshot::begin(), key_getter); } - iterator end() { return iterator(snapshot::end(), key_getter); } + typename super::iterator> iterator; + iterator begin() { return iterator(super::begin(), key_getter); } + iterator end() { return iterator(super::end(), key_getter); } }; + using key_snapshot = key_snapshot_of; static ptr_t getInstance(const KEY& k) { @@ -368,22 +379,25 @@ public: return LockStatic()->mSet.size(); } - // snapshot of std::shared_ptr pointers - class snapshot + // snapshot of std::shared_ptr pointers + template + class snapshot_of { // It's very important that what we store in this snapshot are // weak_ptrs, NOT shared_ptrs. That's how we discover whether any // instance has been deleted during the lifespan of a snapshot. typedef std::vector VectorType; - // Dereferencing our iterator produces a std::shared_ptr for each - // instance that still exists. Since we store weak_ptrs, that involves - // two chained transformations: + // Dereferencing the iterator we publish produces a + // std::shared_ptr for each instance that still exists. + // Since we store weak_ptrs, that involves two chained + // transformations: // - a transform_iterator to lock the weak_ptr and return a shared_ptr - // - a filter_iterator to skip any shared_ptr that has become invalid. - typedef std::shared_ptr strong_ptr; + // - a filter_iterator to skip any shared_ptr that has become invalid + // or references any T instance that isn't SUBCLASS. + typedef std::shared_ptr strong_ptr; static strong_ptr strengthen(typename VectorType::value_type& ptr) { - return ptr.lock(); + return std::dynamic_pointer_cast(ptr.lock()); } static bool dead_skipper(const strong_ptr& ptr) { @@ -391,7 +405,7 @@ public: } public: - snapshot(): + snapshot_of(): // populate our vector with a snapshot of (locked!) InstanceSet // note, this assigns stored shared_ptrs to weak_ptrs for snapshot mData(mLock->mSet.begin(), mLock->mSet.end()) @@ -437,22 +451,33 @@ public: #endif // LL_WINDOWS VectorType mData; }; + using snapshot = snapshot_of; // iterate over this for references to each instance - struct instance_snapshot: public snapshot + template + class instance_snapshot_of: public snapshot_of { - typedef boost::indirect_iterator iterator; - iterator begin() { return iterator(snapshot::begin()); } - iterator end() { return iterator(snapshot::end()); } + private: + using super = snapshot_of; + + public: + typedef boost::indirect_iterator iterator; + iterator begin() { return iterator(super::begin()); } + iterator end() { return iterator(super::end()); } void deleteAll() { - for (auto it(snapshot::begin()), end(snapshot::end()); it != end; ++it) + for (auto it(super::begin()), end(super::end()); it != end; ++it) { delete it->get(); } } }; + using instance_snapshot = instance_snapshot_of; + // key_snapshot_of isn't really meaningful, but define it anyway to avoid + // requiring two different LLInstanceTrackerSubclass implementations. + template + using key_snapshot_of = instance_snapshot_of; protected: LLInstanceTracker() diff --git a/indra/llcommon/llinstancetrackersubclass.h b/indra/llcommon/llinstancetrackersubclass.h new file mode 100644 index 0000000000..ea9a38200f --- /dev/null +++ b/indra/llcommon/llinstancetrackersubclass.h @@ -0,0 +1,98 @@ +/** + * @file llinstancetrackersubclass.h + * @author Nat Goodspeed + * @date 2022-12-09 + * @brief Intermediate class to get subclass-specific types from + * LLInstanceTracker instance-retrieval methods. + * + * $LicenseInfo:firstyear=2022&license=viewerlgpl$ + * Copyright (c) 2022, Linden Research, Inc. + * $/LicenseInfo$ + */ + +#if ! defined(LL_LLINSTANCETRACKERSUBCLASS_H) +#define LL_LLINSTANCETRACKERSUBCLASS_H + +#include // std::shared_ptr, std::weak_ptr + +/** + * Derive your subclass S of a subclass T of LLInstanceTracker from + * LLInstanceTrackerSubclass to perform appropriate downcasting and + * filtering for LLInstanceTracker access methods. + * + * LLInstanceTracker uses CRTP, so that getWeak(), getInstance(), snapshot + * and instance_snapshot return pointers and references to T. The trouble is + * that subclasses T0 and T1 derived from T also get pointers and references + * to their base class T, requiring explicit downcasting. Moreover, + * T0::getInstance() shouldn't find an instance of any T subclass other than + * T0. Nor should T0::snapshot. + * + * @code + * class Tracked: public LLInstanceTracker + * { + * private: + * using super = LLInstanceTracker; + * public: + * Tracked(const std::string& name): super(name) {} + * // All references to Tracked::ptr_t, Tracked::getInstance() etc. + * // appropriately use Tracked. + * // ... + * }; + * + * // But now we derive SubTracked from Tracked. We need SubTracked::ptr_t, + * // SubTracked::getInstance() etc. to use SubTracked, not Tracked. + * // This LLInstanceTrackerSubclass specialization is itself derived from + * // Tracked. + * class SubTracked: public LLInstanceTrackerSubclass + * { + * private: + * using super = LLInstanceTrackerSubclass; + * public: + * // LLInstanceTrackerSubclass's constructor forwards to Tracked's. + * SubTracked(const std::string& name): super(name) {} + * // SubTracked::getInstance() returns std::shared_ptr, etc. + * // ... + * @endcode + */ +template +class LLInstanceTrackerSubclass: public T +{ +public: + using ptr_t = std::shared_ptr; + using weak_t = std::weak_ptr; + + // forward any constructor call to the corresponding T ctor + template + LLInstanceTrackerSubclass(ARGS&&... args): + T(std::forward(args)...) + {} + + weak_t getWeak() + { + // call base-class getWeak(), try to lock, downcast to SUBCLASS + return std::dynamic_pointer_cast(T::getWeak().lock()); + } + + template + static ptr_t getInstance(const KEY& k) + { + return std::dynamic_pointer_cast(T::getInstance(k)); + } + + using snapshot = typename T::template snapshot_of; + using instance_snapshot = typename T::template instance_snapshot_of; + using key_snapshot = typename T::template key_snapshot_of; + + static size_t instanceCount() + { + // T::instanceCount() lies because our snapshot, et al., won't + // necessarily return all the T instances -- only those that are also + // SUBCLASS instances. Count those. + size_t count = 0; + for (const auto& pair : snapshot()) + ++count; + return count; + } +}; + +#endif /* ! defined(LL_LLINSTANCETRACKERSUBCLASS_H) */ -- cgit v1.2.3 From fc424a0db90fd2d2e44e85a19750ad6eaa57b28a Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 9 Dec 2022 13:21:45 -0500 Subject: SL-18809: Add WorkSchedule; remove timestamps from WorkQueue. For work queues that don't need timestamped tasks, eliminate the overhead of a priority queue ordered by timestamp. Timestamped task support moves to WorkSchedule. WorkQueue is a simpler queue that just waits for work. Both WorkQueue and WorkSchedule can be accessed via new WorkQueueBase API. Of course the WorkQueueBase API doesn't deal with timestamps, but a WorkSchedule can be accessed directly to post timestamped tasks and then handled normally (e.g. by ThreadPool) to run them. Most ThreadPool functionality migrates to new ThreadPoolBase class, with template subclass ThreadPoolUsing or ThreadPoolUsing depending on need. ThreadPool is now an alias for ThreadPoolUsing. Importantly, ThreadPoolUsing::getQueue() delivers a reference to the specific queue subclass type, so you can post timestamped tasks on a queue retrieved from ThreadPoolUsing::getQueue(). Since ThreadPool is no longer a simple class but an alias for a particular template specialization, introduce threadpool_fwd.h to forward-declare it. Recast workqueue_test.cpp to exercise WorkSchedule, since some of the tests are time-based. A future todo would be to exercise each applicable test with both WorkQueue and WorkSchedule. --- indra/llcommon/CMakeLists.txt | 1 + indra/llcommon/tests/workqueue_test.cpp | 26 +- indra/llcommon/threadpool.cpp | 27 ++- indra/llcommon/threadpool.h | 60 ++++- indra/llcommon/threadpool_fwd.h | 25 ++ indra/llcommon/workqueue.cpp | 194 +++++++++++---- indra/llcommon/workqueue.h | 416 +++++++++++++++++++------------- 7 files changed, 493 insertions(+), 256 deletions(-) create mode 100644 indra/llcommon/threadpool_fwd.h (limited to 'indra/llcommon') diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 0fe61108ff..96fdb1f924 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -262,6 +262,7 @@ set(llcommon_HEADER_FILES stdtypes.h stringize.h threadpool.h + threadpool_fwd.h threadsafeschedule.h timer.h tuple.h diff --git a/indra/llcommon/tests/workqueue_test.cpp b/indra/llcommon/tests/workqueue_test.cpp index 1d73f7aa0d..41aa858084 100644 --- a/indra/llcommon/tests/workqueue_test.cpp +++ b/indra/llcommon/tests/workqueue_test.cpp @@ -38,7 +38,7 @@ namespace tut { struct workqueue_data { - WorkQueue queue{"queue"}; + WorkSchedule queue{"queue"}; }; typedef test_group workqueue_group; typedef workqueue_group::object object; @@ -49,8 +49,8 @@ namespace tut { set_test_name("name"); ensure_equals("didn't capture name", queue.getKey(), "queue"); - ensure("not findable", WorkQueue::getInstance("queue") == queue.getWeak().lock()); - WorkQueue q2; + ensure("not findable", WorkSchedule::getInstance("queue") == queue.getWeak().lock()); + WorkSchedule q2; ensure("has no name", LLStringUtil::startsWith(q2.getKey(), "WorkQueue")); } @@ -73,16 +73,16 @@ namespace tut { set_test_name("postEvery"); // record of runs - using Shared = std::deque; + using Shared = std::deque; // This is an example of how to share data between the originator of - // postEvery(work) and the work item itself, since usually a WorkQueue + // postEvery(work) and the work item itself, since usually a WorkSchedule // is used to dispatch work to a different thread. Neither of them // should call any of LLCond's wait methods: you don't want to stall // either the worker thread or the originating thread (conventionally // main). Use LLCond or a subclass even if all you want to do is // signal the work item that it can quit; consider LLOneShotCond. LLCond data; - auto start = WorkQueue::TimePoint::clock::now(); + auto start = WorkSchedule::TimePoint::clock::now(); auto interval = 100ms; queue.postEvery( interval, @@ -93,7 +93,7 @@ namespace tut data.update_one( [](Shared& data) { - data.push_back(WorkQueue::TimePoint::clock::now()); + data.push_back(WorkSchedule::TimePoint::clock::now()); }); // by the 3rd call, return false to stop return (++count < 3); @@ -102,7 +102,7 @@ namespace tut // postEvery() running, so run until we have exhausted the iterations // or we time out waiting for (auto finish = start + 10*interval; - WorkQueue::TimePoint::clock::now() < finish && + WorkSchedule::TimePoint::clock::now() < finish && data.get([](const Shared& data){ return data.size(); }) < 3; ) { queue.runPending(); @@ -139,8 +139,8 @@ namespace tut void object::test<4>() { set_test_name("postTo"); - WorkQueue main("main"); - auto qptr = WorkQueue::getInstance("queue"); + WorkSchedule main("main"); + auto qptr = WorkSchedule::getInstance("queue"); int result = 0; main.postTo( qptr, @@ -171,8 +171,8 @@ namespace tut void object::test<5>() { set_test_name("postTo with void return"); - WorkQueue main("main"); - auto qptr = WorkQueue::getInstance("queue"); + WorkSchedule main("main"); + auto qptr = WorkSchedule::getInstance("queue"); std::string observe; main.postTo( qptr, @@ -194,7 +194,7 @@ namespace tut std::string stored; // Try to call waitForResult() on this thread's main coroutine. It // should throw because the main coroutine must service the queue. - auto what{ catch_what( + auto what{ catch_what( [this, &stored](){ stored = queue.waitForResult( [](){ return "should throw"; }); }) }; ensure("lambda should not have run", stored.empty()); diff --git a/indra/llcommon/threadpool.cpp b/indra/llcommon/threadpool.cpp index f49dd40a8b..856306d8f4 100644 --- a/indra/llcommon/threadpool.cpp +++ b/indra/llcommon/threadpool.cpp @@ -23,14 +23,15 @@ #include "llsd.h" #include "stringize.h" -LL::ThreadPool::ThreadPool(const std::string& name, size_t threads, size_t capacity): +LL::ThreadPoolBase::ThreadPoolBase(const std::string& name, size_t threads, + WorkQueueBase* queue): super(name), - mQueue(name, capacity), mName("ThreadPool:" + name), - mThreadCount(getConfiguredWidth(name, threads)) + mThreadCount(getConfiguredWidth(name, threads)), + mQueue(queue) {} -void LL::ThreadPool::start() +void LL::ThreadPoolBase::start() { for (size_t i = 0; i < mThreadCount; ++i) { @@ -58,17 +59,17 @@ void LL::ThreadPool::start() }); } -LL::ThreadPool::~ThreadPool() +LL::ThreadPoolBase::~ThreadPoolBase() { close(); } -void LL::ThreadPool::close() +void LL::ThreadPoolBase::close() { - if (! mQueue.isClosed()) + if (! mQueue->isClosed()) { LL_DEBUGS("ThreadPool") << mName << " closing queue and joining threads" << LL_ENDL; - mQueue.close(); + mQueue->close(); for (auto& pair: mThreads) { LL_DEBUGS("ThreadPool") << mName << " waiting on thread " << pair.first << LL_ENDL; @@ -78,20 +79,20 @@ void LL::ThreadPool::close() } } -void LL::ThreadPool::run(const std::string& name) +void LL::ThreadPoolBase::run(const std::string& name) { LL_DEBUGS("ThreadPool") << name << " starting" << LL_ENDL; run(); LL_DEBUGS("ThreadPool") << name << " stopping" << LL_ENDL; } -void LL::ThreadPool::run() +void LL::ThreadPoolBase::run() { - mQueue.runUntilClose(); + mQueue->runUntilClose(); } //static -size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft) +size_t LL::ThreadPoolBase::getConfiguredWidth(const std::string& name, size_t dft) { LLSD poolSizes; try @@ -132,7 +133,7 @@ size_t LL::ThreadPool::getConfiguredWidth(const std::string& name, size_t dft) } //static -size_t LL::ThreadPool::getWidth(const std::string& name, size_t dft) +size_t LL::ThreadPoolBase::getWidth(const std::string& name, size_t dft) { auto instance{ getInstance(name) }; if (instance) diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h index b49d511257..60f4a0ce1b 100644 --- a/indra/llcommon/threadpool.h +++ b/indra/llcommon/threadpool.h @@ -13,7 +13,9 @@ #if ! defined(LL_THREADPOOL_H) #define LL_THREADPOOL_H +#include "threadpool_fwd.h" #include "workqueue.h" +#include // std::unique_ptr #include #include #include // std::pair @@ -22,26 +24,24 @@ namespace LL { - class ThreadPool: public LLInstanceTracker + class ThreadPoolBase: public LLInstanceTracker { private: - using super = LLInstanceTracker; + using super = LLInstanceTracker; + public: /** - * Pass ThreadPool a string name. This can be used to look up the + * Pass ThreadPoolBase a string name. This can be used to look up the * relevant WorkQueue. * * The number of threads you pass sets the compile-time default. But * if the user has overridden the LLSD map in the "ThreadPoolSizes" * setting with a key matching this ThreadPool name, that setting * overrides this parameter. - * - * Pass an explicit capacity to limit the size of the queue. - * Constraining the queue can cause a submitter to block. Do not - * constrain any ThreadPool accepting work from the main thread. */ - ThreadPool(const std::string& name, size_t threads=1, size_t capacity=1024*1024); - virtual ~ThreadPool(); + ThreadPoolBase(const std::string& name, size_t threads, + WorkQueueBase* queue); + virtual ~ThreadPoolBase(); /** * Launch the ThreadPool. Until this call, a constructed ThreadPool @@ -59,8 +59,6 @@ namespace LL std::string getName() const { return mName; } size_t getWidth() const { return mThreads.size(); } - /// obtain a non-const reference to the WorkQueue to post work to it - WorkQueue& getQueue() { return mQueue; } /** * Override run() if you need special processing. The default run() @@ -87,15 +85,53 @@ namespace LL static size_t getWidth(const std::string& name, size_t dft); + protected: + std::unique_ptr mQueue; + private: void run(const std::string& name); - WorkQueue mQueue; std::string mName; size_t mThreadCount; std::vector> mThreads; }; + /** + * Specialize with WorkQueue or, for timestamped tasks, WorkSchedule + */ + template + struct ThreadPoolUsing: public ThreadPoolBase + { + using queue_t = QUEUE; + + /** + * Pass ThreadPoolUsing a string name. This can be used to look up the + * relevant WorkQueue. + * + * The number of threads you pass sets the compile-time default. But + * if the user has overridden the LLSD map in the "ThreadPoolSizes" + * setting with a key matching this ThreadPool name, that setting + * overrides this parameter. + * + * Pass an explicit capacity to limit the size of the queue. + * Constraining the queue can cause a submitter to block. Do not + * constrain any ThreadPool accepting work from the main thread. + */ + ThreadPoolUsing(const std::string& name, size_t threads=1, size_t capacity=1024*1024): + ThreadPoolBase(name, threads, new queue_t(name, capacity)) + {} + ~ThreadPoolUsing() override {} + + /** + * obtain a non-const reference to the specific WorkQueue subclass to + * post work to it + */ + queue_t& getQueue() { return static_cast(*mQueue); } + }; + + /// ThreadPool is shorthand for using the simpler WorkQueue + using ThreadPool = ThreadPoolUsing; + } // namespace LL #endif /* ! defined(LL_THREADPOOL_H) */ diff --git a/indra/llcommon/threadpool_fwd.h b/indra/llcommon/threadpool_fwd.h new file mode 100644 index 0000000000..1aa3c4a0e2 --- /dev/null +++ b/indra/llcommon/threadpool_fwd.h @@ -0,0 +1,25 @@ +/** + * @file threadpool_fwd.h + * @author Nat Goodspeed + * @date 2022-12-09 + * @brief Forward declarations for ThreadPool et al. + * + * $LicenseInfo:firstyear=2022&license=viewerlgpl$ + * Copyright (c) 2022, Linden Research, Inc. + * $/LicenseInfo$ + */ + +#if ! defined(LL_THREADPOOL_FWD_H) +#define LL_THREADPOOL_FWD_H + +#include "workqueue.h" + +namespace LL +{ + template + struct ThreadPoolUsing; + + using ThreadPool = ThreadPoolUsing; +} // namespace LL + +#endif /* ! defined(LL_THREADPOOL_FWD_H) */ diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index eb06890468..83e0216ae7 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -26,83 +26,65 @@ using Mutex = LLCoros::Mutex; using Lock = LLCoros::LockType; -LL::WorkQueue::WorkQueue(const std::string& name, size_t capacity): - super(makeName(name)), - mQueue(capacity) +/***************************************************************************** +* WorkQueueBase +*****************************************************************************/ +LL::WorkQueueBase::WorkQueueBase(const std::string& name): + super(makeName(name)) { // TODO: register for "LLApp" events so we can implicitly close() on // viewer shutdown. } -void LL::WorkQueue::close() -{ - mQueue.close(); -} - -size_t LL::WorkQueue::size() -{ - return mQueue.size(); -} - -bool LL::WorkQueue::isClosed() -{ - return mQueue.isClosed(); -} - -bool LL::WorkQueue::done() -{ - return mQueue.done(); -} - -void LL::WorkQueue::runUntilClose() +void LL::WorkQueueBase::runUntilClose() { try { for (;;) { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; - callWork(mQueue.pop()); + callWork(pop_()); } } - catch (const Queue::Closed&) + catch (const Closed&) { } } -bool LL::WorkQueue::runPending() +bool LL::WorkQueueBase::runPending() { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; - for (Work work; mQueue.tryPop(work); ) + for (Work work; tryPop_(work); ) { callWork(work); } - return ! mQueue.done(); + return ! done(); } -bool LL::WorkQueue::runOne() +bool LL::WorkQueueBase::runOne() { Work work; - if (mQueue.tryPop(work)) + if (tryPop_(work)) { callWork(work); } - return ! mQueue.done(); + return ! done(); } -bool LL::WorkQueue::runUntil(const TimePoint& until) +bool LL::WorkQueueBase::runUntil(const TimePoint& until) { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; // Should we subtract some slop to allow for typical Work execution time? // How much slop? // runUntil() is simply a time-bounded runPending(). - for (Work work; TimePoint::clock::now() < until && mQueue.tryPop(work); ) + for (Work work; TimePoint::clock::now() < until && tryPop_(work); ) { callWork(work); } - return ! mQueue.done(); + return ! done(); } -std::string LL::WorkQueue::makeName(const std::string& name) +std::string LL::WorkQueueBase::makeName(const std::string& name) { if (! name.empty()) return name; @@ -120,14 +102,7 @@ std::string LL::WorkQueue::makeName(const std::string& name) return STRINGIZE("WorkQueue" << num); } -void LL::WorkQueue::callWork(const Queue::DataTuple& work) -{ - // ThreadSafeSchedule::pop() always delivers a tuple, even when - // there's only one data field per item, as for us. - callWork(std::get<0>(work)); -} - -void LL::WorkQueue::callWork(const Work& work) +void LL::WorkQueueBase::callWork(const Work& work) { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; try @@ -142,12 +117,12 @@ void LL::WorkQueue::callWork(const Work& work) } } -void LL::WorkQueue::error(const std::string& msg) +void LL::WorkQueueBase::error(const std::string& msg) { LL_ERRS("WorkQueue") << msg << LL_ENDL; } -void LL::WorkQueue::checkCoroutine(const std::string& method) +void LL::WorkQueueBase::checkCoroutine(const std::string& method) { // By convention, the default coroutine on each thread has an empty name // string. See also LLCoros::logname(). @@ -156,3 +131,130 @@ void LL::WorkQueue::checkCoroutine(const std::string& method) LLTHROW(Error("Do not call " + method + " from a thread's default coroutine")); } } + +/***************************************************************************** +* WorkQueue +*****************************************************************************/ +LL::WorkQueue::WorkQueue(const std::string& name, size_t capacity): + super(name), + mQueue(capacity) +{ +} + +void LL::WorkQueue::close() +{ + mQueue.close(); +} + +size_t LL::WorkQueue::size() +{ + return mQueue.size(); +} + +bool LL::WorkQueue::isClosed() +{ + return mQueue.isClosed(); +} + +bool LL::WorkQueue::done() +{ + return mQueue.done(); +} + +void LL::WorkQueue::post(const Work& callable) +{ + mQueue.push(callable); +} + +bool LL::WorkQueue::postIfOpen(const Work& callable) +{ + return mQueue.pushIfOpen(callable); +} + +bool LL::WorkQueue::tryPost(const Work& callable) +{ + return mQueue.tryPush(callable); +} + +LL::WorkQueue::Work LL::WorkQueue::pop_() +{ + return mQueue.pop(); +} + +bool LL::WorkQueue::tryPop_(Work& work) +{ + return mQueue.tryPop(work); +} + +/***************************************************************************** +* WorkSchedule +*****************************************************************************/ +LL::WorkSchedule::WorkSchedule(const std::string& name, size_t capacity): + super(name), + mQueue(capacity) +{ +} + +void LL::WorkSchedule::close() +{ + mQueue.close(); +} + +size_t LL::WorkSchedule::size() +{ + return mQueue.size(); +} + +bool LL::WorkSchedule::isClosed() +{ + return mQueue.isClosed(); +} + +bool LL::WorkSchedule::done() +{ + return mQueue.done(); +} + +void LL::WorkSchedule::post(const Work& callable) +{ + // Use TimePoint::clock::now() instead of TimePoint's representation of + // the epoch because this WorkSchedule may contain a mix of past-due + // TimedWork items and TimedWork items scheduled for the future. Sift this + // new item into the correct place. + post(callable, TimePoint::clock::now()); +} + +void LL::WorkSchedule::post(const Work& callable, const TimePoint& time) +{ + mQueue.push(TimedWork(time, callable)); +} + +bool LL::WorkSchedule::postIfOpen(const Work& callable) +{ + return postIfOpen(callable, TimePoint::clock::now()); +} + +bool LL::WorkSchedule::postIfOpen(const Work& callable, const TimePoint& time) +{ + return mQueue.pushIfOpen(TimedWork(time, callable)); +} + +bool LL::WorkSchedule::tryPost(const Work& callable) +{ + return tryPost(callable, TimePoint::clock::now()); +} + +bool LL::WorkSchedule::tryPost(const Work& callable, const TimePoint& time) +{ + return mQueue.tryPush(TimedWork(time, callable)); +} + +LL::WorkSchedule::Work LL::WorkSchedule::pop_() +{ + return std::get<0>(mQueue.pop()); +} + +bool LL::WorkSchedule::tryPop_(Work& work) +{ + return mQueue.tryPop(work); +} diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 28a0b5e040..eea8886a7a 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -15,6 +15,7 @@ #include "llcoros.h" #include "llexception.h" #include "llinstancetracker.h" +#include "llinstancetrackersubclass.h" #include "threadsafeschedule.h" #include #include // std::current_exception @@ -23,27 +24,23 @@ namespace LL { + +/***************************************************************************** +* WorkQueueBase: API for WorkQueue and WorkSchedule +*****************************************************************************/ /** * A typical WorkQueue has a string name that can be used to find it. */ - class WorkQueue: public LLInstanceTracker + class WorkQueueBase: public LLInstanceTracker { private: - using super = LLInstanceTracker; + using super = LLInstanceTracker; public: using Work = std::function; - - private: - using Queue = ThreadSafeSchedule; - // helper for postEvery() - template - class BackJack; - - public: - using TimePoint = Queue::TimePoint; - using TimedWork = Queue::TimeTuple; - using Closed = Queue::Closed; + using Closed = LLThreadSafeQueueInterrupt; + // for runFor() + using TimePoint = std::chrono::steady_clock::time_point; struct Error: public LLException { @@ -51,18 +48,18 @@ namespace LL }; /** - * You may omit the WorkQueue name, in which case a unique name is + * You may omit the WorkQueueBase name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkQueue(const std::string& name = std::string(), size_t capacity=1024); + WorkQueueBase(const std::string& name); /** * Since the point of WorkQueue is to pass work to some other worker - * thread(s) asynchronously, it's important that the WorkQueue continue - * to exist until the worker thread(s) have drained it. To communicate - * that it's time for them to quit, close() the queue. + * thread(s) asynchronously, it's important that it continue to exist + * until the worker thread(s) have drained it. To communicate that + * it's time for them to quit, close() the queue. */ - void close(); + virtual void close() = 0; /** * WorkQueue supports multiple producers and multiple consumers. In @@ -78,158 +75,60 @@ namespace LL * * If you're the only consumer, noticing that size() > 0 is * meaningful. */ - size_t size(); + virtual size_t size() = 0; /// producer end: are we prevented from pushing any additional items? - bool isClosed(); + virtual bool isClosed() = 0; /// consumer end: are we done, is the queue entirely drained? - bool done(); + virtual bool done() = 0; /*---------------------- fire and forget API -----------------------*/ - /// fire-and-forget, but at a particular (future?) time - template - void post(const TimePoint& time, CALLABLE&& callable) - { - // Defer reifying an arbitrary CALLABLE until we hit this or - // postIfOpen(). All other methods should accept CALLABLEs of - // arbitrary type to avoid multiple levels of std::function - // indirection. - mQueue.push(TimedWork(time, std::move(callable))); - } - /// fire-and-forget - template - void post(CALLABLE&& callable) - { - // We use TimePoint::clock::now() instead of TimePoint's - // representation of the epoch because this WorkQueue may contain - // a mix of past-due TimedWork items and TimedWork items scheduled - // for the future. Sift this new item into the correct place. - post(TimePoint::clock::now(), std::move(callable)); - } - - /** - * post work for a particular time, unless the queue is closed before - * we can post - */ - template - bool postIfOpen(const TimePoint& time, CALLABLE&& callable) - { - // Defer reifying an arbitrary CALLABLE until we hit this or - // post(). All other methods should accept CALLABLEs of arbitrary - // type to avoid multiple levels of std::function indirection. - return mQueue.pushIfOpen(TimedWork(time, std::move(callable))); - } + virtual void post(const Work&) = 0; /** * post work, unless the queue is closed before we can post */ - template - bool postIfOpen(CALLABLE&& callable) - { - return postIfOpen(TimePoint::clock::now(), std::move(callable)); - } + virtual bool postIfOpen(const Work&) = 0; /** - * Post work to be run at a specified time to another WorkQueue, which - * may or may not still exist and be open. Return true if we were able - * to post. + * post work, unless the queue is full */ - template - static bool postMaybe(weak_t target, const TimePoint& time, CALLABLE&& callable); + virtual bool tryPost(const Work&) = 0; /** * Post work to another WorkQueue, which may or may not still exist - * and be open. Return true if we were able to post. - */ - template - static bool postMaybe(weak_t target, CALLABLE&& callable) - { - return postMaybe(target, TimePoint::clock::now(), - std::forward(callable)); - } - - /** - * Launch a callable returning bool that will trigger repeatedly at - * specified interval, until the callable returns false. - * - * If you need to signal that callable from outside, DO NOT bind a - * reference to a simple bool! That's not thread-safe. Instead, bind - * an LLCond variant, e.g. LLOneShotCond or LLBoolCond. + * and be open. Support any post() overload. Return true if we were + * able to post. */ - template - void postEvery(const std::chrono::duration& interval, - CALLABLE&& callable); - - template - bool tryPost(const TimePoint& time, CALLABLE&& callable) - { - return mQueue.tryPush(TimedWork(time, std::move(callable))); - } - - template - bool tryPost(CALLABLE&& callable) - { - return mQueue.tryPush(TimePoint::clock::now(), std::move(callable)); - } + template + static bool postMaybe(weak_t target, ARGS&&... args); /*------------------------- handshake API --------------------------*/ - /** - * Post work to another WorkQueue to be run at a specified time, - * requesting a specific callback to be run on this WorkQueue on - * completion. - * - * Returns true if able to post, false if the other WorkQueue is - * inaccessible. - */ - // Apparently some Microsoft header file defines a macro CALLBACK? The - // natural template argument name CALLBACK produces very weird Visual - // Studio compile errors that seem utterly unrelated to this source - // code. - template - bool postTo(weak_t target, - const TimePoint& time, CALLABLE&& callable, FOLLOWUP&& callback); - /** * Post work to another WorkQueue, requesting a specific callback to - * be run on this WorkQueue on completion. + * be run on this WorkQueue on completion. Optional final argument is + * TimePoint for WorkSchedule. * * Returns true if able to post, false if the other WorkQueue is * inaccessible. */ - template - bool postTo(weak_t target, CALLABLE&& callable, FOLLOWUP&& callback) - { - return postTo(target, TimePoint::clock::now(), - std::move(callable), std::move(callback)); - } - - /** - * Post work to another WorkQueue to be run at a specified time, - * blocking the calling coroutine until then, returning the result to - * caller on completion. - * - * In general, we assume that each thread's default coroutine is busy - * servicing its WorkQueue or whatever. To try to prevent mistakes, we - * forbid calling waitForResult() from a thread's default coroutine. - */ - template - auto waitForResult(const TimePoint& time, CALLABLE&& callable); + template + bool postTo(weak_t target, CALLABLE&& callable, FOLLOWUP&& callback, + ARGS&&... args); /** * Post work to another WorkQueue, blocking the calling coroutine - * until then, returning the result to caller on completion. + * until then, returning the result to caller on completion. Optional + * final argument is TimePoint for WorkSchedule. * * In general, we assume that each thread's default coroutine is busy * servicing its WorkQueue or whatever. To try to prevent mistakes, we * forbid calling waitForResult() from a thread's default coroutine. */ - template - auto waitForResult(CALLABLE&& callable) - { - return waitForResult(TimePoint::clock::now(), std::move(callable)); - } + template + auto waitForResult(CALLABLE&& callable, ARGS&&... args); /*--------------------------- worker API ---------------------------*/ @@ -276,7 +175,7 @@ namespace LL */ bool runUntil(const TimePoint& until); - private: + protected: template static auto makeReplyLambda(CALLABLE&& callable, FOLLOWUP&& callback); /// general case: arbitrary C++ return type @@ -296,13 +195,179 @@ namespace LL static void checkCoroutine(const std::string& method); static void error(const std::string& msg); static std::string makeName(const std::string& name); - void callWork(const Queue::DataTuple& work); void callWork(const Work& work); + + private: + virtual Work pop_() = 0; + virtual bool tryPop_(Work&) = 0; + }; + +/***************************************************************************** +* WorkQueue: no timestamped task support +*****************************************************************************/ + class WorkQueue: public LLInstanceTrackerSubclass + { + private: + using super = LLInstanceTrackerSubclass; + + public: + /** + * You may omit the WorkQueue name, in which case a unique name is + * synthesized; for practical purposes that makes it anonymous. + */ + WorkQueue(const std::string& name = std::string(), size_t capacity=1024); + + /** + * Since the point of WorkQueue is to pass work to some other worker + * thread(s) asynchronously, it's important that it continue to exist + * until the worker thread(s) have drained it. To communicate that + * it's time for them to quit, close() the queue. + */ + void close() override; + + /** + * WorkQueue supports multiple producers and multiple consumers. In + * the general case it's misleading to test size(), since any other + * thread might change it the nanosecond the lock is released. On that + * basis, some might argue against publishing a size() method at all. + * + * But there are two specific cases in which a test based on size() + * might be reasonable: + * + * * If you're the only producer, noticing that size() == 0 is + * meaningful. + * * If you're the only consumer, noticing that size() > 0 is + * meaningful. + */ + size_t size() override; + /// producer end: are we prevented from pushing any additional items? + bool isClosed() override; + /// consumer end: are we done, is the queue entirely drained? + bool done() override; + + /*---------------------- fire and forget API -----------------------*/ + + /// fire-and-forget + void post(const Work&) override; + + /** + * post work, unless the queue is closed before we can post + */ + bool postIfOpen(const Work&) override; + + /** + * post work, unless the queue is full + */ + bool tryPost(const Work&) override; + + private: + using Queue = LLThreadSafeQueue; Queue mQueue; + + Work pop_() override; + bool tryPop_(Work&) override; + }; + +/***************************************************************************** +* WorkSchedule: add support for timestamped tasks +*****************************************************************************/ + class WorkSchedule: public LLInstanceTrackerSubclass + { + private: + using super = LLInstanceTrackerSubclass; + using Queue = ThreadSafeSchedule; + // helper for postEvery() + template + class BackJack; + + public: + using TimePoint = Queue::TimePoint; + using TimedWork = Queue::TimeTuple; + + /** + * You may omit the WorkSchedule name, in which case a unique name is + * synthesized; for practical purposes that makes it anonymous. + */ + WorkSchedule(const std::string& name = std::string(), size_t capacity=1024); + + /** + * Since the point of WorkSchedule is to pass work to some other worker + * thread(s) asynchronously, it's important that the WorkSchedule continue + * to exist until the worker thread(s) have drained it. To communicate + * that it's time for them to quit, close() the queue. + */ + void close() override; + + /** + * WorkSchedule supports multiple producers and multiple consumers. In + * the general case it's misleading to test size(), since any other + * thread might change it the nanosecond the lock is released. On that + * basis, some might argue against publishing a size() method at all. + * + * But there are two specific cases in which a test based on size() + * might be reasonable: + * + * * If you're the only producer, noticing that size() == 0 is + * meaningful. + * * If you're the only consumer, noticing that size() > 0 is + * meaningful. + */ + size_t size() override; + /// producer end: are we prevented from pushing any additional items? + bool isClosed() override; + /// consumer end: are we done, is the queue entirely drained? + bool done() override; + + /*---------------------- fire and forget API -----------------------*/ + + /// fire-and-forget + void post(const Work& callable) override; + + /// fire-and-forget, but at a particular (future?) time + void post(const Work& callable, const TimePoint& time); + + /** + * post work, unless the queue is closed before we can post + */ + bool postIfOpen(const Work& callable) override; + + /** + * post work for a particular time, unless the queue is closed before + * we can post + */ + bool postIfOpen(const Work& callable, const TimePoint& time); + + /** + * post work, unless the queue is full + */ + bool tryPost(const Work& callable) override; + + /** + * post work for a particular time, unless the queue is full + */ + bool tryPost(const Work& callable, const TimePoint& time); + + /** + * Launch a callable returning bool that will trigger repeatedly at + * specified interval, until the callable returns false. + * + * If you need to signal that callable from outside, DO NOT bind a + * reference to a simple bool! That's not thread-safe. Instead, bind + * an LLCond variant, e.g. LLOneShotCond or LLBoolCond. + */ + template + void postEvery(const std::chrono::duration& interval, + CALLABLE&& callable); + + private: + Queue mQueue; + + Work pop_() override; + bool tryPop_(Work&) override; }; /** - * BackJack is, in effect, a hand-rolled lambda, binding a WorkQueue, a + * BackJack is, in effect, a hand-rolled lambda, binding a WorkSchedule, a * CALLABLE that returns bool, a TimePoint and an interval at which to * relaunch it. As long as the callable continues returning true, BackJack * keeps resubmitting it to the target WorkQueue. @@ -311,7 +376,7 @@ namespace LL // class method gets its own 'this' pointer -- which we need to resubmit // the whole BackJack callable. template - class WorkQueue::BackJack + class WorkSchedule::BackJack { public: // bind the desired data @@ -325,9 +390,10 @@ namespace LL mCallable(std::move(callable)) {} - // Call by target WorkQueue -- note that although WE require a - // callable returning bool, WorkQueue wants a void callable. We - // consume the bool. + // This operator() method, called by target WorkSchedule, is what + // makes this object a Work item. Although WE require a callable + // returning bool, WorkSchedule wants a void callable. We consume the + // bool. void operator()() { // If mCallable() throws an exception, don't catch it here: if it @@ -343,7 +409,7 @@ namespace LL // register our intent to fire at exact mIntervals. mStart += mInterval; - // We're being called at this moment by the target WorkQueue. + // We're being called at this moment by the target WorkSchedule. // Assume it still exists, rather than checking the result of // lock(). // Resubmit the whole *this callable: that's why we're a class @@ -353,7 +419,8 @@ namespace LL // moved-from. try { - mTarget.lock()->post(mStart, std::move(*this)); + std::dynamic_pointer_cast(mTarget.lock())-> + post(std::move(*this), mStart); } catch (const Closed&) { @@ -370,8 +437,8 @@ namespace LL }; template - void WorkQueue::postEvery(const std::chrono::duration& interval, - CALLABLE&& callable) + void WorkSchedule::postEvery(const std::chrono::duration& interval, + CALLABLE&& callable) { if (interval.count() <= 0) { @@ -394,7 +461,7 @@ namespace LL /// general case: arbitrary C++ return type template - struct WorkQueue::MakeReplyLambda + struct WorkQueueBase::MakeReplyLambda { auto operator()(CALLABLE&& callable, FOLLOWUP&& callback) { @@ -415,7 +482,7 @@ namespace LL /// specialize for CALLABLE returning void template - struct WorkQueue::MakeReplyLambda + struct WorkQueueBase::MakeReplyLambda { auto operator()(CALLABLE&& callable, FOLLOWUP&& callback) { @@ -427,16 +494,16 @@ namespace LL }; template - auto WorkQueue::makeReplyLambda(CALLABLE&& callable, FOLLOWUP&& callback) + auto WorkQueueBase::makeReplyLambda(CALLABLE&& callable, FOLLOWUP&& callback) { return MakeReplyLambda(callable)())>() (std::move(callable), std::move(callback)); } - template - bool WorkQueue::postTo(weak_t target, - const TimePoint& time, CALLABLE&& callable, FOLLOWUP&& callback) + template + bool WorkQueueBase::postTo(weak_t target, CALLABLE&& callable, FOLLOWUP&& callback, + ARGS&&... args) { LL_PROFILE_ZONE_SCOPED; // We're being asked to post to the WorkQueue at target. @@ -450,12 +517,11 @@ namespace LL // lambda that packages our callable, our callback and a weak_ptr // to this originating WorkQueue. tptr->post( - time, [reply = super::getWeak(), callable = std::move(callable), callback = std::move(callback)] - () - mutable { + () mutable + { // Use postMaybe() below in case this originating WorkQueue // has been closed or destroyed. Remember, the outer lambda is // now running on a thread servicing the target WorkQueue, and @@ -478,14 +544,16 @@ namespace LL // originating WorkQueue. Once there, rethrow it. [exc = std::current_exception()](){ std::rethrow_exception(exc); }); } - }); + }, + // if caller passed a TimePoint, pass it along to post() + std::forward(args)...); // looks like we were able to post() return true; } - template - bool WorkQueue::postMaybe(weak_t target, const TimePoint& time, CALLABLE&& callable) + template + bool WorkQueueBase::postMaybe(weak_t target, ARGS&&... args) { LL_PROFILE_ZONE_SCOPED; // target is a weak_ptr: have to lock it to check it @@ -494,7 +562,7 @@ namespace LL { try { - tptr->post(time, std::forward(callable)); + tptr->post(std::forward(args)...); // we were able to post() return true; } @@ -509,13 +577,13 @@ namespace LL /// general case: arbitrary C++ return type template - struct WorkQueue::WaitForResult + struct WorkQueueBase::WaitForResult { - auto operator()(WorkQueue* self, const TimePoint& time, CALLABLE&& callable) + template + auto operator()(WorkQueueBase* self, CALLABLE&& callable, ARGS&&... args) { LLCoros::Promise promise; self->post( - time, // We dare to bind a reference to Promise because it's // specifically designed for cross-thread communication. [&promise, callable = std::move(callable)]() @@ -529,7 +597,9 @@ namespace LL { promise.set_exception(std::current_exception()); } - }); + }, + // if caller passed a TimePoint, pass it to post() + std::forward(args)...); auto future{ LLCoros::getFuture(promise) }; // now, on the calling thread, wait for that result LLCoros::TempStatus st("waiting for WorkQueue::waitForResult()"); @@ -539,13 +609,13 @@ namespace LL /// specialize for CALLABLE returning void template - struct WorkQueue::WaitForResult + struct WorkQueueBase::WaitForResult { - void operator()(WorkQueue* self, const TimePoint& time, CALLABLE&& callable) + template + void operator()(WorkQueueBase* self, CALLABLE&& callable, ARGS&&... args) { LLCoros::Promise promise; self->post( - time, // &promise is designed for cross-thread access [&promise, callable = std::move(callable)]() mutable { @@ -558,7 +628,9 @@ namespace LL { promise.set_exception(std::current_exception()); } - }); + }, + // if caller passed a TimePoint, pass it to post() + std::forward(args)...); auto future{ LLCoros::getFuture(promise) }; // block until set_value() LLCoros::TempStatus st("waiting for void WorkQueue::waitForResult()"); @@ -566,13 +638,13 @@ namespace LL } }; - template - auto WorkQueue::waitForResult(const TimePoint& time, CALLABLE&& callable) + template + auto WorkQueueBase::waitForResult(CALLABLE&& callable, ARGS&&... args) { checkCoroutine("waitForResult()"); // derive callable's return type so we can specialize for void return WaitForResult(callable)())>() - (this, time, std::forward(callable)); + (this, std::forward(callable), std::forward(args)...); } } // namespace LL -- cgit v1.2.3 From 424d3ef83cdb354e66789f22f65394f4db523128 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 13 Dec 2022 20:49:01 -0500 Subject: DRTVWR-559: Fix broken workqueue_test.cpp. Apparently Visual Studio and Xcode disagree on the intended lifespan of a certain temporary expression. Capturing it in a named variable works. --- indra/llcommon/workqueue.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index eea8886a7a..5461ce6c23 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -419,8 +419,8 @@ namespace LL // moved-from. try { - std::dynamic_pointer_cast(mTarget.lock())-> - post(std::move(*this), mStart); + auto target{ std::dynamic_pointer_cast(mTarget.lock()) }; + target->post(std::move(*this), mStart); } catch (const Closed&) { -- cgit v1.2.3 From 274da636a00fd0469f7857dad7995cb11552e4ab Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Mon, 23 Jan 2023 11:48:43 -0600 Subject: SL-18869 Followup -- AMD optimization pass. --- indra/llcommon/llprofiler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index c03b2e93f4..394fbd05f6 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -87,7 +87,7 @@ extern thread_local bool gProfilerEnabled; #include "Tracy.hpp" // Enable OpenGL profiling - #define LL_PROFILER_ENABLE_TRACY_OPENGL 1 + #define LL_PROFILER_ENABLE_TRACY_OPENGL 0 // Enable RenderDoc labeling #define LL_PROFILER_ENABLE_RENDER_DOC 0 -- cgit v1.2.3 From 473ade269628d1fb6cbc7b96a91e1c0aec1b8e59 Mon Sep 17 00:00:00 2001 From: Henri Beauchamp Date: Tue, 31 Jan 2023 17:42:51 +0100 Subject: SL-19110 Fast hashing classes for use in place of the slow LLMD5, where speed matters. (#64) This commit adds the HBXX64 and HBXX128 classes for use as a drop-in replacement for the slow LLMD5 hashing class, where speed matters and backward compatibility (with standard hashing algorithms) and/or cryptographic hashing qualities are not required. It also replaces LLMD5 with HBXX* in a few existing hot (well, ok, just "warm" for some) paths meeting the above requirements, while paving the way for future use cases, such as in the DRTVWR-559 and sibling branches where the slow LLMD5 is used (e.g. to hash materials and vertex buffer cache entries), and could be use such a (way) faster algorithm with very significant benefits and no negative impact. Here is the comment I added in indra/llcommon/hbxx.h: // HBXXH* classes are to be used where speed matters and cryptographic quality // is not required (no "one-way" guarantee, though they are likely not worst in // this respect than MD5 which got busted and is now considered too weak). The // xxHash code they are built upon is vectorized and about 50 times faster than // MD5. A 64 bits hash class is also provided for when 128 bits of entropy are // not needed. The hashes collision rate is similar to MD5's. // See https://github.com/Cyan4973/xxHash#readme for details. --- indra/llcommon/CMakeLists.txt | 2 + indra/llcommon/hbxxh.cpp | 377 ++++++++++++++++++++++++++++++++++++++++++ indra/llcommon/hbxxh.h | 259 +++++++++++++++++++++++++++++ indra/llcommon/lluuid.cpp | 28 +--- 4 files changed, 646 insertions(+), 20 deletions(-) create mode 100644 indra/llcommon/hbxxh.cpp create mode 100644 indra/llcommon/hbxxh.h (limited to 'indra/llcommon') diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 991c3f70a1..fab7550c9a 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -119,6 +119,7 @@ set(llcommon_SOURCE_FILES lluriparser.cpp lluuid.cpp llworkerthread.cpp + hbxxh.cpp u64.cpp threadpool.cpp workqueue.cpp @@ -257,6 +258,7 @@ set(llcommon_HEADER_FILES llwin32headers.h llwin32headerslean.h llworkerthread.h + hbxxh.h lockstatic.h stdtypes.h stringize.h diff --git a/indra/llcommon/hbxxh.cpp b/indra/llcommon/hbxxh.cpp new file mode 100644 index 0000000000..e94581d415 --- /dev/null +++ b/indra/llcommon/hbxxh.cpp @@ -0,0 +1,377 @@ +/** + * @file hbxxh.cpp + * @brief High performances vectorized hashing based on xxHash. + * + * $LicenseInfo:firstyear=2023&license=viewergpl$ + * Second Life Viewer Source Code + * Copyright (c) 2023, Henri Beauchamp. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +// This define ensures that xxHash will be compiled within this module, with +// vectorized (*) and inlined functions (with no exported API symbol); our +// xxhash "pre-built library" package actually only contains the xxhash.h +// header (no library needed at link time). +// (*) SSE2 is normally used for x86(_64) builds, unless you enabled AVX2 +// in your build, in which case the latter would be used instead. For ARM64 +// builds, this would also automatically enable NEON vectorization. +#define XXH_INLINE_ALL +#include "xxhash.h" + +#include "hbxxh.h" + +// How many bytes to grab at a time when hashing files or streams +constexpr size_t BLOCK_LEN = 4096; + +/////////////////////////////////////////////////////////////////////////////// +// HBXXH64 class +/////////////////////////////////////////////////////////////////////////////// + +//static +U64 HBXXH64::digest(const void* buffer, size_t len) +{ + return XXH3_64bits(buffer, len); +} + +//static +U64 HBXXH64::digest(const char* str) +{ + return XXH3_64bits((const void*)str, strlen(str)); +} + +//static +U64 HBXXH64::digest(const std::string& str) +{ + return XXH3_64bits((const void*)str.c_str(), str.size()); +} + +// Must be called by all constructors. +void HBXXH64::init() +{ + mDigest = 0; + mState = (void*)XXH3_createState(); + if (!mState || XXH3_64bits_reset((XXH3_state_t*)mState) != XXH_OK) + { + LL_WARNS() << "Failed to initialize state !" << LL_ENDL; + } +} + +HBXXH64::~HBXXH64() +{ + if (mState) + { + XXH3_freeState((XXH3_state_t*)mState); + } +} + +void HBXXH64::update(const void* buffer, size_t len) +{ + if (mState) + { + XXH3_64bits_update((XXH3_state_t*)mState, buffer, len); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH64::update(const std::string& str) +{ + if (mState) + { + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)str.c_str(), + str.length()); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH64::update(std::istream& stream) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while (stream.good()) + { + stream.read(buffer, BLOCK_LEN); + len = stream.gcount(); + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } +} + +void HBXXH64::update(FILE* file) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while ((len = fread((void*)buffer, 1, BLOCK_LEN, file))) + { + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } + fclose(file); +} + +void HBXXH64::finalize() +{ + if (!mState) + { + LL_WARNS() << "Already finalized !" << LL_ENDL; + return; + } + mDigest = XXH3_64bits_digest((XXH3_state_t*)mState); + XXH3_freeState((XXH3_state_t*)mState); + mState = NULL; +} + +U64 HBXXH64::digest() const +{ + return mState ? XXH3_64bits_digest((XXH3_state_t*)mState) : mDigest; +} + +std::ostream& operator<<(std::ostream& stream, HBXXH64 context) +{ + stream << context.digest(); + return stream; +} + +/////////////////////////////////////////////////////////////////////////////// +// HBXXH128 class +/////////////////////////////////////////////////////////////////////////////// + +//static +LLUUID HBXXH128::digest(const void* buffer, size_t len) +{ + XXH128_hash_t hash = XXH3_128bits(buffer, len); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +LLUUID HBXXH128::digest(const char* str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str, strlen(str)); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +LLUUID HBXXH128::digest(const std::string& str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str.c_str(), str.size()); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +void HBXXH128::digest(LLUUID& result, const void* buffer, size_t len) +{ + XXH128_hash_t hash = XXH3_128bits(buffer, len); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +//static +void HBXXH128::digest(LLUUID& result, const char* str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str, strlen(str)); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +//static +void HBXXH128::digest(LLUUID& result, const std::string& str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str.c_str(), str.size()); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +// Must be called by all constructors. +void HBXXH128::init() +{ + mState = (void*)XXH3_createState(); + if (!mState || XXH3_128bits_reset((XXH3_state_t*)mState) != XXH_OK) + { + LL_WARNS() << "Failed to initialize state !" << LL_ENDL; + } +} + +HBXXH128::~HBXXH128() +{ + if (mState) + { + XXH3_freeState((XXH3_state_t*)mState); + } +} + +void HBXXH128::update(const void* buffer, size_t len) +{ + if (mState) + { + XXH3_128bits_update((XXH3_state_t*)mState, buffer, len); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH128::update(const std::string& str) +{ + if (mState) + { + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)str.c_str(), + str.length()); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH128::update(std::istream& stream) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while (stream.good()) + { + stream.read(buffer, BLOCK_LEN); + len = stream.gcount(); + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } +} + +void HBXXH128::update(FILE* file) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while ((len = fread((void*)buffer, 1, BLOCK_LEN, file))) + { + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } + fclose(file); +} + +void HBXXH128::finalize() +{ + if (!mState) + { + LL_WARNS() << "Already finalized !" << LL_ENDL; + return; + } + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + U64* data = (U64*)mDigest.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + XXH3_freeState((XXH3_state_t*)mState); + mState = NULL; +} + +const LLUUID& HBXXH128::digest() const +{ + if (mState) + { + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + // We cheat the const-ness of the method here, but this is OK, since + // mDigest is private and cannot be accessed indirectly by other + // methods than digest() ones, that do check for mState to decide + // wether mDigest's current value may be provided as is or not. This + // cheat saves us a temporary LLLUID copy. + U64* data = (U64*)mDigest.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + } + return mDigest; +} + +void HBXXH128::digest(LLUUID& result) const +{ + if (!mState) + { + result = mDigest; + return; + } + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +std::ostream& operator<<(std::ostream& stream, HBXXH128 context) +{ + stream << context.digest(); + return stream; +} diff --git a/indra/llcommon/hbxxh.h b/indra/llcommon/hbxxh.h new file mode 100644 index 0000000000..8a5f977648 --- /dev/null +++ b/indra/llcommon/hbxxh.h @@ -0,0 +1,259 @@ +/** + * @file hbxxh.h + * @brief High performances vectorized hashing based on xxHash. + * + * $LicenseInfo:firstyear=2023&license=viewergpl$ + * Second Life Viewer Source Code + * Copyright (c) 2023, Henri Beauchamp. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_HBXXH_H +#define LL_HBXXH_H + +#include "lluuid.h" + +// HBXXH* classes are to be used where speed matters and cryptographic quality +// is not required (no "one-way" guarantee, though they are likely not worst in +// this respect than MD5 which got busted and is now considered too weak). The +// xxHash code they are built upon is vectorized and about 50 times faster than +// MD5. A 64 bits hash class is also provided for when 128 bits of entropy are +// not needed. The hashes collision rate is similar to MD5's. +// See https://github.com/Cyan4973/xxHash#readme for details. + +// 64 bits hashing class + +class HBXXH64 +{ + friend std::ostream& operator<<(std::ostream&, HBXXH64); + +protected: + LOG_CLASS(HBXXH64); + +public: + inline HBXXH64() { init(); } + + // Constructors for special circumstances; they all digest the first passed + // parameter. Set 'do_finalize' to false if you do not want to finalize the + // context, which is useful/needed when you want to update() it afterwards. + // Ideally, the compiler should be smart enough to get our clue and + // optimize out the const bool test during inlining... + + inline HBXXH64(const void* buffer, size_t len, + const bool do_finalize = true) + { + init(); + update(buffer, len); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(const std::string& str, const bool do_finalize = true) + { + init(); + update(str); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(std::istream& s, const bool do_finalize = true) + { + init(); + update(s); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(FILE* file, const bool do_finalize = true) + { + init(); + update(file); + if (do_finalize) + { + finalize(); + } + } + + ~HBXXH64(); + + void update(const void* buffer, size_t len); + void update(const std::string& str); + void update(std::istream& s); + void update(FILE* file); + + // Note that unlike what happens with LLMD5, you do not need to finalize() + // HBXXH64 before using digest(), and you may keep updating() it even after + // you got a first digest() (the next digest would of course change after + // any update). It is still useful to use finalize() when you do not want + // to store a final digest() result in a separate U64; after this method + // has been called, digest() simply returns mDigest value. + void finalize(); + + U64 digest() const; + + // Fast static methods. Use them when hashing just one contiguous block of + // data. + static U64 digest(const void* buffer, size_t len); + static U64 digest(const char* str); // str must be NUL-terminated + static U64 digest(const std::string& str); + +private: + void init(); + +private: + // We use a void pointer to avoid including xxhash.h here for XXH3_state_t + // (which cannot either be trivially forward-declared, due to complex API + // related pre-processor macros in xxhash.h). + void* mState; + U64 mDigest; +}; + +inline bool operator==(const HBXXH64& a, const HBXXH64& b) +{ + return a.digest() == b.digest(); +} + +inline bool operator!=(const HBXXH64& a, const HBXXH64& b) +{ + return a.digest() != b.digest(); +} + +// 128 bits hashing class + +class HBXXH128 +{ + friend std::ostream& operator<<(std::ostream&, HBXXH128); + +protected: + LOG_CLASS(HBXXH128); + +public: + inline HBXXH128() { init(); } + + // Constructors for special circumstances; they all digest the first passed + // parameter. Set 'do_finalize' to false if you do not want to finalize the + // context, which is useful/needed when you want to update() it afterwards. + // Ideally, the compiler should be smart enough to get our clue and + // optimize out the const bool test during inlining... + + inline HBXXH128(const void* buffer, size_t len, + const bool do_finalize = true) + { + init(); + update(buffer, len); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(const std::string& str, const bool do_finalize = true) + { + init(); + update(str); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(std::istream& s, const bool do_finalize = true) + { + init(); + update(s); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(FILE* file, const bool do_finalize = true) + { + init(); + update(file); + if (do_finalize) + { + finalize(); + } + } + + ~HBXXH128(); + + void update(const void* buffer, size_t len); + void update(const std::string& str); + void update(std::istream& s); + void update(FILE* file); + + // Note that unlike what happens with LLMD5, you do not need to finalize() + // HBXXH128 before using digest(), and you may keep updating() it even + // after you got a first digest() (the next digest would of course change + // after any update). It is still useful to use finalize() when you do not + // want to store a final digest() result in a separate LLUUID; after this + // method has been called, digest() simply returns a reference on mDigest. + void finalize(); + + // We use an LLUUID for the digest, since this is a 128 bits wide native + // type available in the viewer code, making it easy to manipulate. It also + // allows to use HBXXH128 efficiently in LLUUID generate() and combine() + // methods. + const LLUUID& digest() const; + + // Here, we avoid an LLUUID copy whenever we already got one to store the + // result *and* we did not yet call finalize(). + void digest(LLUUID& result) const; + + // Fast static methods. Use them when hashing just one contiguous block of + // data. + static LLUUID digest(const void* buffer, size_t len); + static LLUUID digest(const char* str); // str must be NUL-terminated + static LLUUID digest(const std::string& str); + // Same as above, but saves you from an LLUUID copy when you already got + // one for storage use. + static void digest(LLUUID& result, const void* buffer, size_t len); + static void digest(LLUUID& result, const char* str); // str NUL-terminated + static void digest(LLUUID& result, const std::string& str); + +private: + void init(); + +private: + // We use a void pointer to avoid including xxhash.h here for XXH3_state_t + // (which cannot either be trivially forward-declared, due to complex API + // related pre-processor macros in xxhash.h). + void* mState; + LLUUID mDigest; +}; + +inline bool operator==(const HBXXH128& a, const HBXXH128& b) +{ + return a.digest() == b.digest(); +} + +inline bool operator!=(const HBXXH128& a, const HBXXH128& b) +{ + return a.digest() != b.digest(); +} + +#endif // LL_HBXXH_H diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index acce8366ea..8ff6c45760 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -40,11 +40,11 @@ #include "lluuid.h" #include "llerror.h" #include "llrand.h" -#include "llmd5.h" #include "llstring.h" #include "lltimer.h" #include "llthread.h" #include "llmutex.h" +#include "hbxxh.h" const LLUUID LLUUID::null; const LLTransactionID LLTransactionID::tnull; @@ -402,11 +402,9 @@ LLUUID LLUUID::operator^(const LLUUID& rhs) const void LLUUID::combine(const LLUUID& other, LLUUID& result) const { - LLMD5 md5_uuid; - md5_uuid.update((unsigned char*)mData, 16); - md5_uuid.update((unsigned char*)other.mData, 16); - md5_uuid.finalize(); - md5_uuid.raw_digest(result.mData); + HBXXH128 hash((const void*)mData, 16, false); // false = do not finalize + hash.update((const void*)other.mData, 16); + hash.digest(result); } LLUUID LLUUID::combine(const LLUUID &other) const @@ -857,17 +855,12 @@ void LLUUID::generate() tmp >>= 8; mData[8] = (unsigned char) tmp; - LLMD5 md5_uuid; - - md5_uuid.update(mData,16); - md5_uuid.finalize(); - md5_uuid.raw_digest(mData); + HBXXH128::digest(*this, (const void*)mData, 16); } void LLUUID::generate(const std::string& hash_string) { - LLMD5 md5_uuid((U8*)hash_string.c_str()); - md5_uuid.raw_digest(mData); + HBXXH128::digest(*this, hash_string); } U32 LLUUID::getRandomSeed() @@ -885,13 +878,8 @@ U32 LLUUID::getRandomSeed() seed[7]=(unsigned char)(pid); getSystemTime((uuid_time_t *)(&seed[8])); - LLMD5 md5_seed; - - md5_seed.update(seed,16); - md5_seed.finalize(); - md5_seed.raw_digest(seed); - - return(*(U32 *)seed); + U64 seed64 = HBXXH64((const void*)seed, 16).digest(); + return U32(seed64) ^ U32(seed64 >> 32); } BOOL LLUUID::parseUUID(const std::string& buf, LLUUID* value) -- cgit v1.2.3 From b5b0a3fbf6b58c7f1a7e4377bfeb6adb3337e4bf Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Tue, 31 Jan 2023 18:48:16 +0200 Subject: SL-19110 Fix xxhash build link and properly register contribution --- indra/llcommon/hbxxh.cpp | 4 ++-- indra/llcommon/hbxxh.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/hbxxh.cpp b/indra/llcommon/hbxxh.cpp index e94581d415..388269d6c8 100644 --- a/indra/llcommon/hbxxh.cpp +++ b/indra/llcommon/hbxxh.cpp @@ -2,7 +2,7 @@ * @file hbxxh.cpp * @brief High performances vectorized hashing based on xxHash. * - * $LicenseInfo:firstyear=2023&license=viewergpl$ + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (c) 2023, Henri Beauchamp. * @@ -34,7 +34,7 @@ // in your build, in which case the latter would be used instead. For ARM64 // builds, this would also automatically enable NEON vectorization. #define XXH_INLINE_ALL -#include "xxhash.h" +#include "xxhash/xxhash.h" #include "hbxxh.h" diff --git a/indra/llcommon/hbxxh.h b/indra/llcommon/hbxxh.h index 8a5f977648..236716722a 100644 --- a/indra/llcommon/hbxxh.h +++ b/indra/llcommon/hbxxh.h @@ -2,7 +2,7 @@ * @file hbxxh.h * @brief High performances vectorized hashing based on xxHash. * - * $LicenseInfo:firstyear=2023&license=viewergpl$ + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (c) 2023, Henri Beauchamp. * -- cgit v1.2.3 From 3561e9b5d5a7fef8e6a710c7b6f9b13243a4d51c Mon Sep 17 00:00:00 2001 From: Henri Beauchamp Date: Mon, 13 Feb 2023 18:34:00 +0100 Subject: SL-19110 revert LLUUID::combine() to old algorithm to match server code. (#75) As it happens, the change in the LLUUID::combine() algorithm introduced by one of my previous commits is causing invalid assets creation (seen with some clothing items, such as Shape and Universal types); obviously, the server is using the old algorithm for UUID validation purpose of these assets. This commit reverts LLUUID::combine() code to use LLMD5. --- indra/llcommon/lluuid.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index 47c6bc71c8..fc04dca08d 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -44,6 +44,7 @@ #include "lltimer.h" #include "llthread.h" #include "llmutex.h" +#include "llmd5.h" #include "hbxxh.h" const LLUUID LLUUID::null; @@ -400,11 +401,16 @@ LLUUID LLUUID::operator^(const LLUUID& rhs) const return id; } +// WARNING: this algorithm SHALL NOT be changed. It is also used by the server +// and plays a role in some assets validation (e.g. clothing items). Changing +// it would cause invalid assets. void LLUUID::combine(const LLUUID& other, LLUUID& result) const { - HBXXH128 hash((const void*)mData, 16, false); // false = do not finalize - hash.update((const void*)other.mData, 16); - hash.digest(result); + LLMD5 md5_uuid; + md5_uuid.update((unsigned char*)mData, 16); + md5_uuid.update((unsigned char*)other.mData, 16); + md5_uuid.finalize(); + md5_uuid.raw_digest(result.mData); } LLUUID LLUUID::combine(const LLUUID &other) const -- cgit v1.2.3 From 81f2d5b1813288fac77c5d2884fe16aaa00709f3 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Mon, 6 Mar 2023 10:58:46 -0600 Subject: DRTVWR-559 Return to MD5 for LLUUID operations. Existing LLUUID API implementations must not change, but may add "fast" variants where appropriate in the future. --- indra/llcommon/lluuid.cpp | 1443 +++++++++++++++++++++++---------------------- 1 file changed, 725 insertions(+), 718 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index fc04dca08d..3619a64ea9 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -1,31 +1,31 @@ -/** +/** * @file lluuid.cpp * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #include "linden_common.h" -// We can't use WIN32_LEAN_AND_MEAN here, needs lots of includes. + // We can't use WIN32_LEAN_AND_MEAN here, needs lots of includes. #if LL_WINDOWS #include "llwin32headers.h" // ugh, this is ugly. We need to straighten out our linking for this library @@ -40,18 +40,17 @@ #include "lluuid.h" #include "llerror.h" #include "llrand.h" +#include "llmd5.h" #include "llstring.h" #include "lltimer.h" #include "llthread.h" #include "llmutex.h" -#include "llmd5.h" -#include "hbxxh.h" const LLUUID LLUUID::null; const LLTransactionID LLTransactionID::tnull; // static -LLMutex * LLUUID::mMutex = NULL; +LLMutex* LLUUID::mMutex = NULL; @@ -60,15 +59,15 @@ LLMutex * LLUUID::mMutex = NULL; NOT DONE YET!!! static char BASE85_TABLE[] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', - 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', - 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', - 'y', 'z', '!', '#', '$', '%', '&', '(', ')', '*', - '+', '-', ';', '[', '=', '>', '?', '@', '^', '_', - '`', '{', '|', '}', '~', '\0' + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', + 'y', 'z', '!', '#', '$', '%', '&', '(', ')', '*', + '+', '-', ';', '[', '=', '>', '?', '@', '^', '_', + '`', '{', '|', '}', '~', '\0' }; @@ -94,28 +93,28 @@ return ret; void LLUUID::toBase85(char* out) { - U32* me = (U32*)&(mData[0]); - for(S32 i = 0; i < 4; ++i) - { - char* o = &out[i*i]; - for(S32 j = 0; j < 5; ++j) - { - o[4-j] = BASE85_TABLE[ me[i] % 85]; - word /= 85; - } - } + U32* me = (U32*)&(mData[0]); + for(S32 i = 0; i < 4; ++i) + { + char* o = &out[i*i]; + for(S32 j = 0; j < 5; ++j) + { + o[4-j] = BASE85_TABLE[ me[i] % 85]; + word /= 85; + } + } } unsigned int decode( char const * fiveChars ) throw( bad_input_data ) { - unsigned int ret = 0; - for( S32 ix = 0; ix < 5; ++ix ) - { - char * s = strchr( encodeTable, fiveChars[ ix ] ); - ret = ret * 85 + (s-encodeTable); - } - return ret; -} + unsigned int ret = 0; + for( S32 ix = 0; ix < 5; ++ix ) + { + char * s = strchr( encodeTable, fiveChars[ ix ] ); + ret = ret * 85 + (s-encodeTable); + } + return ret; +} */ #define LL_USE_JANKY_RANDOM_NUMBER_GENERATOR 0 @@ -130,8 +129,8 @@ static U64 sJankyRandomSeed(LLUUID::getRandomSeed()); */ U32 janky_fast_random_bytes() { - sJankyRandomSeed = U64L(1664525) * sJankyRandomSeed + U64L(1013904223); - return (U32)sJankyRandomSeed; + sJankyRandomSeed = U64L(1664525) * sJankyRandomSeed + U64L(1013904223); + return (U32)sJankyRandomSeed; } /** @@ -139,8 +138,8 @@ U32 janky_fast_random_bytes() */ U32 janky_fast_random_byes_range(U32 val) { - sJankyRandomSeed = U64L(1664525) * sJankyRandomSeed + U64L(1013904223); - return (U32)(sJankyRandomSeed) % val; + sJankyRandomSeed = U64L(1664525) * sJankyRandomSeed + U64L(1013904223); + return (U32)(sJankyRandomSeed) % val; } /** @@ -148,365 +147,362 @@ U32 janky_fast_random_byes_range(U32 val) */ U32 janky_fast_random_seeded_bytes(U32 seed, U32 val) { - seed = U64L(1664525) * (U64)(seed) + U64L(1013904223); - return (U32)(seed) % val; + seed = U64L(1664525) * (U64)(seed)+U64L(1013904223); + return (U32)(seed) % val; } #endif // Common to all UUID implementations void LLUUID::toString(std::string& out) const { - out = llformat( - "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - (U8)(mData[0]), - (U8)(mData[1]), - (U8)(mData[2]), - (U8)(mData[3]), - (U8)(mData[4]), - (U8)(mData[5]), - (U8)(mData[6]), - (U8)(mData[7]), - (U8)(mData[8]), - (U8)(mData[9]), - (U8)(mData[10]), - (U8)(mData[11]), - (U8)(mData[12]), - (U8)(mData[13]), - (U8)(mData[14]), - (U8)(mData[15])); + out = llformat( + "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + (U8)(mData[0]), + (U8)(mData[1]), + (U8)(mData[2]), + (U8)(mData[3]), + (U8)(mData[4]), + (U8)(mData[5]), + (U8)(mData[6]), + (U8)(mData[7]), + (U8)(mData[8]), + (U8)(mData[9]), + (U8)(mData[10]), + (U8)(mData[11]), + (U8)(mData[12]), + (U8)(mData[13]), + (U8)(mData[14]), + (U8)(mData[15])); } // *TODO: deprecate -void LLUUID::toString(char *out) const +void LLUUID::toString(char* out) const { - std::string buffer; - toString(buffer); - strcpy(out,buffer.c_str()); /* Flawfinder: ignore */ + std::string buffer; + toString(buffer); + strcpy(out, buffer.c_str()); /* Flawfinder: ignore */ } void LLUUID::toCompressedString(std::string& out) const { - char bytes[UUID_BYTES+1]; - memcpy(bytes, mData, UUID_BYTES); /* Flawfinder: ignore */ - bytes[UUID_BYTES] = '\0'; - out.assign(bytes, UUID_BYTES); + char bytes[UUID_BYTES + 1]; + memcpy(bytes, mData, UUID_BYTES); /* Flawfinder: ignore */ + bytes[UUID_BYTES] = '\0'; + out.assign(bytes, UUID_BYTES); } // *TODO: deprecate -void LLUUID::toCompressedString(char *out) const +void LLUUID::toCompressedString(char* out) const { - memcpy(out, mData, UUID_BYTES); /* Flawfinder: ignore */ - out[UUID_BYTES] = '\0'; + memcpy(out, mData, UUID_BYTES); /* Flawfinder: ignore */ + out[UUID_BYTES] = '\0'; } std::string LLUUID::getString() const { - return asString(); + return asString(); } std::string LLUUID::asString() const { - std::string str; - toString(str); - return str; + std::string str; + toString(str); + return str; } BOOL LLUUID::set(const char* in_string, BOOL emit) { - return set(ll_safe_string(in_string),emit); + return set(ll_safe_string(in_string), emit); } BOOL LLUUID::set(const std::string& in_string, BOOL emit) { - BOOL broken_format = FALSE; - - // empty strings should make NULL uuid - if (in_string.empty()) - { - setNull(); - return TRUE; - } - - if (in_string.length() != (UUID_STR_LENGTH - 1)) /* Flawfinder: ignore */ - { - // I'm a moron. First implementation didn't have the right UUID format. - // Shouldn't see any of these any more - if (in_string.length() == (UUID_STR_LENGTH - 2)) /* Flawfinder: ignore */ - { - if(emit) - { - LL_WARNS() << "Warning! Using broken UUID string format" << LL_ENDL; - } - broken_format = TRUE; - } - else - { - // Bad UUID string. Spam as INFO, as most cases we don't care. - if(emit) - { - //don't spam the logs because a resident can't spell. - LL_WARNS() << "Bad UUID string: " << in_string << LL_ENDL; - } - setNull(); - return FALSE; - } - } - - U8 cur_pos = 0; - S32 i; - for (i = 0; i < UUID_BYTES; i++) - { - if ((i == 4) || (i == 6) || (i == 8) || (i == 10)) - { - cur_pos++; - if (broken_format && (i==10)) - { - // Missing - in the broken format - cur_pos--; - } - } - - mData[i] = 0; - - if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) - { - mData[i] += (U8)(in_string[cur_pos] - '0'); - } - else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <='f')) - { - mData[i] += (U8)(10 + in_string[cur_pos] - 'a'); - } - else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <='F')) - { - mData[i] += (U8)(10 + in_string[cur_pos] - 'A'); - } - else - { - if(emit) - { - LL_WARNS() << "Invalid UUID string character" << LL_ENDL; - } - setNull(); - return FALSE; - } - - mData[i] = mData[i] << 4; - cur_pos++; - - if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) - { - mData[i] += (U8)(in_string[cur_pos] - '0'); - } - else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <='f')) - { - mData[i] += (U8)(10 + in_string[cur_pos] - 'a'); - } - else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <='F')) - { - mData[i] += (U8)(10 + in_string[cur_pos] - 'A'); - } - else - { - if(emit) - { - LL_WARNS() << "Invalid UUID string character" << LL_ENDL; - } - setNull(); - return FALSE; - } - cur_pos++; - } - - return TRUE; + BOOL broken_format = FALSE; + + // empty strings should make NULL uuid + if (in_string.empty()) + { + setNull(); + return TRUE; + } + + if (in_string.length() != (UUID_STR_LENGTH - 1)) /* Flawfinder: ignore */ + { + // I'm a moron. First implementation didn't have the right UUID format. + // Shouldn't see any of these any more + if (in_string.length() == (UUID_STR_LENGTH - 2)) /* Flawfinder: ignore */ + { + if (emit) + { + LL_WARNS() << "Warning! Using broken UUID string format" << LL_ENDL; + } + broken_format = TRUE; + } + else + { + // Bad UUID string. Spam as INFO, as most cases we don't care. + if (emit) + { + //don't spam the logs because a resident can't spell. + LL_WARNS() << "Bad UUID string: " << in_string << LL_ENDL; + } + setNull(); + return FALSE; + } + } + + U8 cur_pos = 0; + S32 i; + for (i = 0; i < UUID_BYTES; i++) + { + if ((i == 4) || (i == 6) || (i == 8) || (i == 10)) + { + cur_pos++; + if (broken_format && (i == 10)) + { + // Missing - in the broken format + cur_pos--; + } + } + + mData[i] = 0; + + if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) + { + mData[i] += (U8)(in_string[cur_pos] - '0'); + } + else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <= 'f')) + { + mData[i] += (U8)(10 + in_string[cur_pos] - 'a'); + } + else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <= 'F')) + { + mData[i] += (U8)(10 + in_string[cur_pos] - 'A'); + } + else + { + if (emit) + { + LL_WARNS() << "Invalid UUID string character" << LL_ENDL; + } + setNull(); + return FALSE; + } + + mData[i] = mData[i] << 4; + cur_pos++; + + if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) + { + mData[i] += (U8)(in_string[cur_pos] - '0'); + } + else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <= 'f')) + { + mData[i] += (U8)(10 + in_string[cur_pos] - 'a'); + } + else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <= 'F')) + { + mData[i] += (U8)(10 + in_string[cur_pos] - 'A'); + } + else + { + if (emit) + { + LL_WARNS() << "Invalid UUID string character" << LL_ENDL; + } + setNull(); + return FALSE; + } + cur_pos++; + } + + return TRUE; } BOOL LLUUID::validate(const std::string& in_string) { - BOOL broken_format = FALSE; - if (in_string.length() != (UUID_STR_LENGTH - 1)) /* Flawfinder: ignore */ - { - // I'm a moron. First implementation didn't have the right UUID format. - if (in_string.length() == (UUID_STR_LENGTH - 2)) /* Flawfinder: ignore */ - { - broken_format = TRUE; - } - else - { - return FALSE; - } - } - - U8 cur_pos = 0; - for (U32 i = 0; i < 16; i++) - { - if ((i == 4) || (i == 6) || (i == 8) || (i == 10)) - { - cur_pos++; - if (broken_format && (i==10)) - { - // Missing - in the broken format - cur_pos--; - } - } - - if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) - { - } - else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <='f')) - { - } - else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <='F')) - { - } - else - { - return FALSE; - } - - cur_pos++; - - if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) - { - } - else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <='f')) - { - } - else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <='F')) - { - } - else - { - return FALSE; - } - cur_pos++; - } - return TRUE; + BOOL broken_format = FALSE; + if (in_string.length() != (UUID_STR_LENGTH - 1)) /* Flawfinder: ignore */ + { + // I'm a moron. First implementation didn't have the right UUID format. + if (in_string.length() == (UUID_STR_LENGTH - 2)) /* Flawfinder: ignore */ + { + broken_format = TRUE; + } + else + { + return FALSE; + } + } + + U8 cur_pos = 0; + for (U32 i = 0; i < 16; i++) + { + if ((i == 4) || (i == 6) || (i == 8) || (i == 10)) + { + cur_pos++; + if (broken_format && (i == 10)) + { + // Missing - in the broken format + cur_pos--; + } + } + + if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) + { + } + else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <= 'f')) + { + } + else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <= 'F')) + { + } + else + { + return FALSE; + } + + cur_pos++; + + if ((in_string[cur_pos] >= '0') && (in_string[cur_pos] <= '9')) + { + } + else if ((in_string[cur_pos] >= 'a') && (in_string[cur_pos] <= 'f')) + { + } + else if ((in_string[cur_pos] >= 'A') && (in_string[cur_pos] <= 'F')) + { + } + else + { + return FALSE; + } + cur_pos++; + } + return TRUE; } const LLUUID& LLUUID::operator^=(const LLUUID& rhs) { - U32* me = (U32*)&(mData[0]); - const U32* other = (U32*)&(rhs.mData[0]); - for(S32 i = 0; i < 4; ++i) - { - me[i] = me[i] ^ other[i]; - } - return *this; + U32* me = (U32*)&(mData[0]); + const U32* other = (U32*)&(rhs.mData[0]); + for (S32 i = 0; i < 4; ++i) + { + me[i] = me[i] ^ other[i]; + } + return *this; } LLUUID LLUUID::operator^(const LLUUID& rhs) const { - LLUUID id(*this); - id ^= rhs; - return id; + LLUUID id(*this); + id ^= rhs; + return id; } -// WARNING: this algorithm SHALL NOT be changed. It is also used by the server -// and plays a role in some assets validation (e.g. clothing items). Changing -// it would cause invalid assets. void LLUUID::combine(const LLUUID& other, LLUUID& result) const { - LLMD5 md5_uuid; - md5_uuid.update((unsigned char*)mData, 16); - md5_uuid.update((unsigned char*)other.mData, 16); - md5_uuid.finalize(); - md5_uuid.raw_digest(result.mData); + LLMD5 md5_uuid; + md5_uuid.update((unsigned char*)mData, 16); + md5_uuid.update((unsigned char*)other.mData, 16); + md5_uuid.finalize(); + md5_uuid.raw_digest(result.mData); } -LLUUID LLUUID::combine(const LLUUID &other) const +LLUUID LLUUID::combine(const LLUUID& other) const { - LLUUID combination; - combine(other, combination); - return combination; + LLUUID combination; + combine(other, combination); + return combination; } -std::ostream& operator<<(std::ostream& s, const LLUUID &uuid) +std::ostream& operator<<(std::ostream& s, const LLUUID& uuid) { - std::string uuid_str; - uuid.toString(uuid_str); - s << uuid_str; - return s; + std::string uuid_str; + uuid.toString(uuid_str); + s << uuid_str; + return s; } -std::istream& operator>>(std::istream &s, LLUUID &uuid) +std::istream& operator>>(std::istream& s, LLUUID& uuid) { - U32 i; - char uuid_str[UUID_STR_LENGTH]; /* Flawfinder: ignore */ - for (i = 0; i < UUID_STR_LENGTH-1; i++) - { - s >> uuid_str[i]; - } - uuid_str[i] = '\0'; - uuid.set(std::string(uuid_str)); - return s; + U32 i; + char uuid_str[UUID_STR_LENGTH]; /* Flawfinder: ignore */ + for (i = 0; i < UUID_STR_LENGTH - 1; i++) + { + s >> uuid_str[i]; + } + uuid_str[i] = '\0'; + uuid.set(std::string(uuid_str)); + return s; } -static void get_random_bytes(void *buf, int nbytes) +static void get_random_bytes(void* buf, int nbytes) { - int i; - char *cp = (char *) buf; - - // *NOTE: If we are not using the janky generator ll_rand() - // generates at least 3 good bytes of data since it is 0 to - // RAND_MAX. This could be made more efficient by copying all the - // bytes. - for (i=0; i < nbytes; i++) + int i; + char* cp = (char*)buf; + + // *NOTE: If we are not using the janky generator ll_rand() + // generates at least 3 good bytes of data since it is 0 to + // RAND_MAX. This could be made more efficient by copying all the + // bytes. + for (i = 0; i < nbytes; i++) #if LL_USE_JANKY_RANDOM_NUMBER_GENERATOR - *cp++ = janky_fast_random_bytes() & 0xFF; + * cp++ = janky_fast_random_bytes() & 0xFF; #else - *cp++ = ll_rand() & 0xFF; + * cp++ = ll_rand() & 0xFF; #endif - return; + return; } #if LL_WINDOWS typedef struct _ASTAT_ { - ADAPTER_STATUS adapt; - NAME_BUFFER NameBuff [30]; + ADAPTER_STATUS adapt; + NAME_BUFFER NameBuff[30]; }ASTAT, * PASTAT; // static -S32 LLUUID::getNodeID(unsigned char *node_id) +S32 LLUUID::getNodeID(unsigned char* node_id) { - ASTAT Adapter; - NCB Ncb; - UCHAR uRetCode; - LANA_ENUM lenum; - int i; - int retval = 0; - - memset( &Ncb, 0, sizeof(Ncb) ); - Ncb.ncb_command = NCBENUM; - Ncb.ncb_buffer = (UCHAR *)&lenum; - Ncb.ncb_length = sizeof(lenum); - uRetCode = Netbios( &Ncb ); - - for(i=0; i < lenum.length ;i++) - { - memset( &Ncb, 0, sizeof(Ncb) ); - Ncb.ncb_command = NCBRESET; - Ncb.ncb_lana_num = lenum.lana[i]; - - uRetCode = Netbios( &Ncb ); - - memset( &Ncb, 0, sizeof (Ncb) ); - Ncb.ncb_command = NCBASTAT; - Ncb.ncb_lana_num = lenum.lana[i]; - - strcpy( (char *)Ncb.ncb_callname, "* " ); /* Flawfinder: ignore */ - Ncb.ncb_buffer = (unsigned char *)&Adapter; - Ncb.ncb_length = sizeof(Adapter); - - uRetCode = Netbios( &Ncb ); - if ( uRetCode == 0 ) - { - memcpy(node_id,Adapter.adapt.adapter_address,6); /* Flawfinder: ignore */ - retval = 1; - } - } - return retval; + ASTAT Adapter; + NCB Ncb; + UCHAR uRetCode; + LANA_ENUM lenum; + int i; + int retval = 0; + + memset(&Ncb, 0, sizeof(Ncb)); + Ncb.ncb_command = NCBENUM; + Ncb.ncb_buffer = (UCHAR*)&lenum; + Ncb.ncb_length = sizeof(lenum); + uRetCode = Netbios(&Ncb); + + for (i = 0; i < lenum.length; i++) + { + memset(&Ncb, 0, sizeof(Ncb)); + Ncb.ncb_command = NCBRESET; + Ncb.ncb_lana_num = lenum.lana[i]; + + uRetCode = Netbios(&Ncb); + + memset(&Ncb, 0, sizeof(Ncb)); + Ncb.ncb_command = NCBASTAT; + Ncb.ncb_lana_num = lenum.lana[i]; + + strcpy((char*)Ncb.ncb_callname, "* "); /* Flawfinder: ignore */ + Ncb.ncb_buffer = (unsigned char*)&Adapter; + Ncb.ncb_length = sizeof(Adapter); + + uRetCode = Netbios(&Ncb); + if (uRetCode == 0) + { + memcpy(node_id, Adapter.adapt.adapter_address, 6); /* Flawfinder: ignore */ + retval = 1; + } + } + return retval; } #elif LL_DARWIN @@ -525,66 +521,66 @@ S32 LLUUID::getNodeID(unsigned char *node_id) #include #include -// static -S32 LLUUID::getNodeID(unsigned char *node_id) + // static +S32 LLUUID::getNodeID(unsigned char* node_id) { - int i; - unsigned char *a = NULL; - struct ifaddrs *ifap, *ifa; - int rv; - S32 result = 0; - - if ((rv=getifaddrs(&ifap))==-1) - { - return -1; - } - if (ifap == NULL) - { - return -1; - } - - for (ifa = ifap; ifa != NULL; ifa = ifa->ifa_next) - { -// printf("Interface %s, address family %d, ", ifa->ifa_name, ifa->ifa_addr->sa_family); - for(i=0; i< ifa->ifa_addr->sa_len; i++) - { -// printf("%02X ", (unsigned char)ifa->ifa_addr->sa_data[i]); - } -// printf("\n"); - - if(ifa->ifa_addr->sa_family == AF_LINK) - { - // This is a link-level address - struct sockaddr_dl *lla = (struct sockaddr_dl *)ifa->ifa_addr; - -// printf("\tLink level address, type %02X\n", lla->sdl_type); - - if(lla->sdl_type == IFT_ETHER) - { - // Use the first ethernet MAC in the list. - // For some reason, the macro LLADDR() defined in net/if_dl.h doesn't expand correctly. This is what it would do. - a = (unsigned char *)&((lla)->sdl_data); - a += (lla)->sdl_nlen; - - if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5]) - { - continue; - } - - if (node_id) - { - memcpy(node_id, a, 6); - result = 1; - } - - // We found one. - break; - } - } - } - freeifaddrs(ifap); - - return result; + int i; + unsigned char* a = NULL; + struct ifaddrs* ifap, * ifa; + int rv; + S32 result = 0; + + if ((rv = getifaddrs(&ifap)) == -1) + { + return -1; + } + if (ifap == NULL) + { + return -1; + } + + for (ifa = ifap; ifa != NULL; ifa = ifa->ifa_next) + { + // printf("Interface %s, address family %d, ", ifa->ifa_name, ifa->ifa_addr->sa_family); + for (i = 0; i < ifa->ifa_addr->sa_len; i++) + { + // printf("%02X ", (unsigned char)ifa->ifa_addr->sa_data[i]); + } + // printf("\n"); + + if (ifa->ifa_addr->sa_family == AF_LINK) + { + // This is a link-level address + struct sockaddr_dl* lla = (struct sockaddr_dl*)ifa->ifa_addr; + + // printf("\tLink level address, type %02X\n", lla->sdl_type); + + if (lla->sdl_type == IFT_ETHER) + { + // Use the first ethernet MAC in the list. + // For some reason, the macro LLADDR() defined in net/if_dl.h doesn't expand correctly. This is what it would do. + a = (unsigned char*)&((lla)->sdl_data); + a += (lla)->sdl_nlen; + + if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5]) + { + continue; + } + + if (node_id) + { + memcpy(node_id, a, 6); + result = 1; + } + + // We found one. + break; + } + } + } + freeifaddrs(ifap); + + return result; } #else @@ -611,22 +607,22 @@ S32 LLUUID::getNodeID(unsigned char *node_id) #endif #endif -// static -S32 LLUUID::getNodeID(unsigned char *node_id) + // static +S32 LLUUID::getNodeID(unsigned char* node_id) { - int sd; - struct ifreq ifr, *ifrp; - struct ifconf ifc; - char buf[1024]; - int n, i; - unsigned char *a; - -/* - * BSD 4.4 defines the size of an ifreq to be - * max(sizeof(ifreq), sizeof(ifreq.ifr_name)+ifreq.ifr_addr.sa_len - * However, under earlier systems, sa_len isn't present, so the size is - * just sizeof(struct ifreq) - */ + int sd; + struct ifreq ifr, * ifrp; + struct ifconf ifc; + char buf[1024]; + int n, i; + unsigned char* a; + + /* + * BSD 4.4 defines the size of an ifreq to be + * max(sizeof(ifreq), sizeof(ifreq.ifr_name)+ifreq.ifr_addr.sa_len + * However, under earlier systems, sa_len isn't present, so the size is + * just sizeof(struct ifreq) + */ #ifdef HAVE_SA_LEN #ifndef max #define max(a,b) ((a) > (b) ? (a) : (b)) @@ -637,345 +633,356 @@ S32 LLUUID::getNodeID(unsigned char *node_id) #define ifreq_size(i) sizeof(struct ifreq) #endif /* HAVE_SA_LEN*/ - sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); - if (sd < 0) { - return -1; - } - memset(buf, 0, sizeof(buf)); - ifc.ifc_len = sizeof(buf); - ifc.ifc_buf = buf; - if (ioctl (sd, SIOCGIFCONF, (char *)&ifc) < 0) { - close(sd); - return -1; - } - n = ifc.ifc_len; - for (i = 0; i < n; i+= ifreq_size(*ifr) ) { - ifrp = (struct ifreq *)((char *) ifc.ifc_buf+i); - strncpy(ifr.ifr_name, ifrp->ifr_name, IFNAMSIZ); /* Flawfinder: ignore */ + sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); + if (sd < 0) { + return -1; + } + memset(buf, 0, sizeof(buf)); + ifc.ifc_len = sizeof(buf); + ifc.ifc_buf = buf; + if (ioctl(sd, SIOCGIFCONF, (char*)&ifc) < 0) { + close(sd); + return -1; + } + n = ifc.ifc_len; + for (i = 0; i < n; i += ifreq_size(*ifr)) { + ifrp = (struct ifreq*)((char*)ifc.ifc_buf + i); + strncpy(ifr.ifr_name, ifrp->ifr_name, IFNAMSIZ); /* Flawfinder: ignore */ #ifdef SIOCGIFHWADDR - if (ioctl(sd, SIOCGIFHWADDR, &ifr) < 0) - continue; - a = (unsigned char *) &ifr.ifr_hwaddr.sa_data; + if (ioctl(sd, SIOCGIFHWADDR, &ifr) < 0) + continue; + a = (unsigned char*)&ifr.ifr_hwaddr.sa_data; #else #ifdef SIOCGENADDR - if (ioctl(sd, SIOCGENADDR, &ifr) < 0) - continue; - a = (unsigned char *) ifr.ifr_enaddr; + if (ioctl(sd, SIOCGENADDR, &ifr) < 0) + continue; + a = (unsigned char*)ifr.ifr_enaddr; #else - /* - * XXX we don't have a way of getting the hardware - * address - */ - close(sd); - return 0; + /* + * XXX we don't have a way of getting the hardware + * address + */ + close(sd); + return 0; #endif /* SIOCGENADDR */ #endif /* SIOCGIFHWADDR */ - if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5]) - continue; - if (node_id) { - memcpy(node_id, a, 6); /* Flawfinder: ignore */ - close(sd); - return 1; - } - } - close(sd); - return 0; + if (!a[0] && !a[1] && !a[2] && !a[3] && !a[4] && !a[5]) + continue; + if (node_id) { + memcpy(node_id, a, 6); /* Flawfinder: ignore */ + close(sd); + return 1; + } + } + close(sd); + return 0; } #endif -S32 LLUUID::cmpTime(uuid_time_t *t1, uuid_time_t *t2) +S32 LLUUID::cmpTime(uuid_time_t* t1, uuid_time_t* t2) { - // Compare two time values. + // Compare two time values. - if (t1->high < t2->high) return -1; - if (t1->high > t2->high) return 1; - if (t1->low < t2->low) return -1; - if (t1->low > t2->low) return 1; - return 0; + if (t1->high < t2->high) return -1; + if (t1->high > t2->high) return 1; + if (t1->low < t2->low) return -1; + if (t1->low > t2->low) return 1; + return 0; } -void LLUUID::getSystemTime(uuid_time_t *timestamp) +void LLUUID::getSystemTime(uuid_time_t* timestamp) { - // Get system time with 100ns precision. Time is since Oct 15, 1582. + // Get system time with 100ns precision. Time is since Oct 15, 1582. #if LL_WINDOWS - ULARGE_INTEGER time; - GetSystemTimeAsFileTime((FILETIME *)&time); - // NT keeps time in FILETIME format which is 100ns ticks since - // Jan 1, 1601. UUIDs use time in 100ns ticks since Oct 15, 1582. - // The difference is 17 Days in Oct + 30 (Nov) + 31 (Dec) - // + 18 years and 5 leap days. - time.QuadPart += - (unsigned __int64) (1000*1000*10) // seconds - * (unsigned __int64) (60 * 60 * 24) // days - * (unsigned __int64) (17+30+31+365*18+5); // # of days - - timestamp->high = time.HighPart; - timestamp->low = time.LowPart; + ULARGE_INTEGER time; + GetSystemTimeAsFileTime((FILETIME*)&time); + // NT keeps time in FILETIME format which is 100ns ticks since + // Jan 1, 1601. UUIDs use time in 100ns ticks since Oct 15, 1582. + // The difference is 17 Days in Oct + 30 (Nov) + 31 (Dec) + // + 18 years and 5 leap days. + time.QuadPart += + (unsigned __int64)(1000 * 1000 * 10) // seconds + * (unsigned __int64)(60 * 60 * 24) // days + * (unsigned __int64)(17 + 30 + 31 + 365 * 18 + 5); // # of days + + timestamp->high = time.HighPart; + timestamp->low = time.LowPart; #else - struct timeval tp; - gettimeofday(&tp, 0); - - // Offset between UUID formatted times and Unix formatted times. - // UUID UTC base time is October 15, 1582. - // Unix base time is January 1, 1970. - U64 uuid_time = ((U64)tp.tv_sec * 10000000) + (tp.tv_usec * 10) + - U64L(0x01B21DD213814000); - timestamp->high = (U32) (uuid_time >> 32); - timestamp->low = (U32) (uuid_time & 0xFFFFFFFF); + struct timeval tp; + gettimeofday(&tp, 0); + + // Offset between UUID formatted times and Unix formatted times. + // UUID UTC base time is October 15, 1582. + // Unix base time is January 1, 1970. + U64 uuid_time = ((U64)tp.tv_sec * 10000000) + (tp.tv_usec * 10) + + U64L(0x01B21DD213814000); + timestamp->high = (U32)(uuid_time >> 32); + timestamp->low = (U32)(uuid_time & 0xFFFFFFFF); #endif } -void LLUUID::getCurrentTime(uuid_time_t *timestamp) +void LLUUID::getCurrentTime(uuid_time_t* timestamp) { - // Get current time as 60 bit 100ns ticks since whenever. - // Compensate for the fact that real clock resolution is less - // than 100ns. - - const U32 uuids_per_tick = 1024; - - static uuid_time_t time_last; - static U32 uuids_this_tick; - static BOOL init = FALSE; - - if (!init) { - getSystemTime(&time_last); - uuids_this_tick = uuids_per_tick; - init = TRUE; - mMutex = new LLMutex(); - } - - uuid_time_t time_now = {0,0}; - - while (1) { - getSystemTime(&time_now); - - // if clock reading changed since last UUID generated - if (cmpTime(&time_last, &time_now)) { - // reset count of uuid's generated with this clock reading - uuids_this_tick = 0; - break; - } - if (uuids_this_tick < uuids_per_tick) { - uuids_this_tick++; - break; - } - // going too fast for our clock; spin - } - - time_last = time_now; - - if (uuids_this_tick != 0) { - if (time_now.low & 0x80000000) { - time_now.low += uuids_this_tick; - if (!(time_now.low & 0x80000000)) - time_now.high++; - } else - time_now.low += uuids_this_tick; - } - - timestamp->high = time_now.high; - timestamp->low = time_now.low; + // Get current time as 60 bit 100ns ticks since whenever. + // Compensate for the fact that real clock resolution is less + // than 100ns. + + const U32 uuids_per_tick = 1024; + + static uuid_time_t time_last; + static U32 uuids_this_tick; + static BOOL init = FALSE; + + if (!init) { + getSystemTime(&time_last); + uuids_this_tick = uuids_per_tick; + init = TRUE; + mMutex = new LLMutex(); + } + + uuid_time_t time_now = { 0,0 }; + + while (1) { + getSystemTime(&time_now); + + // if clock reading changed since last UUID generated + if (cmpTime(&time_last, &time_now)) { + // reset count of uuid's generated with this clock reading + uuids_this_tick = 0; + break; + } + if (uuids_this_tick < uuids_per_tick) { + uuids_this_tick++; + break; + } + // going too fast for our clock; spin + } + + time_last = time_now; + + if (uuids_this_tick != 0) { + if (time_now.low & 0x80000000) { + time_now.low += uuids_this_tick; + if (!(time_now.low & 0x80000000)) + time_now.high++; + } + else + time_now.low += uuids_this_tick; + } + + timestamp->high = time_now.high; + timestamp->low = time_now.low; } void LLUUID::generate() { - // Create a UUID. - uuid_time_t timestamp; - - static unsigned char node_id[6]; /* Flawfinder: ignore */ - static int has_init = 0; - - // Create a UUID. - static uuid_time_t time_last = {0,0}; - static U16 clock_seq = 0; + // Create a UUID. + uuid_time_t timestamp; + + static unsigned char node_id[6]; /* Flawfinder: ignore */ + static int has_init = 0; + + // Create a UUID. + static uuid_time_t time_last = { 0,0 }; + static U16 clock_seq = 0; #if LL_USE_JANKY_RANDOM_NUMBER_GENERATOR - static U32 seed = 0L; // dummy seed. reset it below + static U32 seed = 0L; // dummy seed. reset it below #endif - if (!has_init) - { - has_init = 1; - if (getNodeID(node_id) <= 0) - { - get_random_bytes(node_id, 6); - /* - * Set multicast bit, to prevent conflicts - * with IEEE 802 addresses obtained from - * network cards - */ - node_id[0] |= 0x80; - } - - getCurrentTime(&time_last); + if (!has_init) + { + has_init = 1; + if (getNodeID(node_id) <= 0) + { + get_random_bytes(node_id, 6); + /* + * Set multicast bit, to prevent conflicts + * with IEEE 802 addresses obtained from + * network cards + */ + node_id[0] |= 0x80; + } + + getCurrentTime(&time_last); #if LL_USE_JANKY_RANDOM_NUMBER_GENERATOR - seed = time_last.low; + seed = time_last.low; #endif #if LL_USE_JANKY_RANDOM_NUMBER_GENERATOR - clock_seq = (U16)janky_fast_random_seeded_bytes(seed, 65536); + clock_seq = (U16)janky_fast_random_seeded_bytes(seed, 65536); #else - clock_seq = (U16)ll_rand(65536); + clock_seq = (U16)ll_rand(65536); #endif - } - - // get current time - getCurrentTime(×tamp); - U16 our_clock_seq = clock_seq; - - // if clock hasn't changed or went backward, change clockseq - if (cmpTime(×tamp, &time_last) != 1) - { - LLMutexLock lock(mMutex); - clock_seq = (clock_seq + 1) & 0x3FFF; - if (clock_seq == 0) - clock_seq++; - our_clock_seq = clock_seq; // Ensure we're using a different clock_seq value from previous time - } + } + + // get current time + getCurrentTime(×tamp); + U16 our_clock_seq = clock_seq; + + // if clock hasn't changed or went backward, change clockseq + if (cmpTime(×tamp, &time_last) != 1) + { + LLMutexLock lock(mMutex); + clock_seq = (clock_seq + 1) & 0x3FFF; + if (clock_seq == 0) + clock_seq++; + our_clock_seq = clock_seq; // Ensure we're using a different clock_seq value from previous time + } time_last = timestamp; - memcpy(mData+10, node_id, 6); /* Flawfinder: ignore */ - U32 tmp; - tmp = timestamp.low; - mData[3] = (unsigned char) tmp; - tmp >>= 8; - mData[2] = (unsigned char) tmp; - tmp >>= 8; - mData[1] = (unsigned char) tmp; - tmp >>= 8; - mData[0] = (unsigned char) tmp; - - tmp = (U16) timestamp.high; - mData[5] = (unsigned char) tmp; - tmp >>= 8; - mData[4] = (unsigned char) tmp; - - tmp = (timestamp.high >> 16) | 0x1000; - mData[7] = (unsigned char) tmp; - tmp >>= 8; - mData[6] = (unsigned char) tmp; - - tmp = our_clock_seq; - - mData[9] = (unsigned char) tmp; - tmp >>= 8; - mData[8] = (unsigned char) tmp; - - HBXXH128::digest(*this, (const void*)mData, 16); + memcpy(mData + 10, node_id, 6); /* Flawfinder: ignore */ + U32 tmp; + tmp = timestamp.low; + mData[3] = (unsigned char)tmp; + tmp >>= 8; + mData[2] = (unsigned char)tmp; + tmp >>= 8; + mData[1] = (unsigned char)tmp; + tmp >>= 8; + mData[0] = (unsigned char)tmp; + + tmp = (U16)timestamp.high; + mData[5] = (unsigned char)tmp; + tmp >>= 8; + mData[4] = (unsigned char)tmp; + + tmp = (timestamp.high >> 16) | 0x1000; + mData[7] = (unsigned char)tmp; + tmp >>= 8; + mData[6] = (unsigned char)tmp; + + tmp = our_clock_seq; + + mData[9] = (unsigned char)tmp; + tmp >>= 8; + mData[8] = (unsigned char)tmp; + + LLMD5 md5_uuid; + + md5_uuid.update(mData, 16); + md5_uuid.finalize(); + md5_uuid.raw_digest(mData); } void LLUUID::generate(const std::string& hash_string) { - HBXXH128::digest(*this, hash_string); + LLMD5 md5_uuid((U8*)hash_string.c_str()); + md5_uuid.raw_digest(mData); } U32 LLUUID::getRandomSeed() { - static unsigned char seed[16]; /* Flawfinder: ignore */ - - getNodeID(&seed[0]); + static unsigned char seed[16]; /* Flawfinder: ignore */ - // Incorporate the pid into the seed to prevent - // processes that start on the same host at the same - // time from generating the same seed. - pid_t pid = LLApp::getPid(); + getNodeID(&seed[0]); - seed[6]=(unsigned char)(pid >> 8); - seed[7]=(unsigned char)(pid); - getSystemTime((uuid_time_t *)(&seed[8])); + // Incorporate the pid into the seed to prevent + // processes that start on the same host at the same + // time from generating the same seed. + pid_t pid = LLApp::getPid(); - U64 seed64 = HBXXH64((const void*)seed, 16).digest(); - return U32(seed64) ^ U32(seed64 >> 32); + seed[6] = (unsigned char)(pid >> 8); + seed[7] = (unsigned char)(pid); + getSystemTime((uuid_time_t*)(&seed[8])); + + LLMD5 md5_seed; + + md5_seed.update(seed, 16); + md5_seed.finalize(); + md5_seed.raw_digest(seed); + + return(*(U32*)seed); } BOOL LLUUID::parseUUID(const std::string& buf, LLUUID* value) { - if( buf.empty() || value == NULL) - { - return FALSE; - } - - std::string temp( buf ); - LLStringUtil::trim(temp); - if( LLUUID::validate( temp ) ) - { - value->set( temp ); - return TRUE; - } - return FALSE; + if (buf.empty() || value == NULL) + { + return FALSE; + } + + std::string temp(buf); + LLStringUtil::trim(temp); + if (LLUUID::validate(temp)) + { + value->set(temp); + return TRUE; + } + return FALSE; } //static LLUUID LLUUID::generateNewID(std::string hash_string) { - LLUUID new_id; - if (hash_string.empty()) - { - new_id.generate(); - } - else - { - new_id.generate(hash_string); - } - return new_id; + LLUUID new_id; + if (hash_string.empty()) + { + new_id.generate(); + } + else + { + new_id.generate(hash_string); + } + return new_id; } LLAssetID LLTransactionID::makeAssetID(const LLUUID& session) const { - LLAssetID result; - if (isNull()) - { - result.setNull(); - } - else - { - combine(session, result); - } - return result; + LLAssetID result; + if (isNull()) + { + result.setNull(); + } + else + { + combine(session, result); + } + return result; } // Construct LLUUID::LLUUID() { - setNull(); + setNull(); } // Faster than copying from memory - void LLUUID::setNull() +void LLUUID::setNull() { - U32 *word = (U32 *)mData; - word[0] = 0; - word[1] = 0; - word[2] = 0; - word[3] = 0; + U32* word = (U32*)mData; + word[0] = 0; + word[1] = 0; + word[2] = 0; + word[3] = 0; } // Compare - bool LLUUID::operator==(const LLUUID& rhs) const +bool LLUUID::operator==(const LLUUID& rhs) const { - U32 *tmp = (U32 *)mData; - U32 *rhstmp = (U32 *)rhs.mData; - // Note: binary & to avoid branching - return - (tmp[0] == rhstmp[0]) & - (tmp[1] == rhstmp[1]) & - (tmp[2] == rhstmp[2]) & - (tmp[3] == rhstmp[3]); + U32* tmp = (U32*)mData; + U32* rhstmp = (U32*)rhs.mData; + // Note: binary & to avoid branching + return + (tmp[0] == rhstmp[0]) & + (tmp[1] == rhstmp[1]) & + (tmp[2] == rhstmp[2]) & + (tmp[3] == rhstmp[3]); } - bool LLUUID::operator!=(const LLUUID& rhs) const +bool LLUUID::operator!=(const LLUUID& rhs) const { - U32 *tmp = (U32 *)mData; - U32 *rhstmp = (U32 *)rhs.mData; - // Note: binary | to avoid branching - return - (tmp[0] != rhstmp[0]) | - (tmp[1] != rhstmp[1]) | - (tmp[2] != rhstmp[2]) | - (tmp[3] != rhstmp[3]); + U32* tmp = (U32*)mData; + U32* rhstmp = (U32*)rhs.mData; + // Note: binary | to avoid branching + return + (tmp[0] != rhstmp[0]) | + (tmp[1] != rhstmp[1]) | + (tmp[2] != rhstmp[2]) | + (tmp[3] != rhstmp[3]); } /* @@ -983,94 +990,94 @@ LLUUID::LLUUID() // to integers, among other things. Use isNull() or notNull(). LLUUID::operator bool() const { - U32 *word = (U32 *)mData; - return (word[0] | word[1] | word[2] | word[3]) > 0; + U32 *word = (U32 *)mData; + return (word[0] | word[1] | word[2] | word[3]) > 0; } */ - BOOL LLUUID::notNull() const +BOOL LLUUID::notNull() const { - U32 *word = (U32 *)mData; - return (word[0] | word[1] | word[2] | word[3]) > 0; + U32* word = (U32*)mData; + return (word[0] | word[1] | word[2] | word[3]) > 0; } // Faster than == LLUUID::null because doesn't require // as much memory access. - BOOL LLUUID::isNull() const +BOOL LLUUID::isNull() const { - U32 *word = (U32 *)mData; - // If all bits are zero, return !0 == TRUE - return !(word[0] | word[1] | word[2] | word[3]); + U32* word = (U32*)mData; + // If all bits are zero, return !0 == TRUE + return !(word[0] | word[1] | word[2] | word[3]); } - LLUUID::LLUUID(const char *in_string) +LLUUID::LLUUID(const char* in_string) { - if (!in_string || in_string[0] == 0) - { - setNull(); - return; - } - - set(in_string); + if (!in_string || in_string[0] == 0) + { + setNull(); + return; + } + + set(in_string); } - LLUUID::LLUUID(const std::string& in_string) +LLUUID::LLUUID(const std::string& in_string) { - if (in_string.empty()) - { - setNull(); - return; - } + if (in_string.empty()) + { + setNull(); + return; + } - set(in_string); + set(in_string); } // IW: DON'T "optimize" these w/ U32s or you'll scoogie the sort order // IW: this will make me very sad - bool LLUUID::operator<(const LLUUID &rhs) const +bool LLUUID::operator<(const LLUUID& rhs) const { - U32 i; - for( i = 0; i < (UUID_BYTES - 1); i++ ) - { - if( mData[i] != rhs.mData[i] ) - { - return (mData[i] < rhs.mData[i]); - } - } - return (mData[UUID_BYTES - 1] < rhs.mData[UUID_BYTES - 1]); + U32 i; + for (i = 0; i < (UUID_BYTES - 1); i++) + { + if (mData[i] != rhs.mData[i]) + { + return (mData[i] < rhs.mData[i]); + } + } + return (mData[UUID_BYTES - 1] < rhs.mData[UUID_BYTES - 1]); } - bool LLUUID::operator>(const LLUUID &rhs) const +bool LLUUID::operator>(const LLUUID& rhs) const { - U32 i; - for( i = 0; i < (UUID_BYTES - 1); i++ ) - { - if( mData[i] != rhs.mData[i] ) - { - return (mData[i] > rhs.mData[i]); - } - } - return (mData[UUID_BYTES - 1] > rhs.mData[UUID_BYTES - 1]); + U32 i; + for (i = 0; i < (UUID_BYTES - 1); i++) + { + if (mData[i] != rhs.mData[i]) + { + return (mData[i] > rhs.mData[i]); + } + } + return (mData[UUID_BYTES - 1] > rhs.mData[UUID_BYTES - 1]); } - U16 LLUUID::getCRC16() const +U16 LLUUID::getCRC16() const { - // A UUID is 16 bytes, or 8 shorts. - U16 *short_data = (U16*)mData; - U16 out = 0; - out += short_data[0]; - out += short_data[1]; - out += short_data[2]; - out += short_data[3]; - out += short_data[4]; - out += short_data[5]; - out += short_data[6]; - out += short_data[7]; - return out; + // A UUID is 16 bytes, or 8 shorts. + U16* short_data = (U16*)mData; + U16 out = 0; + out += short_data[0]; + out += short_data[1]; + out += short_data[2]; + out += short_data[3]; + out += short_data[4]; + out += short_data[5]; + out += short_data[6]; + out += short_data[7]; + return out; } - U32 LLUUID::getCRC32() const +U32 LLUUID::getCRC32() const { - U32 *tmp = (U32*)mData; - return tmp[0] + tmp[1] + tmp[2] + tmp[3]; + U32* tmp = (U32*)mData; + return tmp[0] + tmp[1] + tmp[2] + tmp[3]; } -- cgit v1.2.3 From 8564da58315d015bc2cd60e5f15395e4173d9adc Mon Sep 17 00:00:00 2001 From: Brad Linden Date: Mon, 6 Mar 2023 11:31:29 -0800 Subject: Improved detail for llvertexbuffer attribute mask assertion failure in DRTVWR-559 --- indra/llcommon/llerror.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llerror.h b/indra/llcommon/llerror.h index d06c0e2132..b7dec3cb7f 100644 --- a/indra/llcommon/llerror.h +++ b/indra/llcommon/llerror.h @@ -82,9 +82,11 @@ const int LL_ERR_NOERR = 0; #ifdef SHOW_ASSERT #define llassert(func) llassert_always_msg(func, #func) +#define llassert_msg(func, msg) llassert_always_msg(func, msg) #define llverify(func) llassert_always_msg(func, #func) #else #define llassert(func) +#define llassert_msg(func, msg) #define llverify(func) do {if (func) {}} while(0) #endif -- cgit v1.2.3 From 25ede8638209fac8dde5b71bece4bc1dfa30ea16 Mon Sep 17 00:00:00 2001 From: Dave Parks Date: Fri, 10 Mar 2023 10:52:35 -0600 Subject: SL-19172 Texture streaming tune up. Incidental decruft. --- indra/llcommon/llcommon.cpp | 29 +++++++++++++++++++++++++---- indra/llcommon/llprofiler.h | 6 +----- indra/llcommon/llprofilercategories.h | 2 +- 3 files changed, 27 insertions(+), 10 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llcommon.cpp b/indra/llcommon/llcommon.cpp index d2c4e66160..6e988260a9 100644 --- a/indra/llcommon/llcommon.cpp +++ b/indra/llcommon/llcommon.cpp @@ -37,12 +37,13 @@ thread_local bool gProfilerEnabled = false; #if (TRACY_ENABLE) // Override new/delete for tracy memory profiling -void *operator new(size_t size) + +void* ll_tracy_new(size_t size) { void* ptr; if (gProfilerEnabled) { - LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; + //LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; ptr = (malloc)(size); } else @@ -57,12 +58,22 @@ void *operator new(size_t size) return ptr; } -void operator delete(void *ptr) noexcept +void* operator new(size_t size) +{ + return ll_tracy_new(size); +} + +void* operator new[](std::size_t count) +{ + return ll_tracy_new(count); +} + +void ll_tracy_delete(void* ptr) { TracyFree(ptr); if (gProfilerEnabled) { - LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; + //LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; (free)(ptr); } else @@ -71,6 +82,16 @@ void operator delete(void *ptr) noexcept } } +void operator delete(void *ptr) noexcept +{ + ll_tracy_delete(ptr); +} + +void operator delete[](void* ptr) noexcept +{ + ll_tracy_delete(ptr); +} + // C-style malloc/free can't be so easily overridden, so we define tracy versions and use // a pre-processor #define in linden_common.h to redirect to them. The parens around the native // functions below prevents recursive substitution by the preprocessor. diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index 394fbd05f6..736a069f49 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -138,10 +138,6 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILE_ZONE_ERR(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0XFF0000 ) // RGB yellow #define LL_PROFILE_ZONE_INFO(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0X00FFFF ) // RGB cyan #define LL_PROFILE_ZONE_WARN(name) LL_PROFILE_ZONE_NAMED_COLOR( name, 0x0FFFF00 ) // RGB red - //#define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) // memory allocation tracking currently not working - //#define LL_PROFILE_FREE(ptr) TracyFree(ptr) - #define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); - #define LL_PROFILE_FREE(ptr) (void)(ptr); #endif #else #define LL_PROFILER_FRAME_END @@ -165,7 +161,7 @@ extern thread_local bool gProfilerEnabled; #define LL_LABEL_OBJECT_GL(type, name, length, label) -#if LL_PROFILER_CONFIG > 1 +#if LL_PROFILER_CONFIGURATION > 1 #define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) #define LL_PROFILE_FREE(ptr) TracyFree(ptr) #else diff --git a/indra/llcommon/llprofilercategories.h b/indra/llcommon/llprofilercategories.h index 8db29468cc..617431f629 100644 --- a/indra/llcommon/llprofilercategories.h +++ b/indra/llcommon/llprofilercategories.h @@ -52,7 +52,7 @@ #define LL_PROFILER_CATEGORY_ENABLE_LOGGING 1 #define LL_PROFILER_CATEGORY_ENABLE_MATERIAL 1 #define LL_PROFILER_CATEGORY_ENABLE_MEDIA 1 -#define LL_PROFILER_CATEGORY_ENABLE_MEMORY 1 +#define LL_PROFILER_CATEGORY_ENABLE_MEMORY 0 #define LL_PROFILER_CATEGORY_ENABLE_NETWORK 1 #define LL_PROFILER_CATEGORY_ENABLE_OCTREE 1 #define LL_PROFILER_CATEGORY_ENABLE_PIPELINE 1 -- cgit v1.2.3 From e09475713b7abe6fcb916f4a770081a1696b57ab Mon Sep 17 00:00:00 2001 From: RunitaiLinden Date: Tue, 2 May 2023 18:47:21 -0500 Subject: DRTVWR-559 Optimization pass, make it so profileAvatar can read back GPU timer without a frame stall. --- indra/llcommon/llerror.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index 56fb7c21ca..a7794cc045 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -1611,6 +1611,7 @@ namespace LLError bool debugLoggingEnabled(const std::string& tag) { + LL_PROFILE_ZONE_SCOPED_CATEGORY_APP; LLMutexTrylock lock(getMutex(), 5); if (!lock.isLocked()) { -- cgit v1.2.3 From 2a10bd406c9155d30a82657ce2ee532b3a677d83 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 3 May 2023 09:55:31 -0400 Subject: DRTVWR-559: Replace debugLoggingEnabled() function with LL_DEBUGS(). The trouble with debugLoggingEnabled() is that it locked mutexes and searched maps every time that call was reached. LL_DEBUGS() has the same functionality (albeit with idiosyncratic syntax) but performs expensive lookups only once per session, caching the result in a local static variable. --- indra/llcommon/llcallstack.h | 12 ++++++------ indra/llcommon/llerror.cpp | 14 -------------- indra/llcommon/llerror.h | 24 +++++++++++++++++++++++- 3 files changed, 29 insertions(+), 21 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llcallstack.h b/indra/llcommon/llcallstack.h index 5acf04a49f..d5a2b7b157 100644 --- a/indra/llcommon/llcallstack.h +++ b/indra/llcommon/llcallstack.h @@ -79,9 +79,9 @@ struct LLContextStatus LL_COMMON_API std::ostream& operator<<(std::ostream& s, const LLContextStatus& context_status); -#define dumpStack(tag) \ - if (debugLoggingEnabled(tag)) \ - { \ - LLCallStack cs; \ - LL_DEBUGS(tag) << "STACK:\n" << "====================\n" << cs << "====================" << LL_ENDL; \ - } +#define dumpStack(tag) \ + LL_DEBUGS(tag) << "STACK:\n" \ + << "====================\n" \ + << LLCallStack() \ + << "====================" \ + << LL_ENDL; diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index 56fb7c21ca..5aa8558878 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -1609,19 +1609,5 @@ namespace LLError } } -bool debugLoggingEnabled(const std::string& tag) -{ - LLMutexTrylock lock(getMutex(), 5); - if (!lock.isLocked()) - { - return false; - } - - SettingsConfigPtr s = Globals::getInstance()->getSettingsConfig(); - LLError::ELevel level = LLError::LEVEL_DEBUG; - bool res = checkLevelMap(s->mTagLevelMap, tag, level); - return res; -} - diff --git a/indra/llcommon/llerror.h b/indra/llcommon/llerror.h index b7dec3cb7f..08eb323c4a 100644 --- a/indra/llcommon/llerror.h +++ b/indra/llcommon/llerror.h @@ -467,7 +467,29 @@ typedef LLError::NoClassInfo _LL_CLASS_TO_LOG; LLError::CallSite& _site(_sites[which]); \ lllog_test_() -// Check at run-time whether logging is enabled, without generating output +/* +// Check at run-time whether logging is enabled, without generating output. +Resist the temptation to add a function like this because it incurs the +expense of locking and map-searching every time control reaches it. bool debugLoggingEnabled(const std::string& tag); +Instead of: + +if debugLoggingEnabled("SomeTag") +{ + // ... presumably expensive operation ... + LL_DEBUGS("SomeTag") << ... << LL_ENDL; +} + +Use this: + +LL_DEBUGS("SomeTag"); +// ... presumably expensive operation ... +LL_CONT << ...; +LL_ENDL; + +LL_DEBUGS("SomeTag") performs the locking and map-searching ONCE, then caches +the result in a static variable. +*/ + #endif // LL_LLERROR_H -- cgit v1.2.3 From 1cadfd44fe5ddafaf3ba3560bd8ec8e1351b347d Mon Sep 17 00:00:00 2001 From: RunitaiLinden Date: Wed, 3 May 2023 17:30:02 -0500 Subject: DRTVWR-559 WIP - optimize ARC calculation et al. --- indra/llcommon/llmutex.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llmutex.h b/indra/llcommon/llmutex.h index 838d7d34c0..0d70da6178 100644 --- a/indra/llcommon/llmutex.h +++ b/indra/llcommon/llmutex.h @@ -36,7 +36,8 @@ //============================================================================ -#define MUTEX_DEBUG (LL_DEBUG || LL_RELEASE_WITH_DEBUG_INFO) +//#define MUTEX_DEBUG (LL_DEBUG || LL_RELEASE_WITH_DEBUG_INFO) +#define MUTEX_DEBUG 0 //disable mutex debugging as it's interfering with profiles #if MUTEX_DEBUG #include @@ -61,7 +62,7 @@ protected: mutable LLThread::id_t mLockingThread; #if MUTEX_DEBUG - std::map mIsLocked; + std::unordered_map mIsLocked; #endif }; -- cgit v1.2.3 From 855cae27ccde4896509e3e22c68c441d6404ccfb Mon Sep 17 00:00:00 2001 From: Rye Mutt Date: Fri, 5 May 2023 17:23:29 -0400 Subject: Fix LLThreadSafeQueueInterrupt in WorkQueueBase::postTo during shutdown by catching and returning false --- indra/llcommon/workqueue.h | 68 ++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 30 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 5461ce6c23..5b704e7198 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -516,37 +516,45 @@ namespace LL // Here we believe target WorkQueue still exists. Post to it a // lambda that packages our callable, our callback and a weak_ptr // to this originating WorkQueue. - tptr->post( - [reply = super::getWeak(), - callable = std::move(callable), - callback = std::move(callback)] - () mutable - { - // Use postMaybe() below in case this originating WorkQueue - // has been closed or destroyed. Remember, the outer lambda is - // now running on a thread servicing the target WorkQueue, and - // real time has elapsed since postTo()'s tptr->post() call. - try - { - // Make a reply lambda to repost to THIS WorkQueue. - // Delegate to makeReplyLambda() so we can partially - // specialize on void return. - postMaybe(reply, makeReplyLambda(std::move(callable), std::move(callback))); - } - catch (...) + try + { + tptr->post( + [reply = super::getWeak(), + callable = std::move(callable), + callback = std::move(callback)] + () mutable { - // Either variant of makeReplyLambda() is responsible for - // calling the caller's callable. If that throws, return - // the exception to the originating thread. - postMaybe( - reply, - // Bind the current exception to transport back to the - // originating WorkQueue. Once there, rethrow it. - [exc = std::current_exception()](){ std::rethrow_exception(exc); }); - } - }, - // if caller passed a TimePoint, pass it along to post() - std::forward(args)...); + // Use postMaybe() below in case this originating WorkQueue + // has been closed or destroyed. Remember, the outer lambda is + // now running on a thread servicing the target WorkQueue, and + // real time has elapsed since postTo()'s tptr->post() call. + try + { + // Make a reply lambda to repost to THIS WorkQueue. + // Delegate to makeReplyLambda() so we can partially + // specialize on void return. + postMaybe(reply, makeReplyLambda(std::move(callable), std::move(callback))); + } + catch (...) + { + // Either variant of makeReplyLambda() is responsible for + // calling the caller's callable. If that throws, return + // the exception to the originating thread. + postMaybe( + reply, + // Bind the current exception to transport back to the + // originating WorkQueue. Once there, rethrow it. + [exc = std::current_exception()](){ std::rethrow_exception(exc); }); + } + }, + // if caller passed a TimePoint, pass it along to post() + std::forward(args)...); + } + catch (const Closed&) + { + // target WorkQueue still exists, but is Closed + return false; + } // looks like we were able to post() return true; -- cgit v1.2.3 From 026ef1935dbdb21ab79159d38fb78e126dd6ac95 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 8 May 2023 12:07:31 -0400 Subject: SL-19690: Follow up on Rye Mutt's fix for shutdown crashes. Rather than continuing to propagate try/catch (Closed) (aka LLThreadSafeQueueInterrupt) constructs through the code base, make WorkQueueBase::post() return bool indicating success (i.e. ! isClosed()). This obviates postIfOpen(), which no one was using anyway. In effect, postIfOpen() is renamed post(), bypassing the exception when isClosed(). Review existing try/catch blocks of that sort, changing to test for post() returning false. --- indra/llcommon/llqueuedthread.cpp | 2 +- indra/llcommon/workqueue.cpp | 21 +----- indra/llcommon/workqueue.h | 135 +++++++++++++++----------------------- 3 files changed, 57 insertions(+), 101 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index 9b1de2e9a5..7da7c1e026 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -146,7 +146,7 @@ S32 LLQueuedThread::updateQueue(F32 max_time_ms) // schedule a call to threadedUpdate for every call to updateQueue if (!isQuitting()) { - mRequestQueue.postIfOpen([=]() + mRequestQueue.post([=]() { LL_PROFILE_ZONE_NAMED_CATEGORY_THREAD("qt - update"); mIdleThread = FALSE; diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index 83e0216ae7..5a77d517dd 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -161,12 +161,7 @@ bool LL::WorkQueue::done() return mQueue.done(); } -void LL::WorkQueue::post(const Work& callable) -{ - mQueue.push(callable); -} - -bool LL::WorkQueue::postIfOpen(const Work& callable) +bool LL::WorkQueue::post(const Work& callable) { return mQueue.pushIfOpen(callable); } @@ -215,26 +210,16 @@ bool LL::WorkSchedule::done() return mQueue.done(); } -void LL::WorkSchedule::post(const Work& callable) +bool LL::WorkSchedule::post(const Work& callable) { // Use TimePoint::clock::now() instead of TimePoint's representation of // the epoch because this WorkSchedule may contain a mix of past-due // TimedWork items and TimedWork items scheduled for the future. Sift this // new item into the correct place. - post(callable, TimePoint::clock::now()); -} - -void LL::WorkSchedule::post(const Work& callable, const TimePoint& time) -{ - mQueue.push(TimedWork(time, callable)); -} - -bool LL::WorkSchedule::postIfOpen(const Work& callable) -{ return postIfOpen(callable, TimePoint::clock::now()); } -bool LL::WorkSchedule::postIfOpen(const Work& callable, const TimePoint& time) +bool LL::WorkSchedule::post(const Work& callable, const TimePoint& time) { return mQueue.pushIfOpen(TimedWork(time, callable)); } diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 5b704e7198..87fdd1517f 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -83,13 +83,10 @@ namespace LL /*---------------------- fire and forget API -----------------------*/ - /// fire-and-forget - virtual void post(const Work&) = 0; - /** * post work, unless the queue is closed before we can post */ - virtual bool postIfOpen(const Work&) = 0; + virtual bool post(const Work&) = 0; /** * post work, unless the queue is full @@ -247,13 +244,10 @@ namespace LL /*---------------------- fire and forget API -----------------------*/ - /// fire-and-forget - void post(const Work&) override; - /** * post work, unless the queue is closed before we can post */ - bool postIfOpen(const Work&) override; + bool post(const Work&) override; /** * post work, unless the queue is full @@ -320,22 +314,16 @@ namespace LL /*---------------------- fire and forget API -----------------------*/ - /// fire-and-forget - void post(const Work& callable) override; - - /// fire-and-forget, but at a particular (future?) time - void post(const Work& callable, const TimePoint& time); - /** * post work, unless the queue is closed before we can post */ - bool postIfOpen(const Work& callable) override; + bool post(const Work& callable) override; /** * post work for a particular time, unless the queue is closed before * we can post */ - bool postIfOpen(const Work& callable, const TimePoint& time); + bool post(const Work& callable, const TimePoint& time); /** * post work, unless the queue is full @@ -356,7 +344,7 @@ namespace LL * an LLCond variant, e.g. LLOneShotCond or LLBoolCond. */ template - void postEvery(const std::chrono::duration& interval, + bool postEvery(const std::chrono::duration& interval, CALLABLE&& callable); private: @@ -417,15 +405,10 @@ namespace LL // move-only callable; but naturally this statement must be // the last time we reference this instance, which may become // moved-from. - try - { - auto target{ std::dynamic_pointer_cast(mTarget.lock()) }; - target->post(std::move(*this), mStart); - } - catch (const Closed&) - { - // Once this queue is closed, oh well, just stop - } + auto target{ std::dynamic_pointer_cast(mTarget.lock()) }; + // Discard bool return: once this queue is closed, oh well, + // just stop + target->post(std::move(*this), mStart); } } @@ -437,7 +420,7 @@ namespace LL }; template - void WorkSchedule::postEvery(const std::chrono::duration& interval, + bool WorkSchedule::postEvery(const std::chrono::duration& interval, CALLABLE&& callable) { if (interval.count() <= 0) @@ -454,7 +437,7 @@ namespace LL // Instantiate and post a suitable BackJack, binding a weak_ptr to // self, the current time, the desired interval and the desired // callable. - post( + return post( BackJack( getWeak(), TimePoint::clock::now(), interval, std::move(callable))); } @@ -516,48 +499,37 @@ namespace LL // Here we believe target WorkQueue still exists. Post to it a // lambda that packages our callable, our callback and a weak_ptr // to this originating WorkQueue. - try - { - tptr->post( - [reply = super::getWeak(), - callable = std::move(callable), - callback = std::move(callback)] - () mutable + return tptr->post( + [reply = super::getWeak(), + callable = std::move(callable), + callback = std::move(callback)] + () mutable + { + // Use postMaybe() below in case this originating WorkQueue + // has been closed or destroyed. Remember, the outer lambda is + // now running on a thread servicing the target WorkQueue, and + // real time has elapsed since postTo()'s tptr->post() call. + try { - // Use postMaybe() below in case this originating WorkQueue - // has been closed or destroyed. Remember, the outer lambda is - // now running on a thread servicing the target WorkQueue, and - // real time has elapsed since postTo()'s tptr->post() call. - try - { - // Make a reply lambda to repost to THIS WorkQueue. - // Delegate to makeReplyLambda() so we can partially - // specialize on void return. - postMaybe(reply, makeReplyLambda(std::move(callable), std::move(callback))); - } - catch (...) - { - // Either variant of makeReplyLambda() is responsible for - // calling the caller's callable. If that throws, return - // the exception to the originating thread. - postMaybe( - reply, - // Bind the current exception to transport back to the - // originating WorkQueue. Once there, rethrow it. - [exc = std::current_exception()](){ std::rethrow_exception(exc); }); - } - }, - // if caller passed a TimePoint, pass it along to post() - std::forward(args)...); - } - catch (const Closed&) - { - // target WorkQueue still exists, but is Closed - return false; - } - - // looks like we were able to post() - return true; + // Make a reply lambda to repost to THIS WorkQueue. + // Delegate to makeReplyLambda() so we can partially + // specialize on void return. + postMaybe(reply, makeReplyLambda(std::move(callable), std::move(callback))); + } + catch (...) + { + // Either variant of makeReplyLambda() is responsible for + // calling the caller's callable. If that throws, return + // the exception to the originating thread. + postMaybe( + reply, + // Bind the current exception to transport back to the + // originating WorkQueue. Once there, rethrow it. + [exc = std::current_exception()](){ std::rethrow_exception(exc); }); + } + }, + // if caller passed a TimePoint, pass it along to post() + std::forward(args)...); } template @@ -568,18 +540,9 @@ namespace LL auto tptr = target.lock(); if (tptr) { - try - { - tptr->post(std::forward(args)...); - // we were able to post() - return true; - } - catch (const Closed&) - { - // target WorkQueue still exists, but is Closed - } + return tptr->post(std::forward(args)...); } - // either target no longer exists, or its WorkQueue is Closed + // target no longer exists return false; } @@ -591,7 +554,7 @@ namespace LL auto operator()(WorkQueueBase* self, CALLABLE&& callable, ARGS&&... args) { LLCoros::Promise promise; - self->post( + bool posted = self->post( // We dare to bind a reference to Promise because it's // specifically designed for cross-thread communication. [&promise, callable = std::move(callable)]() @@ -608,6 +571,10 @@ namespace LL }, // if caller passed a TimePoint, pass it to post() std::forward(args)...); + if (! posted) + { + LLTHROW(Closed); + } auto future{ LLCoros::getFuture(promise) }; // now, on the calling thread, wait for that result LLCoros::TempStatus st("waiting for WorkQueue::waitForResult()"); @@ -623,7 +590,7 @@ namespace LL void operator()(WorkQueueBase* self, CALLABLE&& callable, ARGS&&... args) { LLCoros::Promise promise; - self->post( + bool posted = self->post( // &promise is designed for cross-thread access [&promise, callable = std::move(callable)]() mutable { @@ -639,6 +606,10 @@ namespace LL }, // if caller passed a TimePoint, pass it to post() std::forward(args)...); + if (! posted) + { + LLTHROW(Closed); + } auto future{ LLCoros::getFuture(promise) }; // block until set_value() LLCoros::TempStatus st("waiting for void WorkQueue::waitForResult()"); -- cgit v1.2.3 From d75ecde69c1289f4e3df8b75e9a74c5b05db318c Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 8 May 2023 12:31:57 -0400 Subject: SL-19690: Properly qualify exception type. --- indra/llcommon/workqueue.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 87fdd1517f..ec0700a718 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -573,7 +573,7 @@ namespace LL std::forward(args)...); if (! posted) { - LLTHROW(Closed); + LLTHROW(WorkQueueBase::Closed()); } auto future{ LLCoros::getFuture(promise) }; // now, on the calling thread, wait for that result @@ -608,7 +608,7 @@ namespace LL std::forward(args)...); if (! posted) { - LLTHROW(Closed); + LLTHROW(WorkQueueBase::Closed()); } auto future{ LLCoros::getFuture(promise) }; // block until set_value() -- cgit v1.2.3 From f728808d938666a01f73a039c861358fc4b02a9e Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 8 May 2023 12:45:27 -0400 Subject: SL-19690: Fix a lingering reference to WorkSchedule::postIfOpen() --- indra/llcommon/workqueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index 5a77d517dd..cf80ce0656 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -216,7 +216,7 @@ bool LL::WorkSchedule::post(const Work& callable) // the epoch because this WorkSchedule may contain a mix of past-due // TimedWork items and TimedWork items scheduled for the future. Sift this // new item into the correct place. - return postIfOpen(callable, TimePoint::clock::now()); + return post(callable, TimePoint::clock::now()); } bool LL::WorkSchedule::post(const Work& callable, const TimePoint& time) -- cgit v1.2.3 From 045342ba29aae186e13c711bd4dd84377d4a7e43 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 5 Jun 2023 11:28:41 -0400 Subject: SL-18837: Bump the granularity of WorkQueue timing tests. On a low-powered GitHub Mac runner, the system doesn't wake up as soon as it should, and we get spurious "too late" errors. Try a bigger time increment. --- indra/llcommon/tests/workqueue_test.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/workqueue_test.cpp b/indra/llcommon/tests/workqueue_test.cpp index 1d73f7aa0d..7655a7aa1f 100644 --- a/indra/llcommon/tests/workqueue_test.cpp +++ b/indra/llcommon/tests/workqueue_test.cpp @@ -83,7 +83,11 @@ namespace tut // signal the work item that it can quit; consider LLOneShotCond. LLCond data; auto start = WorkQueue::TimePoint::clock::now(); - auto interval = 100ms; + // 2s seems like a long time to wait, since it directly impacts the + // duration of this test program. Unfortunately GitHub's Mac runners + // are pretty wimpy, and we're getting spurious "too late" errors just + // because the thread doesn't wake up as soon as we want. + auto interval = 2s; queue.postEvery( interval, [&data, count = 0] -- cgit v1.2.3 From ca510f6c299335c8db27c65c10a8553801c06023 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 5 Jun 2023 22:08:26 -0400 Subject: SL-18837: Try giving temp Python scripts a .py extension. On GitHub Windows Actions runners, we're getting permissions errors trying to tell the Python interpreter to run a NamedTempFile script. Try using NamedExtTempFile to give each such script a .py extension. --- indra/llcommon/tests/llleap_test.cpp | 222 ++++++++++++++++---------------- indra/llcommon/tests/llprocess_test.cpp | 4 +- 2 files changed, 113 insertions(+), 113 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 60005fc6a9..99fd073dd2 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -213,9 +213,9 @@ namespace tut void object::test<1>() { set_test_name("multiple LLLeap instances"); - NamedTempFile script("py", - "import time\n" - "time.sleep(1)\n"); + NamedExtTempFile script("py", + "import time\n" + "time.sleep(1)\n"); LLLeapVector instances; instances.push_back(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))->getWeak()); @@ -231,10 +231,10 @@ namespace tut void object::test<2>() { set_test_name("stderr to log"); - NamedTempFile script("py", - "import sys\n" - "sys.stderr.write('''Hello from Python!\n" - "note partial line''')\n"); + NamedExtTempFile script("py", + "import sys\n" + "sys.stderr.write('''Hello from Python!\n" + "note partial line''')\n"); StringVec vcommand{ PYTHON, script.getName() }; CaptureLog log(LLError::LEVEL_INFO); waitfor(LLLeap::create(get_test_name(), vcommand)); @@ -246,8 +246,8 @@ namespace tut void object::test<3>() { set_test_name("bad stdout protocol"); - NamedTempFile script("py", - "print('Hello from Python!')\n"); + NamedExtTempFile script("py", + "print('Hello from Python!')\n"); CaptureLog log(LLError::LEVEL_WARN); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))); @@ -259,10 +259,10 @@ namespace tut void object::test<4>() { set_test_name("leftover stdout"); - NamedTempFile script("py", - "import sys\n" - // note lack of newline - "sys.stdout.write('Hello from Python!')\n"); + NamedExtTempFile script("py", + "import sys\n" + // note lack of newline + "sys.stdout.write('Hello from Python!')\n"); CaptureLog log(LLError::LEVEL_WARN); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))); @@ -274,9 +274,9 @@ namespace tut void object::test<5>() { set_test_name("bad stdout len prefix"); - NamedTempFile script("py", - "import sys\n" - "sys.stdout.write('5a2:something')\n"); + NamedExtTempFile script("py", + "import sys\n" + "sys.stdout.write('5a2:something')\n"); CaptureLog log(LLError::LEVEL_WARN); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))); @@ -381,16 +381,16 @@ namespace tut set_test_name("round trip"); AckAPI api; Result result; - NamedTempFile script("py", - boost::phoenix::placeholders::arg1 << - "from " << reader_module << " import *\n" - // make a request on our little API - "request(pump='" << api.getName() << "', data={})\n" - // wait for its response - "resp = get()\n" - "result = '' if resp == dict(pump=replypump(), data='ack')\\\n" - " else 'bad: ' + str(resp)\n" - "send(pump='" << result.getName() << "', data=result)\n"); + NamedExtTempFile script("py", + boost::phoenix::placeholders::arg1 << + "from " << reader_module << " import *\n" + // make a request on our little API + "request(pump='" << api.getName() << "', data={})\n" + // wait for its response + "resp = get()\n" + "result = '' if resp == dict(pump=replypump(), data='ack')\\\n" + " else 'bad: ' + str(resp)\n" + "send(pump='" << result.getName() << "', data=result)\n"); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))); result.ensure(); } @@ -419,37 +419,37 @@ namespace tut // iterations etc. in OS pipes and the LLLeap/LLProcess implementation. ReqIDAPI api; Result result; - NamedTempFile script("py", - boost::phoenix::placeholders::arg1 << - "import sys\n" - "from " << reader_module << " import *\n" - // Note that since reader imports llsd, this - // 'import *' gets us llsd too. - "sample = llsd.format_notation(dict(pump='" << - api.getName() << "', data=dict(reqid=999999, reply=replypump())))\n" - // The whole packet has length prefix too: "len:data" - "samplen = len(str(len(sample))) + 1 + len(sample)\n" - // guess how many messages it will take to - // accumulate BUFFERED_LENGTH - "count = int(" << BUFFERED_LENGTH << "/samplen)\n" - "print('Sending %s requests' % count, file=sys.stderr)\n" - "for i in range(count):\n" - " request('" << api.getName() << "', dict(reqid=i))\n" - // The assumption in this specific test that - // replies will arrive in the same order as - // requests is ONLY valid because the API we're - // invoking sends replies instantly. If the API - // had to wait for some external event before - // sending its reply, replies could arrive in - // arbitrary order, and we'd have to tick them - // off from a set. - "result = ''\n" - "for i in range(count):\n" - " resp = get()\n" - " if resp['data']['reqid'] != i:\n" - " result = 'expected reqid=%s in %s' % (i, resp)\n" - " break\n" - "send(pump='" << result.getName() << "', data=result)\n"); + NamedExtTempFile script("py", + boost::phoenix::placeholders::arg1 << + "import sys\n" + "from " << reader_module << " import *\n" + // Note that since reader imports llsd, this + // 'import *' gets us llsd too. + "sample = llsd.format_notation(dict(pump='" << + api.getName() << "', data=dict(reqid=999999, reply=replypump())))\n" + // The whole packet has length prefix too: "len:data" + "samplen = len(str(len(sample))) + 1 + len(sample)\n" + // guess how many messages it will take to + // accumulate BUFFERED_LENGTH + "count = int(" << BUFFERED_LENGTH << "/samplen)\n" + "print('Sending %s requests' % count, file=sys.stderr)\n" + "for i in range(count):\n" + " request('" << api.getName() << "', dict(reqid=i))\n" + // The assumption in this specific test that + // replies will arrive in the same order as + // requests is ONLY valid because the API we're + // invoking sends replies instantly. If the API + // had to wait for some external event before + // sending its reply, replies could arrive in + // arbitrary order, and we'd have to tick them + // off from a set. + "result = ''\n" + "for i in range(count):\n" + " resp = get()\n" + " if resp['data']['reqid'] != i:\n" + " result = 'expected reqid=%s in %s' % (i, resp)\n" + " break\n" + "send(pump='" << result.getName() << "', data=result)\n"); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName()))), 300); // needs more realtime than most tests result.ensure(); @@ -462,60 +462,60 @@ namespace tut { ReqIDAPI api; Result result; - NamedTempFile script("py", - boost::phoenix::placeholders::arg1 << - "import sys\n" - "from " << reader_module << " import *\n" - // Generate a very large string value. - "desired = int(sys.argv[1])\n" - // 7 chars per item: 6 digits, 1 comma - "count = int((desired - 50)/7)\n" - "large = ''.join('%06d,' % i for i in range(count))\n" - // Pass 'large' as reqid because we know the API - // will echo reqid, and we want to receive it back. - "request('" << api.getName() << "', dict(reqid=large))\n" - "try:\n" - " resp = get()\n" - "except ParseError as e:\n" - " # try to find where e.data diverges from expectation\n" - // Normally we'd expect a 'pump' key in there, - // too, with value replypump(). But Python - // serializes keys in a different order than C++, - // so incoming data start with 'data'. - // Truthfully, though, if we get as far as 'pump' - // before we find a difference, something's very - // strange. - " expect = llsd.format_notation(dict(data=dict(reqid=large)))\n" - " chunk = 40\n" - " for offset in range(0, max(len(e.data), len(expect)), chunk):\n" - " if e.data[offset:offset+chunk] != \\\n" - " expect[offset:offset+chunk]:\n" - " print('Offset %06d: expect %r,\\n'\\\n" - " ' get %r' %\\\n" - " (offset,\n" - " expect[offset:offset+chunk],\n" - " e.data[offset:offset+chunk]),\n" - " file=sys.stderr)\n" - " break\n" - " else:\n" - " print('incoming data matches expect?!', file=sys.stderr)\n" - " send('" << result.getName() << "', '%s: %s' % (e.__class__.__name__, e))\n" - " sys.exit(1)\n" - "\n" - "echoed = resp['data']['reqid']\n" - "if echoed == large:\n" - " send('" << result.getName() << "', '')\n" - " sys.exit(0)\n" - // Here we know echoed did NOT match; try to find where - "for i in range(count):\n" - " start = 7*i\n" - " end = 7*(i+1)\n" - " if end > len(echoed)\\\n" - " or echoed[start:end] != large[start:end]:\n" - " send('" << result.getName() << "',\n" - " 'at offset %s, expected %r but got %r' %\n" - " (start, large[start:end], echoed[start:end]))\n" - "sys.exit(1)\n"); + NamedExtTempFile script("py", + boost::phoenix::placeholders::arg1 << + "import sys\n" + "from " << reader_module << " import *\n" + // Generate a very large string value. + "desired = int(sys.argv[1])\n" + // 7 chars per item: 6 digits, 1 comma + "count = int((desired - 50)/7)\n" + "large = ''.join('%06d,' % i for i in range(count))\n" + // Pass 'large' as reqid because we know the API + // will echo reqid, and we want to receive it back. + "request('" << api.getName() << "', dict(reqid=large))\n" + "try:\n" + " resp = get()\n" + "except ParseError as e:\n" + " # try to find where e.data diverges from expectation\n" + // Normally we'd expect a 'pump' key in there, + // too, with value replypump(). But Python + // serializes keys in a different order than C++, + // so incoming data start with 'data'. + // Truthfully, though, if we get as far as 'pump' + // before we find a difference, something's very + // strange. + " expect = llsd.format_notation(dict(data=dict(reqid=large)))\n" + " chunk = 40\n" + " for offset in range(0, max(len(e.data), len(expect)), chunk):\n" + " if e.data[offset:offset+chunk] != \\\n" + " expect[offset:offset+chunk]:\n" + " print('Offset %06d: expect %r,\\n'\\\n" + " ' get %r' %\\\n" + " (offset,\n" + " expect[offset:offset+chunk],\n" + " e.data[offset:offset+chunk]),\n" + " file=sys.stderr)\n" + " break\n" + " else:\n" + " print('incoming data matches expect?!', file=sys.stderr)\n" + " send('" << result.getName() << "', '%s: %s' % (e.__class__.__name__, e))\n" + " sys.exit(1)\n" + "\n" + "echoed = resp['data']['reqid']\n" + "if echoed == large:\n" + " send('" << result.getName() << "', '')\n" + " sys.exit(0)\n" + // Here we know echoed did NOT match; try to find where + "for i in range(count):\n" + " start = 7*i\n" + " end = 7*(i+1)\n" + " if end > len(echoed)\\\n" + " or echoed[start:end] != large[start:end]:\n" + " send('" << result.getName() << "',\n" + " 'at offset %s, expected %r but got %r' %\n" + " (start, large[start:end], echoed[start:end]))\n" + "sys.exit(1)\n"); waitfor(LLLeap::create(test_name, sv(list_of (PYTHON) diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 81449b4a42..4adb8d872a 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -191,7 +191,7 @@ struct PythonProcessLauncher LLProcess::Params mParams; LLProcessPtr mPy; std::string mDesc; - NamedTempFile mScript; + NamedExtTempFile mScript; }; /// convenience function for PythonProcessLauncher::run() @@ -355,7 +355,7 @@ namespace tut set_test_name("raw APR nonblocking I/O"); // Create a script file in a temporary place. - NamedTempFile script("py", + NamedExtTempFile script("py", "from __future__ import print_function" EOL "import sys" EOL "import time" EOL -- cgit v1.2.3 From 6516c9d07db42beba5ba9c0c41a33925794a249c Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Jun 2023 07:44:42 -0400 Subject: SL-18837: NamedTempFile back to std::function, use boost::phoenix << It seems the problem addressed by aab769e wasn't some synergy between Boost.Phoenix and Boost.Function, but rather the lack of a Phoenix header file introducing operator<<(). --- indra/llcommon/tests/llleap_test.cpp | 1 + indra/llcommon/tests/llsdserialize_test.cpp | 1 + 2 files changed, 2 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 99fd073dd2..0c91db3e24 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -19,6 +19,7 @@ // external library headers #include #include +#include // operator<<() // other Linden headers #include "../test/lltut.h" #include "../test/namedtempfile.h" diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index d7c11c5021..a0b8519508 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -50,6 +50,7 @@ typedef U32 uint32_t; #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" +#include using namespace boost::phoenix; #include "../llsd.h" -- cgit v1.2.3 From c4366378b61cacb9d87eb2917302fa17e9207491 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 6 Jun 2023 10:04:26 -0400 Subject: SL-18837: Ditch Boost.Phoenix implicit lambda syntax. It's cool to be able to write 'arg1 << "stuff" << var ...;' for a lambda accepting a std::ostream reference, but cascading compile errors mean it's no longer worth trying to make that work -- given actual C++ lambdas. Also clean up a lingering BOOST_FOREACH() and a boost::bind() while at it. --- indra/llcommon/tests/llleap_test.cpp | 18 ++++++++---------- indra/llcommon/tests/llsdserialize_test.cpp | 21 +++++++-------------- 2 files changed, 15 insertions(+), 24 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 0c91db3e24..0fc741d9e1 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -18,8 +18,6 @@ #include // external library headers #include -#include -#include // operator<<() // other Linden headers #include "../test/lltut.h" #include "../test/namedtempfile.h" @@ -105,7 +103,7 @@ namespace tut llleap_data(): reader(".py", // This logic is adapted from vita.viewerclient.receiveEvent() - boost::phoenix::placeholders::arg1 << + [](std::ostream& out){ out << "import re\n" "import os\n" "import sys\n" @@ -189,7 +187,7 @@ namespace tut "def request(pump, data):\n" " # we expect 'data' is a dict\n" " data['reply'] = _reply\n" - " send(pump, data)\n"), + " send(pump, data)\n";}), // Get the actual pathname of the NamedExtTempFile and trim off // the ".py" extension. (We could cache reader.getName() in a // separate member variable, but I happen to know getName() just @@ -383,7 +381,7 @@ namespace tut AckAPI api; Result result; NamedExtTempFile script("py", - boost::phoenix::placeholders::arg1 << + [&](std::ostream& out){ out << "from " << reader_module << " import *\n" // make a request on our little API "request(pump='" << api.getName() << "', data={})\n" @@ -391,7 +389,7 @@ namespace tut "resp = get()\n" "result = '' if resp == dict(pump=replypump(), data='ack')\\\n" " else 'bad: ' + str(resp)\n" - "send(pump='" << result.getName() << "', data=result)\n"); + "send(pump='" << result.getName() << "', data=result)\n";}); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))); result.ensure(); } @@ -421,7 +419,7 @@ namespace tut ReqIDAPI api; Result result; NamedExtTempFile script("py", - boost::phoenix::placeholders::arg1 << + [&](std::ostream& out){ out << "import sys\n" "from " << reader_module << " import *\n" // Note that since reader imports llsd, this @@ -450,7 +448,7 @@ namespace tut " if resp['data']['reqid'] != i:\n" " result = 'expected reqid=%s in %s' % (i, resp)\n" " break\n" - "send(pump='" << result.getName() << "', data=result)\n"); + "send(pump='" << result.getName() << "', data=result)\n";}); waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName()))), 300); // needs more realtime than most tests result.ensure(); @@ -464,7 +462,7 @@ namespace tut ReqIDAPI api; Result result; NamedExtTempFile script("py", - boost::phoenix::placeholders::arg1 << + [&](std::ostream& out){ out << "import sys\n" "from " << reader_module << " import *\n" // Generate a very large string value. @@ -516,7 +514,7 @@ namespace tut " send('" << result.getName() << "',\n" " 'at offset %s, expected %r but got %r' %\n" " (start, large[start:end], echoed[start:end]))\n" - "sys.exit(1)\n"); + "sys.exit(1)\n";}); waitfor(LLLeap::create(test_name, sv(list_of (PYTHON) diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index a0b8519508..08bf7fbc51 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -45,13 +45,6 @@ typedef U32 uint32_t; #endif #include "boost/range.hpp" -#include "boost/foreach.hpp" -#include "boost/function.hpp" -#include "boost/bind.hpp" -#include "boost/phoenix/bind/bind_function.hpp" -#include "boost/phoenix/core/argument.hpp" -#include -using namespace boost::phoenix; #include "../llsd.h" #include "../llsdserialize.h" @@ -1802,7 +1795,7 @@ namespace tut // helper for test<3> static void writeLLSDArray(std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item: llsd::inArray(array)) { LLSDSerialize::toNotation(item, out); // It's important to separate with newlines because Python's llsd @@ -1842,21 +1835,21 @@ namespace tut // Create an llsdXXXXXX file containing 'data' serialized to // notation. NamedTempFile file("llsd", - // NamedTempFile's boost::function constructor + // NamedTempFile's function constructor // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [&](std::ostream& out){ writeLLSDArray(out, cdata); }); python("read C++ notation", - placeholders::arg1 << + [&](std::ostream& out){ out << import_llsd << "def parse_each(iterable):\n" " for item in iterable:\n" " yield llsd.parse(item)\n" << pydata << // Don't forget raw-string syntax for Windows pathnames. - "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); + "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n";}); } template<> template<> @@ -1870,7 +1863,7 @@ namespace tut NamedTempFile file("llsd", ""); python("write Python notation", - placeholders::arg1 << + [&](std::ostream& out){ out << import_llsd << "DATA = [\n" " 17,\n" @@ -1884,7 +1877,7 @@ namespace tut // N.B. Using 'print' implicitly adds newlines. "with open(r'" << file.getName() << "', 'w') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " print(llsd.format_notation(item).decode(), file=f)\n";}); std::ifstream inf(file.getName().c_str()); LLSD item; -- cgit v1.2.3 From e933ace53b24b732d4111169e3c5964a8591a29e Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 14:07:12 -0400 Subject: SL-18837: Try to bypass Windows perm problem with Python indirection. --- indra/llcommon/tests/llprocess_test.cpp | 41 ++++++++++++++++++------------ indra/llcommon/tests/test_python_script.py | 20 +++++++++++++++ 2 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 indra/llcommon/tests/test_python_script.py (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 4adb8d872a..af99e97d66 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -124,6 +124,17 @@ void waitfor(LLProcess::handle h, const std::string& desc, int timeout=60) i < timeout); } +namespace { + +// find test helper, a sibling of this file +// nat 2023-07-07: we're currently using Boost 1.81, but +// path::replace_filename() (which is exactly what we need here) doesn't +// arrive until Boost 1.82. +auto test_python_script{ + (boost::filesystem::path(__FILE__).remove_filename() / "test_python_script.py").string() }; + +} + /** * Construct an LLProcess to run a Python script. */ @@ -145,6 +156,7 @@ struct PythonProcessLauncher mParams.desc = desc + " script"; mParams.executable = PYTHON; + mParams.args.add(test_python_script); mParams.args.add(mScript.getName()); } @@ -214,30 +226,26 @@ static std::string python_out(const std::string& desc, const CONTENT& script) class NamedTempDir: public boost::noncopyable { public: - // Use python() function to create a temp directory: I've found - // nothing in either Boost.Filesystem or APR quite like Python's - // tempfile.mkdtemp(). - // Special extra bonus: on Mac, mkdtemp() reports a pathname - // starting with /var/folders/something, whereas that's really a - // symlink to /private/var/folders/something. Have to use - // realpath() to compare properly. NamedTempDir(): - mPath(python_out("mkdtemp()", - "from __future__ import with_statement\n" - "import os.path, sys, tempfile\n" - "with open(sys.argv[1], 'w') as f:\n" - " f.write(os.path.normcase(os.path.normpath(os.path.realpath(tempfile.mkdtemp()))))\n")) - {} + mPath(NamedTempFile::temp_path()), + mCreated(boost::filesystem::create_directories(mPath)) + { + mPath = boost::filesystem::canonical(mPath); + } ~NamedTempDir() { - aprchk(apr_dir_remove(mPath.c_str(), gAPRPoolp)); + if (mCreated) + { + boost::filesystem::remove_all(mPath); + } } - std::string getName() const { return mPath; } + std::string getName() const { return mPath.string(); } private: - std::string mPath; + boost::filesystem::path mPath; + bool mCreated; }; /***************************************************************************** @@ -390,6 +398,7 @@ namespace tut // Have to have a named copy of this std::string so its c_str() value // will persist. std::string scriptname(script.getName()); + argv.push_back(test_python_script.c_str()); argv.push_back(scriptname.c_str()); argv.push_back(NULL); diff --git a/indra/llcommon/tests/test_python_script.py b/indra/llcommon/tests/test_python_script.py new file mode 100644 index 0000000000..c0c8661aa9 --- /dev/null +++ b/indra/llcommon/tests/test_python_script.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""\ +@file test_python_script.py +@author Nat Goodspeed +@date 2023-07-07 +@brief Work around a problem running Python within integration tests on GitHub + Windows runners. + +$LicenseInfo:firstyear=2023&license=viewerlgpl$ +Copyright (c) 2023, Linden Research, Inc. +$/LicenseInfo$ +""" + +import os +import sys + +# use pop() so that if the referenced script in turn looks at sys.argv, it +# will see its arguments rather than its own filename +_script = sys.argv.pop(1) +exec(open(_script).read()) -- cgit v1.2.3 From c4b5d089dad5680a0dd12b2d386b692318eb5c58 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 16:57:20 -0400 Subject: SL-18837: Partially revert e933ace, keeping useful tweaks. Introducing indirection via test_python_script.py did NOT address the "Access is denied" errors on GitHub Windows runners. --- indra/llcommon/tests/llleap_test.cpp | 25 +++++++++---------------- indra/llcommon/tests/llprocess_test.cpp | 13 ------------- indra/llcommon/tests/test_python_script.py | 20 -------------------- 3 files changed, 9 insertions(+), 49 deletions(-) delete mode 100644 indra/llcommon/tests/test_python_script.py (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 0fc741d9e1..e9edd165df 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -17,7 +17,6 @@ // std headers #include // external library headers -#include // other Linden headers #include "../test/lltut.h" #include "../test/namedtempfile.h" @@ -29,10 +28,6 @@ #include "stringize.h" #include "StringVec.h" -using boost::assign::list_of; - -StringVec sv(const StringVec& listof) { return listof; } - #if defined(LL_WINDOWS) #define sleep(secs) _sleep((secs) * 1000) @@ -217,9 +212,9 @@ namespace tut "time.sleep(1)\n"); LLLeapVector instances; instances.push_back(LLLeap::create(get_test_name(), - sv(list_of(PYTHON)(script.getName())))->getWeak()); + StringVec{PYTHON, script.getName()})->getWeak()); instances.push_back(LLLeap::create(get_test_name(), - sv(list_of(PYTHON)(script.getName())))->getWeak()); + StringVec{PYTHON, script.getName()})->getWeak()); // In this case we're simply establishing that two LLLeap instances // can coexist without throwing exceptions or bombing in any other // way. Wait for them to terminate. @@ -249,7 +244,7 @@ namespace tut "print('Hello from Python!')\n"); CaptureLog log(LLError::LEVEL_WARN); waitfor(LLLeap::create(get_test_name(), - sv(list_of(PYTHON)(script.getName())))); + StringVec{PYTHON, script.getName()})); ensure_contains("error log line", log.messageWith("invalid protocol"), "Hello from Python!"); } @@ -264,7 +259,7 @@ namespace tut "sys.stdout.write('Hello from Python!')\n"); CaptureLog log(LLError::LEVEL_WARN); waitfor(LLLeap::create(get_test_name(), - sv(list_of(PYTHON)(script.getName())))); + StringVec{PYTHON, script.getName()})); ensure_contains("error log line", log.messageWith("Discarding"), "Hello from Python!"); } @@ -278,7 +273,7 @@ namespace tut "sys.stdout.write('5a2:something')\n"); CaptureLog log(LLError::LEVEL_WARN); waitfor(LLLeap::create(get_test_name(), - sv(list_of(PYTHON)(script.getName())))); + StringVec{PYTHON, script.getName()})); ensure_contains("error log line", log.messageWith("invalid protocol"), "5a2:"); } @@ -390,7 +385,8 @@ namespace tut "result = '' if resp == dict(pump=replypump(), data='ack')\\\n" " else 'bad: ' + str(resp)\n" "send(pump='" << result.getName() << "', data=result)\n";}); - waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName())))); + waitfor(LLLeap::create(get_test_name(), + StringVec{PYTHON, script.getName()})); result.ensure(); } @@ -449,7 +445,7 @@ namespace tut " result = 'expected reqid=%s in %s' % (i, resp)\n" " break\n" "send(pump='" << result.getName() << "', data=result)\n";}); - waitfor(LLLeap::create(get_test_name(), sv(list_of(PYTHON)(script.getName()))), + waitfor(LLLeap::create(get_test_name(), StringVec{PYTHON, script.getName()}), 300); // needs more realtime than most tests result.ensure(); } @@ -516,10 +512,7 @@ namespace tut " (start, large[start:end], echoed[start:end]))\n" "sys.exit(1)\n";}); waitfor(LLLeap::create(test_name, - sv(list_of - (PYTHON) - (script.getName()) - (stringize(size)))), + StringVec{PYTHON, script.getName(), stringize(size)}), 180); // try a longer timeout result.ensure(); } diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index af99e97d66..c7d1a2c86a 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -124,17 +124,6 @@ void waitfor(LLProcess::handle h, const std::string& desc, int timeout=60) i < timeout); } -namespace { - -// find test helper, a sibling of this file -// nat 2023-07-07: we're currently using Boost 1.81, but -// path::replace_filename() (which is exactly what we need here) doesn't -// arrive until Boost 1.82. -auto test_python_script{ - (boost::filesystem::path(__FILE__).remove_filename() / "test_python_script.py").string() }; - -} - /** * Construct an LLProcess to run a Python script. */ @@ -156,7 +145,6 @@ struct PythonProcessLauncher mParams.desc = desc + " script"; mParams.executable = PYTHON; - mParams.args.add(test_python_script); mParams.args.add(mScript.getName()); } @@ -398,7 +386,6 @@ namespace tut // Have to have a named copy of this std::string so its c_str() value // will persist. std::string scriptname(script.getName()); - argv.push_back(test_python_script.c_str()); argv.push_back(scriptname.c_str()); argv.push_back(NULL); diff --git a/indra/llcommon/tests/test_python_script.py b/indra/llcommon/tests/test_python_script.py deleted file mode 100644 index c0c8661aa9..0000000000 --- a/indra/llcommon/tests/test_python_script.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 -"""\ -@file test_python_script.py -@author Nat Goodspeed -@date 2023-07-07 -@brief Work around a problem running Python within integration tests on GitHub - Windows runners. - -$LicenseInfo:firstyear=2023&license=viewerlgpl$ -Copyright (c) 2023, Linden Research, Inc. -$/LicenseInfo$ -""" - -import os -import sys - -# use pop() so that if the referenced script in turn looks at sys.argv, it -# will see its arguments rather than its own filename -_script = sys.argv.pop(1) -exec(open(_script).read()) -- cgit v1.2.3 From 1fc8758458c99b3a41965e33b3c62613c83e403a Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 17:31:50 -0400 Subject: SL-18837: Coax APR to log LLProcess launch attempts; show log file. --- indra/llcommon/tests/llprocess_test.cpp | 34 +++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index c7d1a2c86a..fbddc7f909 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -30,6 +30,7 @@ #include "../test/namedtempfile.h" #include "../test/catch_and_store_what_in.h" #include "stringize.h" +#include "lldir.h" #include "llsdutil.h" #include "llevents.h" #include "llstring.h" @@ -151,8 +152,37 @@ struct PythonProcessLauncher /// Launch Python script; verify that it launched void launch() { - mPy = LLProcess::create(mParams); - tut::ensure(STRINGIZE("Couldn't launch " << mDesc << " script"), bool(mPy)); + std::string logpath{ gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "apr.log") }; +#if LL_WINDOWS + _putenv_s("APR_LOG", logpath.c_str()); +#else + setenv("APR_LOG", logpath.c_str(), 1); +#endif + try + { + mPy = LLProcess::create(mParams); + tut::ensure(STRINGIZE("Couldn't launch " << mDesc << " script"), bool(mPy)); + } + catch (const tut::failure& err); + { + std::ifstream inf(logpath.c_str()); + if (! inf.is_open()) + { + LL_WARNS() << "Couldn't open '" << logpath << "'" << LL_ENDL; + } + else + { + LL_WARNS() << "==============================" << LL_ENDL; + LL_WARNS() << "From '" << logpath << "':" << LL_ENDL; + std::string line; + while (std::getline(line, inf)) + { + LL_WARNS() << line << LL_ENDL; + } + LL_WARNS() << "==============================" << LL_ENDL; + } + throw; + } } /// Run Python script and wait for it to complete. -- cgit v1.2.3 From 8f81e1fa87123ff6255e9ee82e68c414efe05cdd Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 17:47:57 -0400 Subject: SL-18837: Fix "lldir.h" #include --- indra/llcommon/tests/llprocess_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index fbddc7f909..9ee7890c7c 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -30,7 +30,7 @@ #include "../test/namedtempfile.h" #include "../test/catch_and_store_what_in.h" #include "stringize.h" -#include "lldir.h" +#include "../llfilesystem/lldir.h" #include "llsdutil.h" #include "llevents.h" #include "llstring.h" -- cgit v1.2.3 From 8aa3a0a7ed8cf3e3fedb2c98d6ea336fdd45e296 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 19:48:02 -0400 Subject: SL-18837: Fix spurious semi --- indra/llcommon/tests/llprocess_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 9ee7890c7c..fb5cf12cb2 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -163,7 +163,7 @@ struct PythonProcessLauncher mPy = LLProcess::create(mParams); tut::ensure(STRINGIZE("Couldn't launch " << mDesc << " script"), bool(mPy)); } - catch (const tut::failure& err); + catch (const tut::failure& err) { std::ifstream inf(logpath.c_str()); if (! inf.is_open()) -- cgit v1.2.3 From 09c5b01997e1d34e799a8a0ee3571bd181f9a665 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 20:02:33 -0400 Subject: SL-18837: Hook in LLDir to allow reading APR log file. --- indra/llcommon/CMakeLists.txt | 2 +- indra/llcommon/tests/llprocess_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 54020a4231..7412514e6b 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -308,7 +308,7 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(llleap "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llmainthreadtask "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llpounceable "" "${test_libs}") - LL_ADD_INTEGRATION_TEST(llprocess "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(llprocess "" "${test_libs};llfilesystem") LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llprocinfo "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}") diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index fb5cf12cb2..c6091bfeb1 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -175,7 +175,7 @@ struct PythonProcessLauncher LL_WARNS() << "==============================" << LL_ENDL; LL_WARNS() << "From '" << logpath << "':" << LL_ENDL; std::string line; - while (std::getline(line, inf)) + while (std::getline(inf, line)) { LL_WARNS() << line << LL_ENDL; } -- cgit v1.2.3 From 908fb3fed6b858da4dc2b1c840b849e30ade2046 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 7 Jul 2023 20:54:34 -0400 Subject: SL-18837: Ditch unreferenced name of caught exception --- indra/llcommon/tests/llprocess_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index c6091bfeb1..827837d62a 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -163,7 +163,7 @@ struct PythonProcessLauncher mPy = LLProcess::create(mParams); tut::ensure(STRINGIZE("Couldn't launch " << mDesc << " script"), bool(mPy)); } - catch (const tut::failure& err) + catch (const tut::failure&) { std::ifstream inf(logpath.c_str()); if (! inf.is_open()) -- cgit v1.2.3 From f37d2c307617302f2ed5dfead7e280da54a7d3e4 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Sat, 8 Jul 2023 09:04:33 -0400 Subject: SL-18837: Don't use LLDir, use NamedTempFile::temp_path. Remove llcommon circular dependency on llfilesystem, which doesn't work for this case anyway. --- indra/llcommon/CMakeLists.txt | 2 +- indra/llcommon/tests/llprocess_test.cpp | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 7412514e6b..54020a4231 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -308,7 +308,7 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(llleap "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llmainthreadtask "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llpounceable "" "${test_libs}") - LL_ADD_INTEGRATION_TEST(llprocess "" "${test_libs};llfilesystem") + LL_ADD_INTEGRATION_TEST(llprocess "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llprocinfo "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}") diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 827837d62a..6fcc6fd8aa 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -30,7 +30,6 @@ #include "../test/namedtempfile.h" #include "../test/catch_and_store_what_in.h" #include "stringize.h" -#include "../llfilesystem/lldir.h" #include "llsdutil.h" #include "llevents.h" #include "llstring.h" @@ -152,11 +151,9 @@ struct PythonProcessLauncher /// Launch Python script; verify that it launched void launch() { - std::string logpath{ gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "apr.log") }; + std::string logpath{ NamedTempFile::temp_path("apr", ".log").string() }; #if LL_WINDOWS _putenv_s("APR_LOG", logpath.c_str()); -#else - setenv("APR_LOG", logpath.c_str(), 1); #endif try { -- cgit v1.2.3 From 1ec6c744048a2905b0f2bf83f035a8fb8798dbdf Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Sat, 8 Jul 2023 11:08:16 -0400 Subject: SL-18837: Set APR_LOG once for the whole job instead of a new value for each LLProcess::create() invocation. Since the internal apr_log() function only looks at APR_LOG once per process, the first test (which succeeded, hence no log file dump) left the log file open with that same original pathname. Resetting the APR_LOG environment variable for subsequent runs only made the new code in llprocess_test look for files that were never created. --- indra/llcommon/tests/llprocess_test.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 6fcc6fd8aa..a01ec84547 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -151,10 +151,6 @@ struct PythonProcessLauncher /// Launch Python script; verify that it launched void launch() { - std::string logpath{ NamedTempFile::temp_path("apr", ".log").string() }; -#if LL_WINDOWS - _putenv_s("APR_LOG", logpath.c_str()); -#endif try { mPy = LLProcess::create(mParams); @@ -162,21 +158,25 @@ struct PythonProcessLauncher } catch (const tut::failure&) { - std::ifstream inf(logpath.c_str()); - if (! inf.is_open()) - { - LL_WARNS() << "Couldn't open '" << logpath << "'" << LL_ENDL; - } - else + const char* APR_LOG = getenv("APR_LOG"); + if (APR_LOG && *APR_LOG) { - LL_WARNS() << "==============================" << LL_ENDL; - LL_WARNS() << "From '" << logpath << "':" << LL_ENDL; - std::string line; - while (std::getline(inf, line)) + std::ifstream inf(APR_LOG); + if (! inf.is_open()) { - LL_WARNS() << line << LL_ENDL; + LL_WARNS() << "Couldn't open '" << APR_LOG << "'" << LL_ENDL; + } + else + { + LL_WARNS() << "==============================" << LL_ENDL; + LL_WARNS() << "From '" << APR_LOG << "':" << LL_ENDL; + std::string line; + while (std::getline(inf, line)) + { + LL_WARNS() << line << LL_ENDL; + } + LL_WARNS() << "==============================" << LL_ENDL; } - LL_WARNS() << "==============================" << LL_ENDL; } throw; } -- cgit v1.2.3 From 7dc6211ad5ea83685a35c6fff740278343aa8b9d Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Sat, 8 Jul 2023 14:08:16 -0400 Subject: SL-18837: Force llprocess_test and llleap_test to use just 'python'. On GitHub Windows runners, trying to make build.yaml set PYTHON=python in the environment doesn't work: integration tests still fail with "Access is denied" because they're still trying to execute the interpreter's full pathname. Instead, make llprocess_test and llleap_test detect the case of GitHub Windows and override the environment variable PYTHON with a baked-in string constant "python". --- indra/llcommon/tests/llleap_test.cpp | 11 ++++++++++- indra/llcommon/tests/llprocess_test.cpp | 13 ++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index e9edd165df..01515ecebf 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -193,11 +193,20 @@ namespace tut reader.getName().substr(0, reader.getName().length()-3))), PYTHON(LLStringUtil::getenv("PYTHON")) { +#if LL_WINDOWS + // Weirdly, on GitHub Windows runners, plain 'python' works much + // better than a full pathname. + const char* RUNNER_TEMP = getenv("RUNNER_TEMP"); + if (RUNNER_TEMP && *RUNNER_TEMP) + { + PYTHON = "python"; + } +#endif ensure("Set PYTHON to interpreter pathname", !PYTHON.empty()); } NamedExtTempFile reader; const std::string reader_module; - const std::string PYTHON; + std::string PYTHON; }; typedef test_group llleap_group; typedef llleap_group::object object; diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index a01ec84547..3ba3a8aab3 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -141,6 +141,15 @@ struct PythonProcessLauncher mScript("py", script) { auto PYTHON(LLStringUtil::getenv("PYTHON")); +#if LL_WINDOWS + // Weirdly, on GitHub Windows runners, plain 'python' works much better + // than a full pathname. + const char* RUNNER_TEMP = getenv("RUNNER_TEMP"); + if (RUNNER_TEMP && *RUNNER_TEMP) + { + PYTHON = "python"; + } +#endif tut::ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); mParams.desc = desc + " script"; @@ -1013,7 +1022,9 @@ namespace tut set_test_name("get*Pipe() validation"); PythonProcessLauncher py(get_test_name(), "from __future__ import print_function\n" - "print('this output is expected')\n"); + "import sys\n" + "print('this output is expected')\n" + "print('run by', sys.executable)\n"); py.mParams.files.add(LLProcess::FileParam("pipe")); // pipe for stdin py.mParams.files.add(LLProcess::FileParam()); // inherit stdout py.mParams.files.add(LLProcess::FileParam("pipe")); // pipe for stderr -- cgit v1.2.3 From 31ccef8a666da54312a55663a7ac03061c4903be Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 10 Jul 2023 14:35:41 -0400 Subject: SL-18837: Revert "Force llprocess_test and llleap_test to use just 'python'." Turns out that the pathname of the Python executable wasn't the issue. This reverts commit 7dc6211ad5ea83685a35c6fff740278343aa8b9d. --- indra/llcommon/tests/llleap_test.cpp | 11 +---------- indra/llcommon/tests/llprocess_test.cpp | 13 +------------ 2 files changed, 2 insertions(+), 22 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 01515ecebf..e9edd165df 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -193,20 +193,11 @@ namespace tut reader.getName().substr(0, reader.getName().length()-3))), PYTHON(LLStringUtil::getenv("PYTHON")) { -#if LL_WINDOWS - // Weirdly, on GitHub Windows runners, plain 'python' works much - // better than a full pathname. - const char* RUNNER_TEMP = getenv("RUNNER_TEMP"); - if (RUNNER_TEMP && *RUNNER_TEMP) - { - PYTHON = "python"; - } -#endif ensure("Set PYTHON to interpreter pathname", !PYTHON.empty()); } NamedExtTempFile reader; const std::string reader_module; - std::string PYTHON; + const std::string PYTHON; }; typedef test_group llleap_group; typedef llleap_group::object object; diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 3ba3a8aab3..a01ec84547 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -141,15 +141,6 @@ struct PythonProcessLauncher mScript("py", script) { auto PYTHON(LLStringUtil::getenv("PYTHON")); -#if LL_WINDOWS - // Weirdly, on GitHub Windows runners, plain 'python' works much better - // than a full pathname. - const char* RUNNER_TEMP = getenv("RUNNER_TEMP"); - if (RUNNER_TEMP && *RUNNER_TEMP) - { - PYTHON = "python"; - } -#endif tut::ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); mParams.desc = desc + " script"; @@ -1022,9 +1013,7 @@ namespace tut set_test_name("get*Pipe() validation"); PythonProcessLauncher py(get_test_name(), "from __future__ import print_function\n" - "import sys\n" - "print('this output is expected')\n" - "print('run by', sys.executable)\n"); + "print('this output is expected')\n"); py.mParams.files.add(LLProcess::FileParam("pipe")); // pipe for stdin py.mParams.files.add(LLProcess::FileParam()); // inherit stdout py.mParams.files.add(LLProcess::FileParam("pipe")); // pipe for stderr -- cgit v1.2.3 From d8292a629149c2cfdda6ae9df4e87aa117153c21 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 10 Jul 2023 14:46:14 -0400 Subject: SL-18837: Disable APR_LOG for now, but leave notes for the future. --- indra/llcommon/tests/llprocess_test.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index a01ec84547..9ca664c80c 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -158,6 +158,9 @@ struct PythonProcessLauncher } catch (const tut::failure&) { + // On Windows, if APR_LOG is set, our version of APR's + // apr_create_proc() logs to the specified file. If this test + // failed, try to report that log. const char* APR_LOG = getenv("APR_LOG"); if (APR_LOG && *APR_LOG) { -- cgit v1.2.3 From c77737b925e3687e47d3a1dce1b7e8b481302741 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 10 Jul 2023 15:26:21 -0400 Subject: SL-18837: Windows failures in setWorkingDirectory(): C: vs. c: (sigh) Normalize the case of the name of the temp directory for string comparison. --- indra/llcommon/tests/llprocess_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 9ca664c80c..c1cb2af7fe 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -591,7 +591,7 @@ namespace tut " f.write(os.path.normcase(os.path.normpath(os.getcwd())))\n"); // Before running, call setWorkingDirectory() py.mParams.cwd = tempdir.getName(); - ensure_equals("os.getcwd()", py.run_read(), tempdir.getName()); + ensure_equals("os.getcwd()", py.run_read(), utf8str_tolower(tempdir.getName())); } template<> template<> -- cgit v1.2.3 From 54f9ca5404c05a4031c1c12caf24b88048704cbd Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 17 Jul 2023 15:41:26 -0400 Subject: SL-18837: Merge branch 'actions' into actions-build-sh --- indra/llcommon/llprocessor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llprocessor.cpp b/indra/llcommon/llprocessor.cpp index 4a1a81f083..28f8bc2b93 100644 --- a/indra/llcommon/llprocessor.cpp +++ b/indra/llcommon/llprocessor.cpp @@ -746,7 +746,7 @@ private: __cpuid(0x1, eax, ebx, ecx, edx); if(feature_infos[0] != (S32)edx) { - LL_ERRS() << "machdep.cpu.feature_bits doesn't match expected cpuid result!" << LL_ENDL; + LL_WARNS() << "machdep.cpu.feature_bits doesn't match expected cpuid result!" << LL_ENDL; } #endif // LL_RELEASE_FOR_DOWNLOAD -- cgit v1.2.3 From 167ac704c8387c531630949860e33fb5a59789a8 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 17 Jul 2023 16:20:59 -0400 Subject: SL-18837: Clean up some redundancy in llrand.cpp. --- indra/llcommon/llrand.cpp | 96 +++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 41 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llrand.cpp b/indra/llcommon/llrand.cpp index cb28a8f5c3..4e4345f37a 100644 --- a/indra/llcommon/llrand.cpp +++ b/indra/llcommon/llrand.cpp @@ -58,14 +58,18 @@ * to restore uniform distribution. */ +template +inline REAL ll_internal_random(); + // *NOTE: The system rand implementation is probably not correct. #define LL_USE_SYSTEM_RAND 0 +/***************************************************************************** +* The LL_USE_SYSTEM_RAND implementation has been disabled since June 2008. +*****************************************************************************/ #if LL_USE_SYSTEM_RAND #include -#endif -#if LL_USE_SYSTEM_RAND class LLSeedRand { public: @@ -79,25 +83,25 @@ public: } }; static LLSeedRand sRandomSeeder; -inline F64 ll_internal_random_double() + +template <> +inline F64 ll_internal_random() { #if LL_WINDOWS - return (F64)rand() / (F64)RAND_MAX; + return (F64)rand() / (F64)(RAND_MAX+1); #else return drand48(); #endif } -inline F32 ll_internal_random_float() -{ -#if LL_WINDOWS - return (F32)rand() / (F32)RAND_MAX; -#else - return (F32)drand48(); -#endif -} -#else + +/***************************************************************************** +* This is the implementation we've been using. +*****************************************************************************/ +#else // LL_USE_SYSTEM_RAND static LLRandLagFib2281 gRandomGenerator(LLUUID::getRandomSeed()); -inline F64 ll_internal_random_double() + +template <> +inline F64 ll_internal_random() { // *HACK: Through experimentation, we have found that dual core // CPUs (or at least multi-threaded processes) seem to @@ -107,16 +111,40 @@ inline F64 ll_internal_random_double() if(!((rv >= 0.0) && (rv < 1.0))) return fmod(rv, 1.0); return rv; } +#endif // LL_USE_SYSTEM_RAND + +/***************************************************************************** +* Functions common to both implementations +*****************************************************************************/ +template <> +inline F32 ll_internal_random() +{ + return F32(ll_internal_random()); +} + +/*------------------------------ F64 aliases -------------------------------*/ +inline F64 ll_internal_random_double() +{ + return ll_internal_random(); +} + +F64 ll_drand() +{ + return ll_internal_random_double(); +} +/*------------------------------ F32 aliases -------------------------------*/ inline F32 ll_internal_random_float() { - // The clamping rules are described above. - F32 rv = (F32)gRandomGenerator(); - if(!((rv >= 0.0f) && (rv < 1.0f))) return fmod(rv, 1.f); - return rv; + return ll_internal_random(); } -#endif +F32 ll_frand() +{ + return ll_internal_random_float(); +} + +/*-------------------------- clamped random range --------------------------*/ S32 ll_rand() { return ll_rand(RAND_MAX); @@ -130,42 +158,28 @@ S32 ll_rand(S32 val) return rv; } -F32 ll_frand() -{ - return ll_internal_random_float(); -} - -F32 ll_frand(F32 val) +template +REAL ll_grand(REAL val) { // The clamping rules are described above. - F32 rv = ll_internal_random_float() * val; + REAL rv = ll_internal_random() * val; if(val > 0) { - if(rv >= val) return 0.0f; + if(rv >= val) return REAL(); } else { - if(rv <= val) return 0.0f; + if(rv <= val) return REAL(); } return rv; } -F64 ll_drand() +F32 ll_frand(F32 val) { - return ll_internal_random_double(); + return ll_grand(val); } F64 ll_drand(F64 val) { - // The clamping rules are described above. - F64 rv = ll_internal_random_double() * val; - if(val > 0) - { - if(rv >= val) return 0.0; - } - else - { - if(rv <= val) return 0.0; - } - return rv; + return ll_grand(val); } -- cgit v1.2.3 From 4b158580e5654615d2a5510267bf76392c9666fa Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Mon, 17 Jul 2023 16:47:50 -0400 Subject: SL-18837: Lowercasing pathname for string compare is Windows-only. --- indra/llcommon/tests/llprocess_test.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index c1cb2af7fe..b6b297b8d7 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -591,7 +591,13 @@ namespace tut " f.write(os.path.normcase(os.path.normpath(os.getcwd())))\n"); // Before running, call setWorkingDirectory() py.mParams.cwd = tempdir.getName(); - ensure_equals("os.getcwd()", py.run_read(), utf8str_tolower(tempdir.getName())); + std::string expected{ tempdir.getName() }; +#if LL_WINDOWS + // SIGH, don't get tripped up by "C:" != "c:" -- + // but on the Mac, using tolower() fails because "/users" != "/Users"! + expected = utf8str_tolower(expected); +#endif + ensure_equals("os.getcwd()", py.run_read(), expected); } template<> template<> -- cgit v1.2.3 From 14d0b514af4e70956ecc2fbf6fe4c2e745e144a8 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 18 Jul 2023 09:45:00 -0400 Subject: SL-18837: Ditch inactive llrand.cpp LL_USE_SYSTEM_RAND code. LL_USE_SYSTEM_RAND has been disabled since June 2008; that code only clutters the implementation we actually use. --- indra/llcommon/llrand.cpp | 46 +++------------------------------------------- 1 file changed, 3 insertions(+), 43 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llrand.cpp b/indra/llcommon/llrand.cpp index 4e4345f37a..33afc50cf7 100644 --- a/indra/llcommon/llrand.cpp +++ b/indra/llcommon/llrand.cpp @@ -58,48 +58,12 @@ * to restore uniform distribution. */ +static LLRandLagFib2281 gRandomGenerator(LLUUID::getRandomSeed()); + +// no default implementation, only specific F64 and F32 specializations template inline REAL ll_internal_random(); -// *NOTE: The system rand implementation is probably not correct. -#define LL_USE_SYSTEM_RAND 0 - -/***************************************************************************** -* The LL_USE_SYSTEM_RAND implementation has been disabled since June 2008. -*****************************************************************************/ -#if LL_USE_SYSTEM_RAND -#include - -class LLSeedRand -{ -public: - LLSeedRand() - { -#if LL_WINDOWS - srand(LLUUID::getRandomSeed()); -#else - srand48(LLUUID::getRandomSeed()); -#endif - } -}; -static LLSeedRand sRandomSeeder; - -template <> -inline F64 ll_internal_random() -{ -#if LL_WINDOWS - return (F64)rand() / (F64)(RAND_MAX+1); -#else - return drand48(); -#endif -} - -/***************************************************************************** -* This is the implementation we've been using. -*****************************************************************************/ -#else // LL_USE_SYSTEM_RAND -static LLRandLagFib2281 gRandomGenerator(LLUUID::getRandomSeed()); - template <> inline F64 ll_internal_random() { @@ -111,11 +75,7 @@ inline F64 ll_internal_random() if(!((rv >= 0.0) && (rv < 1.0))) return fmod(rv, 1.0); return rv; } -#endif // LL_USE_SYSTEM_RAND -/***************************************************************************** -* Functions common to both implementations -*****************************************************************************/ template <> inline F32 ll_internal_random() { -- cgit v1.2.3 From 25efba151f98308a0e2d9af52a76173be3f8aa04 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 29 Aug 2023 11:12:32 -0400 Subject: SL-18837: On Windows, LLLeap partial final line test failed. Add DEBUG log output to try to diagnose. --- indra/llcommon/llleap.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index c87c0758fe..060e96557d 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -376,6 +376,17 @@ public: // Read all remaining bytes and log. LL_INFOS("LLLeap") << mDesc << ": " << rest << LL_ENDL; } + /*--------------------------- diagnostic ---------------------------*/ + else if (data["eof"].asBoolean()) + { + LL_DEBUGS("LLLeap") << mDesc << " ended, no partial line" << LL_ENDL; + } + else + { + LL_DEBUGS("LLLeap") << mDesc << " (still running, " << childerr.size() + << " bytes pending)" << LL_ENDL; + } + /*------------------------- end diagnostic -------------------------*/ return false; } -- cgit v1.2.3 From 6a219d147d0761fce1fd75dc75f98b4e5283feb7 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 15 Aug 2023 15:32:35 -0400 Subject: SL-18837: Enlarge default coroutine stack size. A test executable on a GitHub Windows runner failed with C00000FD, which reports stack overflow. (cherry picked from commit aab7b4ba3812e5876b1205285bcfd8cff96bcac9) --- indra/llcommon/llcoros.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llcoros.cpp b/indra/llcommon/llcoros.cpp index 70d8dfc8b9..cfaf3415e7 100644 --- a/indra/llcommon/llcoros.cpp +++ b/indra/llcommon/llcoros.cpp @@ -123,11 +123,7 @@ LLCoros::LLCoros(): // Previously we used // boost::context::guarded_stack_allocator::default_stacksize(); // empirically this is insufficient. -#if ADDRESS_SIZE == 64 - mStackSize(512*1024), -#else - mStackSize(256*1024), -#endif + mStackSize(768*1024), // mCurrent does NOT own the current CoroData instance -- it simply // points to it. So initialize it with a no-op deleter. mCurrent{ [](CoroData*){} } -- cgit v1.2.3 From 95aa00f7427b7d19ab502862b3018012c1bf1904 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 7 Sep 2023 13:40:46 -0400 Subject: SL-18837: Fix minor merge glitch. --- indra/llcommon/tests/llsdserialize_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 2de7df6f36..76e9ecc293 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -2081,7 +2081,7 @@ namespace tut " for item in DATA:\n" " serialized = llsd." << pyformatter << "(item)\n" " f.write(lenformat.pack(len(serialized)))\n" - " f.write(serialized)\n"); + " f.write(serialized)\n";}); std::ifstream inf(file.getName().c_str()); LLSD item; -- cgit v1.2.3 From a45c9f68c3e2600f48b25cc5cc74ef47cd83005b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 8 Sep 2023 08:58:32 -0400 Subject: SL-18837: Add debugging output to llsdserialize_test.cpp. --- indra/llcommon/tests/llsdserialize_test.cpp | 90 ++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 76e9ecc293..ca63e74c6c 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -55,7 +55,9 @@ typedef U32 uint32_t; #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include #include +#include typedef std::function FormatterFunction; typedef std::function ParserFunction; @@ -65,6 +67,81 @@ std::vector string_to_vector(const std::string& str) return std::vector(str.begin(), str.end()); } +// Format a given byte string as 2-digit hex values, no separators +// Usage: std::cout << hexdump(somestring) << ... +class hexdump +{ +public: + hexdump(const char* data, size_t len): + hexdump(reinterpret_cast(data), len) + {} + + hexdump(const U8* data, size_t len): + mData(data, data + len) + {} + + hexdump(const hexdump&) = delete; + hexdump& operator=(const hexdump&) = delete; + + friend std::ostream& operator<<(std::ostream& out, const hexdump& self) + { + auto oldfmt{ out.flags() }; + auto oldfill{ out.fill() }; + out.setf(std::ios_base::hex, std::ios_base::basefield); + out.fill('0'); + for (auto c : self.mData) + { + out << std::setw(2) << unsigned(c); + } + out.setf(oldfmt, std::ios_base::basefield); + out.fill(oldfill); + return out; + } + +private: + std::vector mData; +}; + +// Format a given byte string as a mix of printable characters and, for each +// non-printable character, "\xnn" +// Usage: std::cout << hexmix(somestring) << ... +class hexmix +{ +public: + hexmix(const char* data, size_t len): + mData(data, len) + {} + + hexmix(const hexmix&) = delete; + hexmix& operator=(const hexmix&) = delete; + + friend std::ostream& operator<<(std::ostream& out, const hexmix& self) + { + auto oldfmt{ out.flags() }; + auto oldfill{ out.fill() }; + out.setf(std::ios_base::hex, std::ios_base::basefield); + out.fill('0'); + for (auto c : self.mData) + { + // std::isprint() must be passed an unsigned char! + if (std::isprint(static_cast(c))) + { + out << c; + } + else + { + out << "\\x" << std::setw(2) << unsigned(c); + } + } + out.setf(oldfmt, std::ios_base::basefield); + out.fill(oldfill); + return out; + } + +private: + std::string mData; +}; + namespace tut { struct sd_xml_data @@ -1909,7 +1986,14 @@ namespace tut auto buffstr{ buffer.str() }; int bufflen{ static_cast(buffstr.length()) }; out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); + LL_DEBUGS("topy") << "Wrote length: " + << hexdump(reinterpret_cast(&bufflen), + sizeof(bufflen)) + << LL_ENDL; out.write(buffstr.c_str(), buffstr.length()); + LL_DEBUGS("topy") << "Wrote data: " + << hexmix(buffstr.c_str(), buffstr.length()) + << LL_ENDL; } } @@ -1938,8 +2022,8 @@ namespace tut " else:\n" " raise AssertionError('Too many data items')\n"; - // Create an llsdXXXXXX file containing 'data' serialized to - // notation. + // Create an llsdXXXXXX file containing 'data' serialized per + // FormatterFunction. NamedTempFile file("llsd", // NamedTempFile's function constructor // takes a callable. To this callable it passes the @@ -1958,11 +2042,13 @@ namespace tut "lenformat = struct.Struct('i')\n" "def parse_each(inf):\n" " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " print('Read length:', ''.join(('%02x' % b) for b in rawlen))\n" " len = lenformat.unpack(rawlen)[0]\n" // Since llsd.parse() has no max_bytes argument, instead of // passing the input stream directly to parse(), read the item // into a distinct bytes object and parse that. " data = inf.read(len)\n" + " print('Read data: ', repr(data))\n" " try:\n" " frombytes = llsd.parse(data)\n" " except llsd.LLSDParseError as err:\n" -- cgit v1.2.3 From c7546ea65e55143ff3d2d82d8c289bbac7fffe0f Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Fri, 8 Sep 2023 14:14:09 -0400 Subject: SL-18837: Make llsdserialize_test debug output conditional. Move hexdump() and hexmix() stream formatters to new hexdump.h for potential use by other tests. In toPythonUsing() helper function, add a temp file to receive Python script debug output, and direct debug output to that file. On test failure, dump the contents of that file to the log. Give NamedTempFile::peep() an optional target std::ostream; refactor implementation as peep_via() that accepts a callable to process each text line. Add operator<<() to stream the contents of a NamedTempFile object to ostream -- but don't use that with LL_DEBUGS(), as it flattens the file contents into a single log line. Instead add peep_log(), which streams each individual text line to LL_DEBUGS(). --- indra/llcommon/tests/llsdserialize_test.cpp | 202 +++++++++++----------------- 1 file changed, 75 insertions(+), 127 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index ca63e74c6c..ac40125f75 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -52,12 +52,12 @@ typedef U32 uint32_t; #include "llformat.h" #include "llmemorystream.h" +#include "../test/hexdump.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" -#include +#include "StringVec.h" #include -#include typedef std::function FormatterFunction; typedef std::function ParserFunction; @@ -67,81 +67,6 @@ std::vector string_to_vector(const std::string& str) return std::vector(str.begin(), str.end()); } -// Format a given byte string as 2-digit hex values, no separators -// Usage: std::cout << hexdump(somestring) << ... -class hexdump -{ -public: - hexdump(const char* data, size_t len): - hexdump(reinterpret_cast(data), len) - {} - - hexdump(const U8* data, size_t len): - mData(data, data + len) - {} - - hexdump(const hexdump&) = delete; - hexdump& operator=(const hexdump&) = delete; - - friend std::ostream& operator<<(std::ostream& out, const hexdump& self) - { - auto oldfmt{ out.flags() }; - auto oldfill{ out.fill() }; - out.setf(std::ios_base::hex, std::ios_base::basefield); - out.fill('0'); - for (auto c : self.mData) - { - out << std::setw(2) << unsigned(c); - } - out.setf(oldfmt, std::ios_base::basefield); - out.fill(oldfill); - return out; - } - -private: - std::vector mData; -}; - -// Format a given byte string as a mix of printable characters and, for each -// non-printable character, "\xnn" -// Usage: std::cout << hexmix(somestring) << ... -class hexmix -{ -public: - hexmix(const char* data, size_t len): - mData(data, len) - {} - - hexmix(const hexmix&) = delete; - hexmix& operator=(const hexmix&) = delete; - - friend std::ostream& operator<<(std::ostream& out, const hexmix& self) - { - auto oldfmt{ out.flags() }; - auto oldfill{ out.fill() }; - out.setf(std::ios_base::hex, std::ios_base::basefield); - out.fill('0'); - for (auto c : self.mData) - { - // std::isprint() must be passed an unsigned char! - if (std::isprint(static_cast(c))) - { - out << c; - } - else - { - out << "\\x" << std::setw(2) << unsigned(c); - } - } - out.setf(oldfmt, std::ios_base::basefield); - out.fill(oldfill); - return out; - } - -private: - std::string mData; -}; - namespace tut { struct sd_xml_data @@ -1868,16 +1793,12 @@ namespace tut // helper for TestPythonCompatible static std::string import_llsd("import os.path\n" "import sys\n" - "try:\n" - // new freestanding llsd package - " import llsd\n" - "except ImportError:\n" - // older llbase.llsd module - " from llbase import llsd\n"); + "import llsd\n"); // helper for TestPythonCompatible - template - void python(const std::string& desc, const CONTENT& script, int expect=0) + template + void python_expect(const std::string& desc, const CONTENT& script, int expect=0, + ARGS&&... args) { auto PYTHON(LLStringUtil::getenv("PYTHON")); ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); @@ -1888,7 +1809,8 @@ namespace tut std::string q("\""); std::string qPYTHON(q + PYTHON + q); std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), + std::forward(args)..., NULL); if (rc == -1) { char buffer[256]; @@ -1904,6 +1826,10 @@ namespace tut LLProcess::Params params; params.executable = PYTHON; params.args.add(scriptfile.getName()); + for (const std::string& arg : StringVec{ std::forward(args)... }) + { + params.args.add(arg); + } LLProcessPtr py(LLProcess::create(params)); ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); // Implementing timeout would mean messing with alarm() and @@ -1938,6 +1864,14 @@ namespace tut #endif } + // helper for TestPythonCompatible + template + void python(const std::string& desc, const CONTENT& script, ARGS&&... args) + { + // plain python() expects rc 0 + python_expect(desc, script, 0, std::forward(args)...); + } + struct TestPythonCompatible { TestPythonCompatible() {} @@ -1952,10 +1886,10 @@ namespace tut void TestPythonCompatibleObject::test<1>() { set_test_name("verify python()"); - python("hello", - "import sys\n" - "sys.exit(17)\n", - 17); // expect nonzero rc + python_expect("hello", + "import sys\n" + "sys.exit(17)\n", + 17); // expect nonzero rc } template<> template<> @@ -1986,14 +1920,14 @@ namespace tut auto buffstr{ buffer.str() }; int bufflen{ static_cast(buffstr.length()) }; out.write(reinterpret_cast(&bufflen), sizeof(bufflen)); - LL_DEBUGS("topy") << "Wrote length: " - << hexdump(reinterpret_cast(&bufflen), - sizeof(bufflen)) - << LL_ENDL; + LL_DEBUGS() << "Wrote length: " + << hexdump(reinterpret_cast(&bufflen), + sizeof(bufflen)) + << LL_ENDL; out.write(buffstr.c_str(), buffstr.length()); - LL_DEBUGS("topy") << "Wrote data: " - << hexmix(buffstr.c_str(), buffstr.length()) - << LL_ENDL; + LL_DEBUGS() << "Wrote data: " + << hexmix(buffstr.c_str(), buffstr.length()) + << LL_ENDL; } } @@ -2033,36 +1967,50 @@ namespace tut (std::ostream& out) { writeLLSDArray(serialize, out, cdata); }); - python("read C++ " + desc, - [&](std::ostream& out){ out << - import_llsd << - "from functools import partial\n" - "import io\n" - "import struct\n" - "lenformat = struct.Struct('i')\n" - "def parse_each(inf):\n" - " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" - " print('Read length:', ''.join(('%02x' % b) for b in rawlen))\n" - " len = lenformat.unpack(rawlen)[0]\n" - // Since llsd.parse() has no max_bytes argument, instead of - // passing the input stream directly to parse(), read the item - // into a distinct bytes object and parse that. - " data = inf.read(len)\n" - " print('Read data: ', repr(data))\n" - " try:\n" - " frombytes = llsd.parse(data)\n" - " except llsd.LLSDParseError as err:\n" - " print(f'*** {err}')\n" - " print(f'Bad content:\\n{data!r}')\n" - " raise\n" - // Also try parsing from a distinct stream. - " stream = io.BytesIO(data)\n" - " fromstream = llsd.parse(stream)\n" - " assert frombytes == fromstream\n" - " yield frombytes\n" - << pydata << - // Don't forget raw-string syntax for Windows pathnames. - "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n";}); + // 'debug' starts empty because it's intended as an output file + NamedTempFile debug("debug", ""); + + try + { + python("read C++ " + desc, + [&](std::ostream& out){ out << + import_llsd << + "from functools import partial\n" + "import io\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " print('Read length:', ''.join(('%02x' % b) for b in rawlen),\n" + " file=debug)\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " print('Read data: ', repr(data), file=debug)\n" + " try:\n" + " frombytes = llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" + << pydata << + // Don't forget raw-string syntax for Windows pathnames. + "debug = open(r'" << debug.getName() << "', 'w')\n" + "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n";}); + } + catch (const failure&) + { + LL_DEBUGS() << "Script debug output:" << LL_ENDL; + debug.peep_log(); + throw; + } } template<> template<> -- cgit v1.2.3 From 7504b1c319373c950e8b8c2c7a8b2f0d9abf1d8b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 5 Oct 2023 10:17:09 -0400 Subject: SL-18837: When llrand_test.cpp fails, display the failing value. It's frustrating and unactionable to have a failing test report merely that the random value was greater than the specified high end. Okay, so what was the value? If it's supposed to be less than the high end, did it happen to be equal? Or was it garbage? We can't reproduce the failure by rerunning! The new ensure_in_exc_range(), ensure_in_inc_range() mechanism is somewhat complex because exactly one test allows equality with the high end of the expected range, where the rest mandate that the function return less than the high end. If that's a bug in the test -- if every llrand function is supposed to return less than the high end -- then we could simplify the test logic. --- indra/llcommon/tests/llrand_test.cpp | 60 +++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 28 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llrand_test.cpp b/indra/llcommon/tests/llrand_test.cpp index 383e6f9e0a..c8e2d19372 100644 --- a/indra/llcommon/tests/llrand_test.cpp +++ b/indra/llcommon/tests/llrand_test.cpp @@ -29,7 +29,30 @@ #include "../test/lltut.h" #include "../llrand.h" +#include "stringize.h" +template +void ensure_in_range_using(const std::string_view& name, + NUMBER value, NUMBER low, NUMBER high, + const std::string_view& compdesc, HIGHCOMP&& highcomp) +{ + auto failmsg{ stringize(name, " >= ", low, " (", value, ')') }; + tut::ensure(failmsg, (value >= low)); + failmsg = stringize(name, ' ', compdesc, ' ', high, " (", value, ')'); + tut::ensure(failmsg, std::forward(highcomp)(value, high)); +} + +template +void ensure_in_exc_range(const std::string_view& name, NUMBER value, NUMBER low, NUMBER high) +{ + ensure_in_range_using(name, value, low, high, "<", std::less()); +} + +template +void ensure_in_inc_range(const std::string_view& name, NUMBER value, NUMBER low, NUMBER high) +{ + ensure_in_range_using(name, value, low, high, "<=", std::less_equal()); +} namespace tut { @@ -44,84 +67,65 @@ namespace tut template<> template<> void random_object_t::test<1>() { - F32 number = 0.0f; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_frand(); - ensure("frand >= 0", (number >= 0.0f)); - ensure("frand < 1", (number < 1.0f)); + ensure_in_exc_range("frand", ll_frand(), 0.0f, 1.0f); } } template<> template<> void random_object_t::test<2>() { - F64 number = 0.0f; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_drand(); - ensure("drand >= 0", (number >= 0.0)); - ensure("drand < 1", (number < 1.0)); + ensure_in_exc_range("drand", ll_drand(), 0.0, 1.0); } } template<> template<> void random_object_t::test<3>() { - F32 number = 0.0f; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_frand(2.0f) - 1.0f; - ensure("frand >= 0", (number >= -1.0f)); - ensure("frand < 1", (number <= 1.0f)); + ensure_in_inc_range("frand(2.0f)", ll_frand(2.0f) - 1.0f, -1.0f, 1.0f); } } template<> template<> void random_object_t::test<4>() { - F32 number = 0.0f; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_frand(-7.0); - ensure("drand <= 0", (number <= 0.0)); - ensure("drand > -7", (number > -7.0)); + // Negate the result so we don't have to allow a templated low-end + // comparison as well. + ensure_in_exc_range("-frand(-7.0)", -ll_frand(-7.0), 0.0f, 7.0f); } } template<> template<> void random_object_t::test<5>() { - F64 number = 0.0f; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_drand(-2.0); - ensure("drand <= 0", (number <= 0.0)); - ensure("drand > -2", (number > -2.0)); + ensure_in_exc_range("-drand(-2.0)", -ll_drand(-2.0), 0.0, 2.0); } } template<> template<> void random_object_t::test<6>() { - S32 number = 0; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_rand(100); - ensure("rand >= 0", (number >= 0)); - ensure("rand < 100", (number < 100)); + ensure_in_exc_range("rand(100)", ll_rand(100), 0, 100); } } template<> template<> void random_object_t::test<7>() { - S32 number = 0; for(S32 ii = 0; ii < 100000; ++ii) { - number = ll_rand(-127); - ensure("rand <= 0", (number <= 0)); - ensure("rand > -127", (number > -127)); + ensure_in_exc_range("-rand(-127)", -ll_rand(-127), 0, 127); } } } -- cgit v1.2.3 From f5a34fd074bda091732a8ae0a4cf6f4e0358a140 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 12 Oct 2023 16:55:04 -0400 Subject: SL-18837: Unify all llrand_test.cpp in-range tests. The header file documents that no llrand function should ever return a value equal to the passed extent, so the one test in llrand_test.cpp that checked less than or equal to the high end of the range was anomalous. But changing that to an exclusive range means that we no longer need separate exclusive range and inclusive range functions. Replace ensure_in_range_using(), ensure_in_exc_range() and ensure_in_inc_range() with a grand unified (simplified) ensure_in_range() function. --- indra/llcommon/tests/llrand_test.cpp | 43 +++++++++++++++--------------------- 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/tests/llrand_test.cpp b/indra/llcommon/tests/llrand_test.cpp index c8e2d19372..ac5a33d0ba 100644 --- a/indra/llcommon/tests/llrand_test.cpp +++ b/indra/llcommon/tests/llrand_test.cpp @@ -31,27 +31,20 @@ #include "../llrand.h" #include "stringize.h" -template -void ensure_in_range_using(const std::string_view& name, - NUMBER value, NUMBER low, NUMBER high, - const std::string_view& compdesc, HIGHCOMP&& highcomp) +// In llrand.h, every function is documented to return less than the high end +// -- specifically, because you can pass a negative extent, they're documented +// never to return a value equal to the extent. +// So that we don't need two different versions of ensure_in_range(), when +// testing extent < 0, negate the return value and the extent before passing +// into ensure_in_range(). +template +void ensure_in_range(const std::string_view& name, + NUMBER value, NUMBER low, NUMBER high) { auto failmsg{ stringize(name, " >= ", low, " (", value, ')') }; tut::ensure(failmsg, (value >= low)); - failmsg = stringize(name, ' ', compdesc, ' ', high, " (", value, ')'); - tut::ensure(failmsg, std::forward(highcomp)(value, high)); -} - -template -void ensure_in_exc_range(const std::string_view& name, NUMBER value, NUMBER low, NUMBER high) -{ - ensure_in_range_using(name, value, low, high, "<", std::less()); -} - -template -void ensure_in_inc_range(const std::string_view& name, NUMBER value, NUMBER low, NUMBER high) -{ - ensure_in_range_using(name, value, low, high, "<=", std::less_equal()); + failmsg = stringize(name, " < ", high, " (", value, ')'); + tut::ensure(failmsg, (value < high)); } namespace tut @@ -69,7 +62,7 @@ namespace tut { for(S32 ii = 0; ii < 100000; ++ii) { - ensure_in_exc_range("frand", ll_frand(), 0.0f, 1.0f); + ensure_in_range("frand", ll_frand(), 0.0f, 1.0f); } } @@ -78,7 +71,7 @@ namespace tut { for(S32 ii = 0; ii < 100000; ++ii) { - ensure_in_exc_range("drand", ll_drand(), 0.0, 1.0); + ensure_in_range("drand", ll_drand(), 0.0, 1.0); } } @@ -87,7 +80,7 @@ namespace tut { for(S32 ii = 0; ii < 100000; ++ii) { - ensure_in_inc_range("frand(2.0f)", ll_frand(2.0f) - 1.0f, -1.0f, 1.0f); + ensure_in_range("frand(2.0f)", ll_frand(2.0f) - 1.0f, -1.0f, 1.0f); } } @@ -98,7 +91,7 @@ namespace tut { // Negate the result so we don't have to allow a templated low-end // comparison as well. - ensure_in_exc_range("-frand(-7.0)", -ll_frand(-7.0), 0.0f, 7.0f); + ensure_in_range("-frand(-7.0)", -ll_frand(-7.0), 0.0f, 7.0f); } } @@ -107,7 +100,7 @@ namespace tut { for(S32 ii = 0; ii < 100000; ++ii) { - ensure_in_exc_range("-drand(-2.0)", -ll_drand(-2.0), 0.0, 2.0); + ensure_in_range("-drand(-2.0)", -ll_drand(-2.0), 0.0, 2.0); } } @@ -116,7 +109,7 @@ namespace tut { for(S32 ii = 0; ii < 100000; ++ii) { - ensure_in_exc_range("rand(100)", ll_rand(100), 0, 100); + ensure_in_range("rand(100)", ll_rand(100), 0, 100); } } @@ -125,7 +118,7 @@ namespace tut { for(S32 ii = 0; ii < 100000; ++ii) { - ensure_in_exc_range("-rand(-127)", -ll_rand(-127), 0, 127); + ensure_in_range("-rand(-127)", -ll_rand(-127), 0, 127); } } } -- cgit v1.2.3 From 651353560bfe23b6423ecf7690d86645a71c0cbc Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 17 Oct 2023 14:56:10 -0400 Subject: SL-20476: Don't let the compiler know we intend to crash. clang has gotten smart enough to recognize an inline attempt to store to address zero. Fool it by storing to an address passed as a parameter, and pass nullptr from a different source file. --- indra/llcommon/llerror.cpp | 17 +++++++++++++++-- indra/llcommon/llerror.h | 10 +++++----- 2 files changed, 20 insertions(+), 7 deletions(-) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index 02cb186275..05e719b494 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -1617,5 +1617,18 @@ bool debugLoggingEnabled(const std::string& tag) return res; } - - +void crashdriver(void (*callback)(int*)) +{ + // The LLERROR_CRASH macro used to have inline code of the form: + //int* make_me_crash = NULL; + //*make_me_crash = 0; + + // But compilers are getting smart enough to recognize that, so we must + // assign to an address supplied by a separate source file. We could do + // the assignment here in crashdriver() -- but then BugSplat would group + // all LL_ERRS() crashes as the fault of this one function, instead of + // identifying the specific LL_ERRS() source line. So instead, do the + // assignment in a lambda in the caller's source. We just provide the + // nullptr target. + callback(nullptr); +} diff --git a/indra/llcommon/llerror.h b/indra/llcommon/llerror.h index 020f05e8f5..624a5fb37a 100644 --- a/indra/llcommon/llerror.h +++ b/indra/llcommon/llerror.h @@ -383,11 +383,9 @@ typedef LLError::NoClassInfo _LL_CLASS_TO_LOG; #define LL_NEWLINE '\n' // Use this only in LL_ERRS or in a place that LL_ERRS may not be used -#define LLERROR_CRASH \ -{ \ - int* make_me_crash = NULL;\ - *make_me_crash = 0; \ - exit(*make_me_crash); \ +#define LLERROR_CRASH \ +{ \ + crashdriver([](int* ptr){ *ptr = 0; exit(*ptr); }); \ } #define LL_ENDL \ @@ -466,5 +464,7 @@ typedef LLError::NoClassInfo _LL_CLASS_TO_LOG; // Check at run-time whether logging is enabled, without generating output bool debugLoggingEnabled(const std::string& tag); +// used by LLERROR_CRASH +void crashdriver(void (*)(int*)); #endif // LL_LLERROR_H -- cgit v1.2.3 From b9b38db5678556e1aa2710a86004dc5a14f97242 Mon Sep 17 00:00:00 2001 From: "Brad Kittenbrink (Brad Linden)" Date: Fri, 28 Jul 2023 16:52:06 -0700 Subject: Fix for SL-19968 objects missing timing bug due to stall during login ensure inventory skeleton loading doesn't block the message system from processing packets. --- indra/llcommon/llsdserialize.cpp | 6 ++++++ indra/llcommon/llsdserialize_xml.cpp | 2 ++ 2 files changed, 8 insertions(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 3db456ddb3..a475be6293 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -475,6 +475,7 @@ LLSDNotationParser::~LLSDNotationParser() // virtual S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD // map: { string:object, string:object } // array: [ object, object, object ] // undef: ! @@ -734,6 +735,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) c S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD // map: { string:object, string:object } map = LLSD::emptyMap(); S32 parse_count = 0; @@ -794,6 +796,7 @@ S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) c S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD // array: [ object, object, object ] array = LLSD::emptyArray(); S32 parse_count = 0; @@ -833,6 +836,7 @@ S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array, S32 max_dept bool LLSDNotationParser::parseString(std::istream& istr, LLSD& data) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD std::string value; auto count = deserialize_string(istr, value, mMaxBytesLeft); if(PARSE_FAILURE == count) return false; @@ -843,6 +847,7 @@ bool LLSDNotationParser::parseString(std::istream& istr, LLSD& data) const bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD // binary: b##"ff3120ab1" // or: b(len)"..." @@ -945,6 +950,7 @@ LLSDBinaryParser::~LLSDBinaryParser() // virtual S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD /** * Undefined: '!'
* Boolean: '1' for true '0' for false
diff --git a/indra/llcommon/llsdserialize_xml.cpp b/indra/llcommon/llsdserialize_xml.cpp index ac128c9f86..38b11eb32b 100644 --- a/indra/llcommon/llsdserialize_xml.cpp +++ b/indra/llcommon/llsdserialize_xml.cpp @@ -923,6 +923,8 @@ void LLSDXMLParser::parsePart(const char *buf, llssize len) // virtual S32 LLSDXMLParser::doParse(std::istream& input, LLSD& data, S32 max_depth) const { + LL_PROFILE_ZONE_SCOPED_CATEGORY_LLSD + #ifdef XML_PARSER_PERFORMANCE_TESTS XML_Timer timer( &parseTime ); #endif // XML_PARSER_PERFORMANCE_TESTS -- cgit v1.2.3 From dfb44aec706532cd26a1fcc6415cc7fdadec9907 Mon Sep 17 00:00:00 2001 From: Brad Linden Date: Thu, 2 Nov 2023 17:28:10 -0700 Subject: Fix build error from overly fancy tracy macro usage that nobody else is using for DRTVWR-559 --- indra/llcommon/llprofiler.h | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/llcommon') diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index 736a069f49..af5e5777bf 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -114,6 +114,7 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILER_SET_THREAD_NAME( name ) (void)(name) #define LL_RECORD_BLOCK_TIME(name) const LLTrace::BlockTimer& LL_GLUE_TOKENS(block_time_recorder, __LINE__)(LLTrace::timeThisBlock(name)); (void)LL_GLUE_TOKENS(block_time_recorder, __LINE__); #define LL_PROFILE_ZONE_NAMED(name) // LL_PROFILE_ZONE_NAMED is a no-op when Tracy is disabled + #define LL_PROFILE_ZONE_NAMED_COLOR(name,color) // LL_PROFILE_ZONE_NAMED_COLOR is a no-op when Tracy is disabled #define LL_PROFILE_ZONE_SCOPED // LL_PROFILE_ZONE_SCOPED is a no-op when Tracy is disabled #define LL_PROFILE_ZONE_COLOR(name,color) // LL_RECORD_BLOCK_TIME(name) -- cgit v1.2.3