SL-14399 Enqueue into 'LLViewerAssetStorage::assetRequestCoro' failed

6 files changed, 89 insertions, 21 deletions

diff --git a/indra/llmessage/llassetstorage.h b/indra/llmessage/llassetstorage.h
index c799d8eefc..7e4572f50f 100644
--- a/indra/llmessage/llassetstorage.h
+++ b/indra/llmessage/llassetstorage.h
@@ -262,7 +262,7 @@ public:
 
     virtual void logAssetStorageInfo() = 0;
     
-	void checkForTimeouts();
+	virtual void checkForTimeouts();
 
 	void getEstateAsset(const LLHost &object_sim, const LLUUID &agent_id, const LLUUID &session_id,
 									const LLUUID &asset_id, LLAssetType::EType atype, EstateAssetType etype,
diff --git a/indra/llmessage/llcoproceduremanager.cpp b/indra/llmessage/llcoproceduremanager.cpp
index a4fe3a2a8e..4d76dacdf7 100644
--- a/indra/llmessage/llcoproceduremanager.cpp
+++ b/indra/llmessage/llcoproceduremanager.cpp
@@ -47,7 +47,7 @@ static const std::map<std::string, U32> DefaultPoolSizes{
 };
 
 static const U32 DEFAULT_POOL_SIZE = 5;
-static const U32 DEFAULT_QUEUE_SIZE = 4096;
+const U32 LLCoprocedureManager::DEFAULT_QUEUE_SIZE = 4096;
 
 //=========================================================================
 class LLCoprocedurePool: private boost::noncopyable
@@ -194,7 +194,7 @@ void LLCoprocedureManager::setPropertyMethods(SettingQuery_t queryfn, SettingUpd
     mPropertyDefineFn = updatefn;
 
     // workaround until we get mutex into initializePool
-    initializePool("VAssetStorage");
+    initializePool("AssetStorage");
     initializePool("Upload");
 }
 
@@ -281,7 +281,7 @@ LLCoprocedurePool::LLCoprocedurePool(const std::string &poolName, size_t size):
     mPoolSize(size),
     mActiveCoprocsCount(0),
     mPending(0),
-    mPendingCoprocs(boost::make_shared<CoprocQueue_t>(DEFAULT_QUEUE_SIZE)),
+    mPendingCoprocs(boost::make_shared<CoprocQueue_t>(LLCoprocedureManager::DEFAULT_QUEUE_SIZE)),
     mHTTPPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID),
     mCoroMapping()
 {
@@ -332,7 +332,7 @@ LLCoprocedurePool::LLCoprocedurePool(const std::string &poolName, size_t size):
         mCoroMapping.insert(CoroAdapterMap_t::value_type(pooledCoro, httpAdapter));
     }
 
-    LL_INFOS("CoProcMgr") << "Created coprocedure pool named \"" << mPoolName << "\" with " << size << " items, queue max " << DEFAULT_QUEUE_SIZE << LL_ENDL;
+    LL_INFOS("CoProcMgr") << "Created coprocedure pool named \"" << mPoolName << "\" with " << size << " items, queue max " << LLCoprocedureManager::DEFAULT_QUEUE_SIZE << LL_ENDL;
 }
 
 LLCoprocedurePool::~LLCoprocedurePool() 
diff --git a/indra/llmessage/llcoproceduremanager.h b/indra/llmessage/llcoproceduremanager.h
index 70204ba02b..d5557c129f 100644
--- a/indra/llmessage/llcoproceduremanager.h
+++ b/indra/llmessage/llcoproceduremanager.h
@@ -91,6 +91,9 @@ private:
 
     SettingQuery_t mPropertyQueryFn;
     SettingUpdate_t mPropertyDefineFn;
+
+public:
+    static const U32 DEFAULT_QUEUE_SIZE;
 };
 
 #endif
diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index 48a89b40a5..bd1b25c863 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -15147,7 +15147,7 @@
         <key>Value</key>
             <real>1</real>
         </map>
-    <key>PoolSizeVAssetStorage</key>
+    <key>PoolSizeAssetStorage</key>
         <map>
         <key>Comment</key>
             <string>Coroutine Pool size for AssetStorage requests</string>
diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp
index 54f80a2995..7842d24279 100644
--- a/indra/newview/llviewerassetstorage.cpp
+++ b/indra/newview/llviewerassetstorage.cpp
@@ -49,8 +49,8 @@
 /// LLViewerAssetRequest
 ///----------------------------------------------------------------------------
 
- // There is also PoolSizeVAssetStorage value in setting that should mirror this name
-static const std::string VIEWER_ASSET_STORAGE_CORO_POOL = "VAssetStorage";
+ // There is also PoolSizeAssetStorage value in setting that should mirror this name
+static const std::string VIEWER_ASSET_STORAGE_CORO_POOL = "AssetStorage";
 
 /**
  * @brief Local class to encapsulate asset fetch requests with a timestamp.
@@ -137,6 +137,14 @@ LLViewerAssetStorage::~LLViewerAssetStorage()
         // This class has dedicated coroutine pool, clean it up, otherwise coroutines will crash later. 
         LLCoprocedureManager::instance().close(VIEWER_ASSET_STORAGE_CORO_POOL);
     }
+
+    while (mCoroWaitList.size() > 0)
+    {
+        CoroWaitList &request = mCoroWaitList.front();
+        // Clean up pending downloads, delete request and trigger callbacks
+        removeAndCallbackPendingDownloads(request.mId, request.mType, request.mId, request.mType, LL_ERR_NOERR, LLExtStat::NONE);
+        mCoroWaitList.pop_front();
+    }
 }
 
 // virtual 
@@ -350,6 +358,27 @@ void LLViewerAssetStorage::storeAssetData(
     }
 }
 
+void LLViewerAssetStorage::checkForTimeouts()
+{
+    LLAssetStorage::checkForTimeouts();
+
+    // Restore requests
+    LLCoprocedureManager* manager = LLCoprocedureManager::getInstance();
+    while (mCoroWaitList.size() > 0
+           && manager->count(VIEWER_ASSET_STORAGE_CORO_POOL) < LLCoprocedureManager::DEFAULT_QUEUE_SIZE)
+    {
+        CoroWaitList &request = mCoroWaitList.front();
+        
+        bool with_http = true;
+        bool is_temp = false;
+        LLViewerAssetStatsFF::record_enqueue(request.mType, with_http, is_temp);
+
+        manager->enqueueCoprocedure(VIEWER_ASSET_STORAGE_CORO_POOL, "LLViewerAssetStorage::assetRequestCoro",
+            boost::bind(&LLViewerAssetStorage::assetRequestCoro, this, request.mRequest, request.mId, request.mType, request.mCallback, request.mUserData));
+
+        mCoroWaitList.pop_front();
+    }
+}
 
 /**
  * @brief Allocate and queue an asset fetch request for the viewer
@@ -407,12 +436,20 @@ void LLViewerAssetStorage::queueRequestHttp(
     // This is the same as the current UDP logic - don't re-request a duplicate.
     if (!duplicate)
     {
-        bool with_http = true;
-        bool is_temp = false;
-        LLViewerAssetStatsFF::record_enqueue(atype, with_http, is_temp);
+        // Coroutine buffer has fixed size (synchronization buffer, so we have no alternatives), so buffer any request above limit
+        if (LLCoprocedureManager::instance().count(VIEWER_ASSET_STORAGE_CORO_POOL) < LLCoprocedureManager::DEFAULT_QUEUE_SIZE)
+        {
+            bool with_http = true;
+            bool is_temp = false;
+            LLViewerAssetStatsFF::record_enqueue(atype, with_http, is_temp);
 
-        LLCoprocedureManager::instance().enqueueCoprocedure(VIEWER_ASSET_STORAGE_CORO_POOL,"LLViewerAssetStorage::assetRequestCoro",
-            boost::bind(&LLViewerAssetStorage::assetRequestCoro, this, req, uuid, atype, callback, user_data));
+            LLCoprocedureManager::instance().enqueueCoprocedure(VIEWER_ASSET_STORAGE_CORO_POOL, "LLViewerAssetStorage::assetRequestCoro",
+                boost::bind(&LLViewerAssetStorage::assetRequestCoro, this, req, uuid, atype, callback, user_data));
+        }
+        else
+        {
+            mCoroWaitList.emplace_back(req, uuid, atype, callback, user_data);
+        }
     }
 }
 
diff --git a/indra/newview/llviewerassetstorage.h b/indra/newview/llviewerassetstorage.h
index ef01d179b7..5af16bd73e 100644
--- a/indra/newview/llviewerassetstorage.h
+++ b/indra/newview/llviewerassetstorage.h
@@ -45,7 +45,7 @@ public:
 
 	~LLViewerAssetStorage();
 
-	virtual void storeAssetData(
+	void storeAssetData(
 		const LLTransactionID& tid,
 		LLAssetType::EType atype,
 		LLStoreAssetCallback callback,
@@ -54,9 +54,9 @@ public:
 		bool is_priority = false,
 		bool store_local = false,
 		bool user_waiting=FALSE,
-		F64Seconds timeout=LL_ASSET_STORAGE_TIMEOUT);
-	
-	virtual void storeAssetData(
+		F64Seconds timeout=LL_ASSET_STORAGE_TIMEOUT) override;
+
+	void storeAssetData(
 		const std::string& filename,
 		const LLTransactionID& tid,
 		LLAssetType::EType type,
@@ -65,16 +65,17 @@ public:
 		bool temp_file = false,
 		bool is_priority = false,
 		bool user_waiting=FALSE,
-		F64Seconds timeout=LL_ASSET_STORAGE_TIMEOUT);
+		F64Seconds timeout=LL_ASSET_STORAGE_TIMEOUT) override;
+
+    void checkForTimeouts() override;
 
 protected:
-	// virtual
 	void _queueDataRequest(const LLUUID& uuid,
 						   LLAssetType::EType type,
                            LLGetAssetCallback callback,
 						   void *user_data,
 						   BOOL duplicate,
-						   BOOL is_priority);
+						   BOOL is_priority) override;
 
     void queueRequestHttp(const LLUUID& uuid,
                           LLAssetType::EType type,
@@ -94,7 +95,34 @@ protected:
     std::string getAssetURL(const std::string& cap_url, const LLUUID& uuid, LLAssetType::EType atype);
 
     void logAssetStorageInfo();
-    
+
+    // Asset storage works through coroutines and coroutines have limited queue capacity
+    // This class is meant to temporary store requests when fiber's queue is full
+    class CoroWaitList
+    {
+    public:
+        CoroWaitList(LLViewerAssetRequest *req,
+            const LLUUID& uuid,
+            LLAssetType::EType atype,
+            LLGetAssetCallback &callback,
+            void *user_data)
+          : mRequest(req),
+            mId(uuid),
+            mType(atype),
+            mCallback(callback),
+            mUserData(user_data)
+        {
+        }
+
+        LLViewerAssetRequest* mRequest;
+        LLUUID mId;
+        LLAssetType::EType mType;
+        LLGetAssetCallback mCallback;
+        void *mUserData;
+    };
+    typedef std::list<CoroWaitList> wait_list_t;
+    wait_list_t mCoroWaitList;
+
     std::string mViewerAssetUrl;
     S32 mAssetCoroCount;
     S32 mCountRequests;