MAINT-5575: Convert the Experience cache into a coro based singleton.

--HG--
branch : MAINT-5575

5 files changed, 849 insertions, 333 deletions

diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt
index 9739f7c607..12fc1bbcfc 100755
--- a/indra/llmessage/CMakeLists.txt
+++ b/indra/llmessage/CMakeLists.txt
@@ -40,6 +40,7 @@ set(llmessage_SOURCE_FILES
     llchainio.cpp
     llcircuit.cpp
     llclassifiedflags.cpp
+    llcoproceduremanager.cpp
     llcorehttputil.cpp
     llcurl.cpp
     lldatapacker.cpp
@@ -128,6 +129,7 @@ set(llmessage_HEADER_FILES
     llcipher.h
     llcircuit.h
     llclassifiedflags.h
+    llcoproceduremanager.h
     llcorehttputil.h
     llcurl.h
     lldatapacker.h
diff --git a/indra/llmessage/llcoproceduremanager.cpp b/indra/llmessage/llcoproceduremanager.cpp
new file mode 100644
index 0000000000..062f2e6e42
--- /dev/null
+++ b/indra/llmessage/llcoproceduremanager.cpp
@@ -0,0 +1,414 @@
+/**
+* @file LLCoprocedurePool.cpp
+* @author Rider Linden
+* @brief Singleton class for managing asset uploads to the sim.
+*
+* $LicenseInfo:firstyear=2015&license=viewerlgpl$
+* Second Life Viewer Source Code
+* Copyright (C) 2015, Linden Research, Inc.
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation;
+* version 2.1 of the License only.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*
+* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+* $/LicenseInfo$
+*/
+
+#include "linden_common.h" 
+#include "llcoproceduremanager.h"
+
+//=========================================================================
+// Map of pool sizes for known pools
+static std::map<std::string, U32> DefaultPoolSizes;
+
+// *TODO$: When C++11 this can be initialized here as follows:
+// = {{"AIS", 25}, {"Upload", 1}}
+
+#define DEFAULT_POOL_SIZE 5
+
+//=========================================================================
+class LLCoprocedurePool: private boost::noncopyable
+{
+public:
+    typedef LLCoprocedureManager::CoProcedure_t CoProcedure_t;
+
+    LLCoprocedurePool(const std::string &name, size_t size);
+    virtual ~LLCoprocedurePool();
+
+    /// Places the coprocedure on the queue for processing. 
+    /// 
+    /// @param name Is used for debugging and should identify this coroutine.
+    /// @param proc Is a bound function to be executed 
+    /// 
+    /// @return This method returns a UUID that can be used later to cancel execution.
+    LLUUID enqueueCoprocedure(const std::string &name, CoProcedure_t proc);
+
+    /// Cancel a coprocedure. If the coprocedure is already being actively executed 
+    /// this method calls cancelYieldingOperation() on the associated HttpAdapter
+    /// If it has not yet been dequeued it is simply removed from the queue.
+    bool cancelCoprocedure(const LLUUID &id);
+
+    /// Requests a shutdown of the upload manager. Passing 'true' will perform 
+    /// an immediate kill on the upload coroutine.
+    void shutdown(bool hardShutdown = false);
+
+    /// Returns the number of coprocedures in the queue awaiting processing.
+    ///
+    inline size_t countPending() const
+    {
+        return mPendingCoprocs.size();
+    }
+
+    /// Returns the number of coprocedures actively being processed.
+    ///
+    inline size_t countActive() const
+    {
+        return mActiveCoprocs.size();
+    }
+
+    /// Returns the total number of coprocedures either queued or in active processing.
+    ///
+    inline size_t count() const
+    {
+        return countPending() + countActive();
+    }
+
+private:
+    struct QueuedCoproc
+    {
+        typedef boost::shared_ptr<QueuedCoproc> ptr_t;
+
+        QueuedCoproc(const std::string &name, const LLUUID &id, CoProcedure_t proc) :
+            mName(name),
+            mId(id),
+            mProc(proc)
+        {}
+
+        std::string mName;
+        LLUUID mId;
+        CoProcedure_t mProc;
+    };
+
+    // we use a deque here rather than std::queue since we want to be able to 
+    // iterate through the queue and potentially erase an entry from the middle.
+    typedef std::deque<QueuedCoproc::ptr_t>  CoprocQueue_t;
+    typedef std::map<LLUUID, LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t> ActiveCoproc_t;
+
+    std::string     mPoolName;
+    size_t          mPoolSize;
+    CoprocQueue_t   mPendingCoprocs;
+    ActiveCoproc_t  mActiveCoprocs;
+    bool            mShutdown;
+    LLEventStream   mWakeupTrigger;
+
+    typedef std::map<std::string, LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t> CoroAdapterMap_t;
+    LLCore::HttpRequest::policy_t mHTTPPolicy;
+
+    CoroAdapterMap_t mCoroMapping;
+
+    void coprocedureInvokerCoro(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter);
+
+};
+
+//=========================================================================
+LLCoprocedureManager::LLCoprocedureManager()
+{
+    DefaultPoolSizes.insert(std::map<std::string, U32>::value_type("Upload", 1));
+    DefaultPoolSizes.insert(std::map<std::string, U32>::value_type("AIS", 25));
+}
+
+LLCoprocedureManager::~LLCoprocedureManager()
+{
+
+}
+
+LLCoprocedureManager::poolPtr_t LLCoprocedureManager::initializePool(const std::string &poolName)
+{
+    // Attempt to look up a pool size in the configuration.  If found use that
+    std::string keyName = "PoolSize" + poolName;
+    int size = 0;
+
+    if (mPropertyQueryFn && !mPropertyQueryFn.empty())
+    {
+        size = mPropertyQueryFn(keyName);
+    }
+
+    if (size == 0)
+    {   // if not found grab the know default... if there is no known 
+        // default use a reasonable number like 5.
+        std::map<std::string, U32>::iterator it = DefaultPoolSizes.find(poolName);
+        if (it == DefaultPoolSizes.end())
+            size = DEFAULT_POOL_SIZE;
+        else
+            size = (*it).second;
+
+        if (mPropertyDefineFn && !mPropertyDefineFn.empty())
+            mPropertyDefineFn(keyName, size, "Coroutine Pool size for " + poolName);
+        LL_WARNS() << "LLCoprocedureManager: No setting for \"" << keyName << "\" setting pool size to default of " << size << LL_ENDL;
+    }
+
+    poolPtr_t pool = poolPtr_t(new LLCoprocedurePool(poolName, size));
+    mPoolMap.insert(poolMap_t::value_type(poolName, pool));
+
+    return pool;
+}
+
+//-------------------------------------------------------------------------
+LLUUID LLCoprocedureManager::enqueueCoprocedure(const std::string &pool, const std::string &name, CoProcedure_t proc)
+{
+    // Attempt to find the pool and enqueue the procedure.  If the pool does 
+    // not exist, create it.
+    poolPtr_t targetPool;
+    poolMap_t::iterator it = mPoolMap.find(pool);
+
+    if (it == mPoolMap.end())
+    {
+        targetPool = initializePool(pool);
+    }
+    else
+    {
+        targetPool = (*it).second;
+    }
+
+    if (!targetPool)
+    {
+        LL_WARNS() << "LLCoprocedureManager unable to create coprocedure pool named \"" << pool << "\"" << LL_ENDL;
+        return LLUUID::null;
+    }
+
+    return targetPool->enqueueCoprocedure(name, proc);
+}
+
+void LLCoprocedureManager::cancelCoprocedure(const LLUUID &id)
+{
+    for (poolMap_t::const_iterator it = mPoolMap.begin(); it != mPoolMap.end(); ++it)
+    {
+        if ((*it).second->cancelCoprocedure(id))
+            return;
+    }
+    LL_INFOS() << "Coprocedure not found." << LL_ENDL;
+}
+
+void LLCoprocedureManager::shutdown(bool hardShutdown)
+{
+    for (poolMap_t::const_iterator it = mPoolMap.begin(); it != mPoolMap.end(); ++it)
+    {
+        (*it).second->shutdown(hardShutdown);
+    }
+    mPoolMap.clear();
+}
+
+void LLCoprocedureManager::setPropertyMethods(SettingQuery_t queryfn, SettingUpdate_t updatefn)
+{
+    mPropertyQueryFn = queryfn;
+    mPropertyDefineFn = updatefn;
+}
+
+//-------------------------------------------------------------------------
+size_t LLCoprocedureManager::countPending() const
+{
+    size_t count = 0;
+    for (poolMap_t::const_iterator it = mPoolMap.begin(); it != mPoolMap.end(); ++it)
+    {
+        count += (*it).second->countPending();
+    }
+    return count;
+}
+
+size_t LLCoprocedureManager::countPending(const std::string &pool) const
+{
+    poolMap_t::const_iterator it = mPoolMap.find(pool);
+
+    if (it == mPoolMap.end())
+        return 0;
+    return (*it).second->countPending();
+}
+
+size_t LLCoprocedureManager::countActive() const
+{
+    size_t count = 0;
+    for (poolMap_t::const_iterator it = mPoolMap.begin(); it != mPoolMap.end(); ++it)
+    {
+        count += (*it).second->countActive();
+    }
+    return count;
+}
+
+size_t LLCoprocedureManager::countActive(const std::string &pool) const
+{
+    poolMap_t::const_iterator it = mPoolMap.find(pool);
+
+    if (it == mPoolMap.end())
+        return 0;
+    return (*it).second->countActive();
+}
+
+size_t LLCoprocedureManager::count() const
+{
+    size_t count = 0;
+    for (poolMap_t::const_iterator it = mPoolMap.begin(); it != mPoolMap.end(); ++it)
+    {
+        count += (*it).second->count();
+    }
+    return count;
+}
+
+size_t LLCoprocedureManager::count(const std::string &pool) const
+{
+    poolMap_t::const_iterator it = mPoolMap.find(pool);
+
+    if (it == mPoolMap.end())
+        return 0;
+    return (*it).second->count();
+}
+
+//=========================================================================
+LLCoprocedurePool::LLCoprocedurePool(const std::string &poolName, size_t size):
+    mPoolName(poolName),
+    mPoolSize(size),
+    mPendingCoprocs(),
+    mShutdown(false),
+    mWakeupTrigger("CoprocedurePool" + poolName, true),
+    mCoroMapping(),
+    mHTTPPolicy(LLCore::HttpRequest::DEFAULT_POLICY_ID)
+{
+    for (size_t count = 0; count < mPoolSize; ++count)
+    {
+        LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter =
+            LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t(
+            new LLCoreHttpUtil::HttpCoroutineAdapter( mPoolName + "Adapter", mHTTPPolicy));
+
+        std::string uploadCoro = LLCoros::instance().launch("LLCoprocedurePool("+mPoolName+")::coprocedureInvokerCoro",
+            boost::bind(&LLCoprocedurePool::coprocedureInvokerCoro, this, httpAdapter));
+
+        mCoroMapping.insert(CoroAdapterMap_t::value_type(uploadCoro, httpAdapter));
+    }
+
+    LL_INFOS() << "Created coprocedure pool named \"" << mPoolName << "\" with " << size << " items." << LL_ENDL;
+
+    mWakeupTrigger.post(LLSD());
+}
+
+LLCoprocedurePool::~LLCoprocedurePool() 
+{
+    shutdown();
+}
+
+//-------------------------------------------------------------------------
+void LLCoprocedurePool::shutdown(bool hardShutdown)
+{
+    CoroAdapterMap_t::iterator it;
+
+    for (it = mCoroMapping.begin(); it != mCoroMapping.end(); ++it)
+    {
+        if (!(*it).first.empty())
+        {
+            if (hardShutdown)
+            {
+                LLCoros::instance().kill((*it).first);
+            }
+        }
+        if ((*it).second)
+        {
+            (*it).second->cancelYieldingOperation();
+        }
+    }
+
+    mShutdown = true;
+    mCoroMapping.clear();
+    mPendingCoprocs.clear();
+}
+
+//-------------------------------------------------------------------------
+LLUUID LLCoprocedurePool::enqueueCoprocedure(const std::string &name, LLCoprocedurePool::CoProcedure_t proc)
+{
+    LLUUID id(LLUUID::generateNewID());
+
+    mPendingCoprocs.push_back(QueuedCoproc::ptr_t(new QueuedCoproc(name, id, proc)));
+    LL_INFOS() << "Coprocedure(" << name << ") enqueued with id=" << id.asString() << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+
+    mWakeupTrigger.post(LLSD());
+
+    return id;
+}
+
+bool LLCoprocedurePool::cancelCoprocedure(const LLUUID &id)
+{
+    // first check the active coroutines.  If there, remove it and return.
+    ActiveCoproc_t::iterator itActive = mActiveCoprocs.find(id);
+    if (itActive != mActiveCoprocs.end())
+    {
+        LL_INFOS() << "Found and canceling active coprocedure with id=" << id.asString() << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+        (*itActive).second->cancelYieldingOperation();
+        mActiveCoprocs.erase(itActive);
+        return true;
+    }
+
+    for (CoprocQueue_t::iterator it = mPendingCoprocs.begin(); it != mPendingCoprocs.end(); ++it)
+    {
+        if ((*it)->mId == id)
+        {
+            LL_INFOS() << "Found and removing queued coroutine(" << (*it)->mName << ") with Id=" << id.asString() << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+            mPendingCoprocs.erase(it);
+            return true;
+        }
+    }
+
+    LL_INFOS() << "Coprocedure with Id=" << id.asString() << " was not found." << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+    return false;
+}
+
+//-------------------------------------------------------------------------
+void LLCoprocedurePool::coprocedureInvokerCoro(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t httpAdapter)
+{
+    LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest);
+
+    while (!mShutdown)
+    {
+        llcoro::waitForEventOn(mWakeupTrigger);
+        if (mShutdown)
+            break;
+        
+        while (!mPendingCoprocs.empty())
+        {
+            QueuedCoproc::ptr_t coproc = mPendingCoprocs.front();
+            mPendingCoprocs.pop_front();
+            mActiveCoprocs.insert(ActiveCoproc_t::value_type(coproc->mId, httpAdapter));
+
+            LL_INFOS() << "Dequeued and invoking coprocedure(" << coproc->mName << ") with id=" << coproc->mId.asString() << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+
+            try
+            {
+                coproc->mProc(httpAdapter, coproc->mId);
+            }
+            catch (std::exception &e)
+            {
+                LL_WARNS() << "Coprocedure(" << coproc->mName << ") id=" << coproc->mId.asString() <<
+                    " threw an exception! Message=\"" << e.what() << "\"" << LL_ENDL;
+            }
+            catch (...)
+            {
+                LL_WARNS() << "A non std::exception was thrown from " << coproc->mName << " with id=" << coproc->mId << "." << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+            }
+
+            LL_INFOS() << "Finished coprocedure(" << coproc->mName << ")" << " in pool \"" << mPoolName << "\"" << LL_ENDL;
+
+            ActiveCoproc_t::iterator itActive = mActiveCoprocs.find(coproc->mId);
+            if (itActive != mActiveCoprocs.end())
+            {
+                mActiveCoprocs.erase(itActive);
+            }
+        }
+    }
+}
diff --git a/indra/llmessage/llcoproceduremanager.h b/indra/llmessage/llcoproceduremanager.h
new file mode 100644
index 0000000000..497367b80c
--- /dev/null
+++ b/indra/llmessage/llcoproceduremanager.h
@@ -0,0 +1,98 @@
+/**
+* @file llcoproceduremanager.h
+* @author Rider Linden
+* @brief Singleton class for managing asset uploads to the sim.
+*
+* $LicenseInfo:firstyear=2015&license=viewerlgpl$
+* Second Life Viewer Source Code
+* Copyright (C) 2015, Linden Research, Inc.
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation;
+* version 2.1 of the License only.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*
+* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
+* $/LicenseInfo$
+*/
+
+#ifndef LL_COPROCEDURE_MANAGER_H
+#define LL_COPROCEDURE_MANAGER_H
+
+#include "lleventcoro.h"
+#include "llcoros.h"
+#include "llcorehttputil.h"
+#include "lluuid.h"
+
+class LLCoprocedurePool;
+
+class LLCoprocedureManager : public LLSingleton < LLCoprocedureManager >
+{
+    friend class LLSingleton < LLCoprocedureManager > ;
+
+public:
+    typedef boost::function<U32(const std::string &)> SettingQuery_t;
+    typedef boost::function<void(const std::string &, U32, const std::string &)> SettingUpdate_t;
+
+    typedef boost::function<void(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t &, const LLUUID &id)> CoProcedure_t;
+
+    LLCoprocedureManager();
+    virtual ~LLCoprocedureManager();
+
+    /// Places the coprocedure on the queue for processing. 
+    /// 
+    /// @param name Is used for debugging and should identify this coroutine.
+    /// @param proc Is a bound function to be executed 
+    /// 
+    /// @return This method returns a UUID that can be used later to cancel execution.
+    LLUUID enqueueCoprocedure(const std::string &pool, const std::string &name, CoProcedure_t proc);
+
+    /// Cancel a coprocedure. If the coprocedure is already being actively executed 
+    /// this method calls cancelYieldingOperation() on the associated HttpAdapter
+    /// If it has not yet been dequeued it is simply removed from the queue.
+    void cancelCoprocedure(const LLUUID &id);
+
+    /// Requests a shutdown of the upload manager. Passing 'true' will perform 
+    /// an immediate kill on the upload coroutine.
+    void shutdown(bool hardShutdown = false);
+
+    void setPropertyMethods(SettingQuery_t queryfn, SettingUpdate_t updatefn);
+
+    /// Returns the number of coprocedures in the queue awaiting processing.
+    ///
+    size_t countPending() const;
+    size_t countPending(const std::string &pool) const;
+
+    /// Returns the number of coprocedures actively being processed.
+    ///
+    size_t countActive() const;
+    size_t countActive(const std::string &pool) const;
+
+    /// Returns the total number of coprocedures either queued or in active processing.
+    ///
+    size_t count() const;
+    size_t count(const std::string &pool) const;
+
+private:
+
+    typedef boost::shared_ptr<LLCoprocedurePool> poolPtr_t;
+    typedef std::map<std::string, poolPtr_t> poolMap_t;
+
+    poolMap_t mPoolMap;
+
+    poolPtr_t initializePool(const std::string &poolName);
+
+    SettingQuery_t mPropertyQueryFn;
+    SettingUpdate_t mPropertyDefineFn;
+};
+
+#endif
diff --git a/indra/llmessage/llexperiencecache.cpp b/indra/llmessage/llexperiencecache.cpp
index 34c4210359..36a4fc8823 100644
--- a/indra/llmessage/llexperiencecache.cpp
+++ b/indra/llmessage/llexperiencecache.cpp
@@ -26,53 +26,51 @@
 #include "llexperiencecache.h"
 
 #include "llavatarname.h"
-#include "llframetimer.h"
 #include "llhttpclient.h"
 #include "llsdserialize.h"
+#include "llcoros.h"
+#include "lleventcoro.h"
+#include "lleventfilter.h"
+#include "llcoproceduremanager.h"
+#include "lldir.h"
 #include <set>
 #include <map>
-#include "boost/tokenizer.hpp"
+#include <boost/tokenizer.hpp>
 #include <boost/concept_check.hpp>
 
-
-typedef std::map<LLUUID, LLUUID> KeyMap;
-KeyMap privateToPublicKeyMap;
-
-
-std::string sLookupURL;
-
-typedef std::map<LLUUID, F64> pending_queue_t;
-pending_queue_t sPendingQueue;
-
-int sMaximumLookups = 10;
-
-LLFrameTimer sRequestTimer;
-
-// Periodically clean out expired entries from the cache
-LLFrameTimer sEraseExpiredTimer;
-
-
 //=========================================================================
 namespace LLExperienceCacheImpl
 {
-	bool max_age_from_cache_control(const std::string& cache_control, S32 *max_age);
 	void mapKeys(const LLSD& legacyKeys);
+    F64 getErrorRetryDeltaTime(S32 status, LLSD headers);
+    bool maxAgeFromCacheControl(const std::string& cache_control, S32 *max_age);
+
+    static const std::string PRIVATE_KEY    = "private_id";
+    static const std::string EXPERIENCE_ID  = "public_id";
+
+    static const std::string MAX_AGE("max-age");
+    static const boost::char_separator<char> EQUALS_SEPARATOR("=");
+    static const boost::char_separator<char> COMMA_SEPARATOR(",");
+
+    // *TODO$: this seems to be tied to mapKeys which is used by bootstrap.... but I don't think that bootstrap is used.
+    typedef std::map<LLUUID, LLUUID> KeyMap;
+    KeyMap privateToPublicKeyMap;
 }
 
 //=========================================================================
 const std::string LLExperienceCache::PRIVATE_KEY	= "private_id";
 const std::string LLExperienceCache::MISSING       	= "DoesNotExist";
 
-const std::string LLExperienceCache::AGENT_ID      	= "agent_id";
-const std::string LLExperienceCache::GROUP_ID      	= "group_id";
+const std::string LLExperienceCache::AGENT_ID      = "agent_id";
+const std::string LLExperienceCache::GROUP_ID      = "group_id";
 const std::string LLExperienceCache::EXPERIENCE_ID	= "public_id";
 const std::string LLExperienceCache::NAME			= "name";
 const std::string LLExperienceCache::PROPERTIES		= "properties";
 const std::string LLExperienceCache::EXPIRES		= "expiration";  
 const std::string LLExperienceCache::DESCRIPTION	= "description";
 const std::string LLExperienceCache::QUOTA         	= "quota";
-const std::string LLExperienceCache::MATURITY      	= "maturity";
-const std::string LLExperienceCache::METADATA      	= "extended_metadata";
+const std::string LLExperienceCache::MATURITY      = "maturity";
+const std::string LLExperienceCache::METADATA      = "extended_metadata";
 const std::string LLExperienceCache::SLURL         	= "slurl";
 
 // should be in sync with experience-api/experiences/models.py
@@ -80,20 +78,51 @@ const int LLExperienceCache::PROPERTY_INVALID		= 1 << 0;
 const int LLExperienceCache::PROPERTY_PRIVILEGED	= 1 << 3;
 const int LLExperienceCache::PROPERTY_GRID			= 1 << 4;
 const int LLExperienceCache::PROPERTY_PRIVATE		= 1 << 5;
-const int LLExperienceCache::PROPERTY_DISABLED		= 1 << 6;  
-const int LLExperienceCache::PROPERTY_SUSPENDED		= 1 << 7;
+const int LLExperienceCache::PROPERTY_DISABLED	= 1 << 6;  
+const int LLExperienceCache::PROPERTY_SUSPENDED	= 1 << 7;
 
 // default values
-const F64 LLExperienceCache::DEFAULT_EXPIRATION		= 600.0;
+const F64 LLExperienceCache::DEFAULT_EXPIRATION	= 600.0;
 const S32 LLExperienceCache::DEFAULT_QUOTA			= 128; // this is megabytes
 
 //=========================================================================
-LLExperienceCache::LLExperienceCache()
+LLExperienceCache::LLExperienceCache():
+    mShutdown(false)
 {
 }
 
 LLExperienceCache::~LLExperienceCache()
 {
+
+}
+
+void LLExperienceCache::initSingleton()
+{
+    mCacheFileName = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, "experience_cache.xml");
+
+    LL_INFOS("ExperienceCache") << "Loading " << mCacheFileName << LL_ENDL;
+    llifstream cache_stream(mCacheFileName.c_str());
+
+    if (cache_stream.is_open())
+    {
+        cache_stream >> (*this);
+    }
+
+    LLCoros::instance().launch("LLExperienceCache::idleCoro",
+        boost::bind(&LLExperienceCache::idleCoro, this));
+
+}
+
+void LLExperienceCache::cleanup()
+{
+    LL_INFOS("ExperienceCache") << "Saving " << mCacheFileName << LL_ENDL;
+
+    llofstream cache_stream(mCacheFileName.c_str());
+    if (cache_stream.is_open())
+    {
+        cache_stream << (*this);
+    }
+    mShutdown = true;
 }
 
 //-------------------------------------------------------------------------
@@ -110,18 +139,18 @@ void LLExperienceCache::importFile(std::istream& istr)
     for (; it != experiences.endMap(); ++it)
     {
         public_key.set(it->first);
-        sCache[public_key] = it->second;
+        mCache[public_key] = it->second;
     }
 
-    LL_DEBUGS("ExperienceCache") << "importFile() loaded " << sCache.size() << LL_ENDL;
+    LL_DEBUGS("ExperienceCache") << "importFile() loaded " << mCache.size() << LL_ENDL;
 }
 
 void LLExperienceCache::exportFile(std::ostream& ostr) const
 {
     LLSD experiences;
 
-    cache_t::const_iterator it = sCache.begin();
-    for (; it != sCache.end(); ++it)
+    cache_t::const_iterator it = mCache.begin();
+    for (; it != mCache.end(); ++it)
     {
         if (!it->second.has(EXPERIENCE_ID) || it->second[EXPERIENCE_ID].asUUID().isNull() ||
             it->second.has("DoesNotExist") || (it->second.has(PROPERTIES) && it->second[PROPERTIES].asInteger() & PROPERTY_INVALID))
@@ -136,7 +165,7 @@ void LLExperienceCache::exportFile(std::ostream& ostr) const
     LLSDSerialize::toPrettyXML(data, ostr);
 }
 
-// *TODO$: Rider: These three functions not seem to be used... it may be useful in testing.
+// *TODO$: Rider: This method does not seem to be used... it may be useful in testing.
 void LLExperienceCache::bootstrap(const LLSD& legacyKeys, int initialExpiration)
 {
 	LLExperienceCacheImpl::mapKeys(legacyKeys);
@@ -166,8 +195,8 @@ LLUUID LLExperienceCache::getExperienceId(const LLUUID& private_key, bool null_i
     if (private_key.isNull())
         return LLUUID::null;
 
-    KeyMap::const_iterator it = privateToPublicKeyMap.find(private_key);
-    if (it == privateToPublicKeyMap.end())
+    LLExperienceCacheImpl::KeyMap::const_iterator it = LLExperienceCacheImpl::privateToPublicKeyMap.find(private_key);
+    if (it == LLExperienceCacheImpl::privateToPublicKeyMap.end())
     {
         if (null_if_not_found)
         {
@@ -182,8 +211,10 @@ LLUUID LLExperienceCache::getExperienceId(const LLUUID& private_key, bool null_i
 //=========================================================================
 void LLExperienceCache::processExperience(const LLUUID& public_key, const LLSD& experience)
 {
-	sCache[public_key]=experience;
-	LLSD & row = sCache[public_key];
+    LL_INFOS("ExperienceCache") << "Processing experience \"" << experience[NAME] << "\" with key " << public_key.asString() << LL_ENDL;
+
+	mCache[public_key]=experience;
+	LLSD & row = mCache[public_key];
 
 	if(row.has(EXPIRES))
 	{
@@ -192,233 +223,148 @@ void LLExperienceCache::processExperience(const LLUUID& public_key, const LLSD&
 
 	if(row.has(EXPERIENCE_ID))
 	{
-		sPendingQueue.erase(row[EXPERIENCE_ID].asUUID());
+		mPendingQueue.erase(row[EXPERIENCE_ID].asUUID());
 	}
 
 	//signal
-	signal_map_t::iterator sig_it =	sSignalMap.find(public_key);
-	if (sig_it != sSignalMap.end())
+	signal_map_t::iterator sig_it =	mSignalMap.find(public_key);
+	if (sig_it != mSignalMap.end())
 	{
 		signal_ptr signal = sig_it->second;
 		(*signal)(experience);
 
-		sSignalMap.erase(public_key);
+		mSignalMap.erase(public_key);
 	}
 }
 
 const LLExperienceCache::cache_t& LLExperienceCache::getCached()
 {
-	return sCache;
-}
-
-void LLExperienceCache::setMaximumLookups(int maximumLookups)
-{
-	sMaximumLookups = maximumLookups;
+	return mCache;
 }
 
-
-bool LLExperienceCache::expirationFromCacheControl(LLSD headers, F64 *expires)
+void LLExperienceCache::requestExperiencesCoro(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t &httpAdapter, std::string url, RequestQueue_t requests)
 {
-	// Allow the header to override the default
-	LLSD cache_control_header = headers["cache-control"];
-	if (cache_control_header.isDefined())
-	{
-		S32 max_age = 0;
-		std::string cache_control = cache_control_header.asString();
-		if (max_age_from_cache_control(cache_control, &max_age))
-		{
-			LL_WARNS("ExperienceCache") 
-				<< "got EXPIRES from headers, max_age " << max_age 
-				<< LL_ENDL;
-			F64 now = LLFrameTimer::getTotalSeconds();
-			*expires = now + (F64)max_age;
-			return true;
-		}
-	}
-	return false;
-}
+    LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest());
 
+    //LL_INFOS("requestExperiencesCoro") << "url: " << url << LL_ENDL;
 
-static const std::string MAX_AGE("max-age");
-static const boost::char_separator<char> EQUALS_SEPARATOR("=");
-static const boost::char_separator<char> COMMA_SEPARATOR(",");
+    LLSD result = httpAdapter->getAndYield(httpRequest, url);
+        
+    LLSD httpResults = result[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS];
+    LLCore::HttpStatus status = LLCoreHttpUtil::HttpCoroutineAdapter::getStatusFromLLSD(httpResults);
 
+    if (!status)
+    {
+        F64 now = LLFrameTimer::getTotalSeconds();
 
-class LLExperienceResponder : public LLHTTPClient::Responder
-{
-public:
-	LLExperienceResponder(const ask_queue_t& keys)
-		:mKeys(keys)
-	{
-
-	}
-
-	/*virtual*/ void httpCompleted()
-	{
-		LLSD experiences = getContent()["experience_keys"];
-		LLSD::array_const_iterator it = experiences.beginArray();
-		for( /**/ ; it != experiences.endArray(); ++it)
-		{
-			const LLSD& row = *it;
-			LLUUID public_key = row[EXPERIENCE_ID].asUUID();
-
-
-			LL_DEBUGS("ExperienceCache") << "Received result for " << public_key 
-				<< " display '" << row[LLExperienceCache::NAME].asString() << "'" << LL_ENDL ;
-
-			processExperience(public_key, row);
-		}
-
-		LLSD error_ids = getContent()["error_ids"];
-		LLSD::array_const_iterator errIt = error_ids.beginArray();
-		for( /**/ ; errIt != error_ids.endArray() ; ++errIt )
-		{
-			LLUUID id = errIt->asUUID();		
-			LLSD exp;
-			exp[EXPIRES]=DEFAULT_EXPIRATION;
-			exp[EXPERIENCE_ID] = id;
-			exp[PROPERTIES]=PROPERTY_INVALID;
-			exp[MISSING]=true;
-            exp[QUOTA] = DEFAULT_QUOTA;
-
-			processExperience(id, exp);
-			LL_WARNS("ExperienceCache") << "LLExperienceResponder::result() error result for " << id << LL_ENDL ;
-		}
-
-		LL_DEBUGS("ExperienceCache") << sCache.size() << " cached experiences" << LL_ENDL;
-	}
-
-	/*virtual*/ void httpFailure()
-	{
- 		LL_WARNS("ExperienceCache") << "Request failed "<<getStatus()<<" "<<getReason()<< LL_ENDL;
- 		// We're going to construct a dummy record and cache it for a while,
- 		// either briefly for a 503 Service Unavailable, or longer for other
- 		// errors.
- 		F64 retry_timestamp = errorRetryTimestamp(getStatus());
- 
- 
- 		// Add dummy records for all agent IDs in this request
- 		ask_queue_t::const_iterator it = mKeys.begin();
- 		for ( ; it != mKeys.end(); ++it)
-		{
-
-			LLSD exp = get(it->first);
+        LLSD headers = httpResults[LLCoreHttpUtil::HttpCoroutineAdapter::HTTP_RESULTS_HEADERS];
+        // build dummy entries for the failed requests
+        for (RequestQueue_t::const_iterator it = requests.begin(); it != requests.end(); ++it)
+        {
+            LLSD exp = get(*it);
             //leave the properties alone if we already have a cache entry for this xp
-            if(exp.isUndefined())
+            if (exp.isUndefined())
             {
-                exp[PROPERTIES]=PROPERTY_INVALID;
+                exp[PROPERTIES] = PROPERTY_INVALID;
             }
-			exp[EXPIRES]=retry_timestamp;
-			exp[EXPERIENCE_ID] = it->first;
-			exp["key_type"] = it->second;
-			exp["uuid"] = it->first;
-			exp["error"] = (LLSD::Integer)getStatus();
+            exp[EXPIRES] = now + LLExperienceCacheImpl::getErrorRetryDeltaTime(status, headers);
+            exp[EXPERIENCE_ID] = *it;
+            exp["key_type"] = EXPERIENCE_ID;
+            exp["uuid"] = *it;
+            exp["error"] = (LLSD::Integer)status.getType();
             exp[QUOTA] = DEFAULT_QUOTA;
 
- 			LLExperienceCache::processExperience(it->first, exp);
- 		}
-
-	}
-
-	// Return time to retry a request that generated an error, based on
-	// error type and headers.  Return value is seconds-since-epoch.
-	F64 errorRetryTimestamp(S32 status)
-	{
+            processExperience(*it, exp);
+        }
+        return;
+    }
 
-		// Retry-After takes priority
-		LLSD retry_after = getResponseHeaders()["retry-after"];
-		if (retry_after.isDefined())
-		{
-			// We only support the delta-seconds type
-			S32 delta_seconds = retry_after.asInteger();
-			if (delta_seconds > 0)
-			{
-				// ...valid delta-seconds
-				return F64(delta_seconds);
-			}
-		}
+    LLSD experiences = result["experience_keys"];
+    
+    for (LLSD::array_const_iterator it = experiences.beginArray(); 
+        it != experiences.endArray(); ++it)
+    {
+        const LLSD& row = *it;
+        LLUUID public_key = row[EXPERIENCE_ID].asUUID();
 
-		// If no Retry-After, look for Cache-Control max-age
-		F64 expires = 0.0;
-		if (LLExperienceCache::expirationFromCacheControl(getResponseHeaders(), &expires))
-		{
-			return expires;
-		}
+        LL_DEBUGS("ExperienceCache") << "Received result for " << public_key
+            << " display '" << row[LLExperienceCache::NAME].asString() << "'" << LL_ENDL;
 
-		// No information in header, make a guess
-		if (status == 503)
-		{
-			// ...service unavailable, retry soon
-			const F64 SERVICE_UNAVAILABLE_DELAY = 600.0; // 10 min
-			return SERVICE_UNAVAILABLE_DELAY;
-		}
-		else if (status == 499)
-		{
-			// ...we were probably too busy, retry quickly
-			const F64 BUSY_DELAY = 10.0; // 10 seconds
-			return BUSY_DELAY;
+        processExperience(public_key, row);
+    }
 
-		}
-		else
-		{
-			// ...other unexpected error
-			const F64 DEFAULT_DELAY = 3600.0; // 1 hour
-			return DEFAULT_DELAY;
-		}
-	}
+    LLSD error_ids = result["error_ids"];
+    
+    for (LLSD::array_const_iterator errIt = error_ids.beginArray(); 
+        errIt != error_ids.endArray(); ++errIt)
+    {
+        LLUUID id = errIt->asUUID();
+        LLSD exp;
+        exp[EXPIRES] = DEFAULT_EXPIRATION;
+        exp[EXPERIENCE_ID] = id;
+        exp[PROPERTIES] = PROPERTY_INVALID;
+        exp[MISSING] = true;
+        exp[QUOTA] = DEFAULT_QUOTA;
+
+        processExperience(id, exp);
+        LL_WARNS("ExperienceCache") << "LLExperienceResponder::result() error result for " << id << LL_ENDL;
+    }
 
-private:
-	ask_queue_t mKeys;
-};
+}
 
 
 void LLExperienceCache::requestExperiences()
 {
-	if(sAskQueue.empty() || sLookupURL.empty())
-		return;
-
-	F64 now = LLFrameTimer::getTotalSeconds();
+    if (mCapability.empty())
+    {
+        LL_WARNS("ExperienceCache") << "Capability query method not set." << LL_ENDL;
+        return;
+    }
 
-	const U32 EXP_URL_SEND_THRESHOLD = 3000;
-	const U32 PAGE_SIZE = EXP_URL_SEND_THRESHOLD/UUID_STR_LENGTH;
+    std::string urlBase = mCapability("GetExperienceInfo");
+    if (urlBase.empty())
+    {
+        LL_WARNS("ExperienceCache") << "No Experience capability." << LL_ENDL;
+        return;
+    }
 
-	std::ostringstream ostr;
+    if (*urlBase.rbegin() != '/')
+    {
+        urlBase += "/";
+    }
+    urlBase += "id/";
 
-	ask_queue_t keys;
 
-	ostr << sLookupURL << "?page_size=" << PAGE_SIZE;
+	F64 now = LLFrameTimer::getTotalSeconds();
 
-	int request_count = 0;
-	while(!sAskQueue.empty() && request_count < sMaximumLookups)
-	{
-		ask_queue_t::iterator it = sAskQueue.begin();
-		const LLUUID& key = it->first;
-		const std::string& key_type = it->second;
+    const U32 EXP_URL_SEND_THRESHOLD = 3000;
+    const U32 PAGE_SIZE = EXP_URL_SEND_THRESHOLD / UUID_STR_LENGTH;
 
-		ostr << '&' << key_type << '=' << key.asString() ;
-		
-		keys[key]=key_type;
-		request_count++;
+    std::ostringstream ostr;
+    ostr << urlBase << "?page_size=" << PAGE_SIZE;
+    RequestQueue_t  requests;
 
-		sPendingQueue[key] = now;
-			
-		if(ostr.tellp() > EXP_URL_SEND_THRESHOLD)
-		{
-			LL_DEBUGS("ExperienceCache") <<  "requestExperiences() query: " << ostr.str() << LL_ENDL;
-			LLHTTPClient::get(ostr.str(), new LLExperienceResponder(keys));
-			ostr.clear();
-			ostr.str(sLookupURL);
-			ostr << "?page_size=" << PAGE_SIZE;
-			keys.clear();
-		}
-		sAskQueue.erase(it);
-	}
+    while (!mRequestQueue.empty())
+    {
+        RequestQueue_t::iterator it = mRequestQueue.begin();
+        LLUUID key = (*it);
+        mRequestQueue.erase(it);
+        requests.insert(key);
+
+        ostr << "&" << EXPERIENCE_ID << "=" << key.asString();
+        mPendingQueue[key] = now;
+        
+        if (mRequestQueue.empty() || (ostr.tellp() > EXP_URL_SEND_THRESHOLD))
+        {   // request is placed in the coprocedure pool for the ExpCache cache.  Throttling is done by the pool itself.
+            LLCoprocedureManager::getInstance()->enqueueCoprocedure("ExpCache", "Request",
+                boost::bind(&LLExperienceCache::requestExperiencesCoro, this, _1, ostr.str(), requests) );
+
+            ostr.str(std::string());
+            ostr << urlBase << "?page_size=" << PAGE_SIZE;
+            requests.clear();
+        }
+    }
 
-	if(ostr.tellp() > sLookupURL.size())
-	{
-		LL_DEBUGS("ExperienceCache") <<  "requestExperiences() query 2: " << ostr.str() << LL_ENDL;
-		LLHTTPClient::get(ostr.str(), new LLExperienceResponder(keys));
-	}
 }
 
 
@@ -427,9 +373,9 @@ bool LLExperienceCache::isRequestPending(const LLUUID& public_key)
 	bool isPending = false;
 	const F64 PENDING_TIMEOUT_SECS = 5.0 * 60.0;
 
-	pending_queue_t::const_iterator it = sPendingQueue.find(public_key);
+    PendingQueue_t::const_iterator it = mPendingQueue.find(public_key);
 
-	if(it != sPendingQueue.end())
+	if(it != mPendingQueue.end())
 	{
 		F64 expire_time = LLFrameTimer::getTotalSeconds() - PENDING_TIMEOUT_SECS;
 		isPending = (it->second > expire_time);
@@ -438,69 +384,70 @@ bool LLExperienceCache::isRequestPending(const LLUUID& public_key)
 	return isPending;
 }
 
-
-void LLExperienceCache::setLookupURL(const std::string& lookup_url)
+void LLExperienceCache::setCapabilityQuery(LLExperienceCache::CapabilityQuery_t queryfn)
 {
-	sLookupURL = lookup_url;
-	if(!sLookupURL.empty())
-	{
-		sLookupURL += "id/";
-	}
+    mCapability = queryfn;
 }
 
-bool LLExperienceCache::hasLookupURL()
-{
-	return !sLookupURL.empty();
-}
 
-void LLExperienceCache::idle()
+void LLExperienceCache::idleCoro()
 {
-	const F32 SECS_BETWEEN_REQUESTS = 0.1f;
-	if (!sRequestTimer.checkExpirationAndReset(SECS_BETWEEN_REQUESTS))
-	{
-		return;
-	}
+    const F32 SECS_BETWEEN_REQUESTS = 0.5f;
+    const F32 ERASE_EXPIRED_TIMEOUT = 60.f; // seconds
 
-	// Must be large relative to above
-	const F32 ERASE_EXPIRED_TIMEOUT = 60.f; // seconds
-	if (sEraseExpiredTimer.checkExpirationAndReset(ERASE_EXPIRED_TIMEOUT))
-	{
-		eraseExpired();
-	}
+    LL_INFOS("ExperienceCache") << "Launching Experience cache idle coro." << LL_ENDL;
+    LLEventTimeout timeout;
 
-	if(!sAskQueue.empty())
-	{
-		requestExperiences();
-	}
+    do 
+    {
+        timeout.eventAfter(SECS_BETWEEN_REQUESTS, LLSD());
+        llcoro::waitForEventOn(timeout);
+
+        if (mEraseExpiredTimer.checkExpirationAndReset(ERASE_EXPIRED_TIMEOUT))
+        {
+            eraseExpired();
+        }
+
+        if (!mRequestQueue.empty())
+        {
+            requestExperiences();
+        }
+
+    } while (!mShutdown);
+
+    // The coroutine system will likely be shut down by the time we get to this point
+    // (or at least no further cycling will occur on it since the user has decided to quit.)
 }
 
 void LLExperienceCache::erase(const LLUUID& key)
 {
-	cache_t::iterator it = sCache.find(key);
+	cache_t::iterator it = mCache.find(key);
 				
-	if(it != sCache.end())
+	if(it != mCache.end())
 	{
-		sCache.erase(it);
+		mCache.erase(it);
 	}
 }
 
 void LLExperienceCache::eraseExpired()
 {
 	F64 now = LLFrameTimer::getTotalSeconds();
-	cache_t::iterator it = sCache.begin();
-	while (it != sCache.end())
+	cache_t::iterator it = mCache.begin();
+	while (it != mCache.end())
 	{
 		cache_t::iterator cur = it;
 		LLSD& exp = cur->second;
 		++it;
 
+        //LL_INFOS("ExperienceCache") << "Testing experience \"" << exp[NAME] << "\" with exp time " << exp[EXPIRES].asReal() << "(now = " << now << ")" << LL_ENDL;
+
 		if(exp.has(EXPIRES) && exp[EXPIRES].asReal() < now)
 		{
             if(!exp.has(EXPERIENCE_ID))
 			{
                 LL_WARNS("ExperienceCache") << "Removing experience with no id " << LL_ENDL ;
-                sCache.erase(cur);
-				}
+                mCache.erase(cur);
+			}
             else
             {
                 LLUUID id = exp[EXPERIENCE_ID].asUUID();
@@ -512,7 +459,7 @@ void LLExperienceCache::eraseExpired()
 				else
 				{
                     LL_WARNS("ExperienceCache") << "Removing invalid experience " << id << LL_ENDL ;
-					sCache.erase(cur);
+					mCache.erase(cur);
 				}
 			}
 		}
@@ -521,11 +468,11 @@ void LLExperienceCache::eraseExpired()
 	
 bool LLExperienceCache::fetch(const LLUUID& key, bool refresh/* = true*/)
 {
-	if(!key.isNull() && !isRequestPending(key) && (refresh || sCache.find(key)==sCache.end()))
+	if(!key.isNull() && !isRequestPending(key) && (refresh || mCache.find(key)==mCache.end()))
 	{
-		LL_DEBUGS("ExperienceCache") << " queue request for " << EXPERIENCE_ID << " " << key << LL_ENDL ;
-		sAskQueue[key]=EXPERIENCE_ID;
+		LL_DEBUGS("ExperienceCache") << " queue request for " << EXPERIENCE_ID << " " << key << LL_ENDL;
 
+        mRequestQueue.insert(key);
 		return true;
 	}
 	return false;
@@ -549,13 +496,12 @@ const LLSD& LLExperienceCache::get(const LLUUID& key)
 	
 	if(key.isNull()) 
 		return empty;
-	cache_t::const_iterator it = sCache.find(key);
+	cache_t::const_iterator it = mCache.find(key);
 
-	if (it != sCache.end())
+	if (it != mCache.end())
 	{
 		return it->second;
 	}
-
 	fetch(key);
 
 	return empty;
@@ -565,8 +511,8 @@ void LLExperienceCache::get(const LLUUID& key, LLExperienceCache::Callback_t slo
 	if(key.isNull()) 
 		return;
 
-	cache_t::const_iterator it = sCache.find(key);
-	if (it != sCache.end())
+	cache_t::const_iterator it = mCache.find(key);
+	if (it != mCache.end())
 	{
 		// ...name already exists in cache, fire callback now
 		callback_signal_t signal;
@@ -580,28 +526,10 @@ void LLExperienceCache::get(const LLUUID& key, LLExperienceCache::Callback_t slo
 
 	signal_ptr signal = signal_ptr(new callback_signal_t());
 	
-	std::pair<signal_map_t::iterator, bool> result = sSignalMap.insert(signal_map_t::value_type(key, signal));
+	std::pair<signal_map_t::iterator, bool> result = mSignalMap.insert(signal_map_t::value_type(key, signal));
 	if (!result.second)
-		signal = result.first.second;
+		signal = (*result.first).second;
 	signal->connect(slot);
-	
-#if 0
-	// always store additional callback, even if request is pending
-	signal_map_t::iterator sig_it = sSignalMap.find(key);
-	if (sig_it == sSignalMap.end())
-	{
-		// ...new callback for this id
-		signal_ptr signal = signal_ptr(new callback_signal_t());
-		signal->connect(slot);
-		sSignalMap[key] = signal;
-	}
-	else
-	{
-		// ...existing callback, bind additional slot
-		callback_signal_t* signal = sig_it->second;
-		signal->connect(slot);
-	}
-#endif
 }
 
 //=========================================================================
@@ -610,14 +538,71 @@ void LLExperienceCacheImpl::mapKeys(const LLSD& legacyKeys)
 	LLSD::array_const_iterator exp = legacyKeys.beginArray();
 	for (/**/; exp != legacyKeys.endArray(); ++exp)
 	{
-		if (exp->has(LLExperienceCache::EXPERIENCE_ID) && exp->has(LLExperienceCache::PRIVATE_KEY))
+        if (exp->has(LLExperienceCacheImpl::EXPERIENCE_ID) && exp->has(LLExperienceCacheImpl::PRIVATE_KEY))
 		{
-			privateToPublicKeyMap[(*exp)[LLExperienceCache::PRIVATE_KEY].asUUID()] = (*exp)[LLExperienceCache::EXPERIENCE_ID].asUUID();
+            LLExperienceCacheImpl::privateToPublicKeyMap[(*exp)[LLExperienceCacheImpl::PRIVATE_KEY].asUUID()] = 
+                (*exp)[LLExperienceCacheImpl::EXPERIENCE_ID].asUUID();
 		}
 	}
 }
 
-bool LLExperienceCacheImpl::max_age_from_cache_control(const std::string& cache_control, S32 *max_age)
+// Return time to retry a request that generated an error, based on
+// error type and headers.  Return value is seconds-since-epoch.
+F64 LLExperienceCacheImpl::getErrorRetryDeltaTime(S32 status, LLSD headers)
+{
+
+    // Retry-After takes priority
+    LLSD retry_after = headers["retry-after"];
+    if (retry_after.isDefined())
+    {
+        // We only support the delta-seconds type
+        S32 delta_seconds = retry_after.asInteger();
+        if (delta_seconds > 0)
+        {
+            // ...valid delta-seconds
+            return F64(delta_seconds);
+        }
+    }
+
+    // If no Retry-After, look for Cache-Control max-age
+    // Allow the header to override the default
+    LLSD cache_control_header = headers["cache-control"];
+    if (cache_control_header.isDefined())
+    {
+        S32 max_age = 0;
+        std::string cache_control = cache_control_header.asString();
+        if (LLExperienceCacheImpl::maxAgeFromCacheControl(cache_control, &max_age))
+        {
+            LL_WARNS("ExperienceCache")
+                << "got EXPIRES from headers, max_age " << max_age
+                << LL_ENDL;
+            return (F64)max_age;
+        }
+    }
+
+    // No information in header, make a guess
+    if (status == 503)
+    {
+        // ...service unavailable, retry soon
+        const F64 SERVICE_UNAVAILABLE_DELAY = 600.0; // 10 min
+        return SERVICE_UNAVAILABLE_DELAY;
+    }
+    else if (status == 499)
+    {
+        // ...we were probably too busy, retry quickly
+        const F64 BUSY_DELAY = 10.0; // 10 seconds
+        return BUSY_DELAY;
+
+    }
+    else
+    {
+        // ...other unexpected error
+        const F64 DEFAULT_DELAY = 3600.0; // 1 hour
+        return DEFAULT_DELAY;
+    }
+}
+
+bool LLExperienceCacheImpl::maxAgeFromCacheControl(const std::string& cache_control, S32 *max_age)
 {
 	// Split the string on "," to get a list of directives
 	typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
diff --git a/indra/llmessage/llexperiencecache.h b/indra/llmessage/llexperiencecache.h
index 8da038a8c3..937225a80a 100644
--- a/indra/llmessage/llexperiencecache.h
+++ b/indra/llmessage/llexperiencecache.h
@@ -31,7 +31,9 @@
 
 #include "linden_common.h"
 #include "llsingleton.h"
+#include "llframetimer.h"
 #include "llsd.h"
+#include "llcorehttputil.h"
 #include <boost/signals2.hpp>
 #include <boost/function.hpp>
 
@@ -44,8 +46,11 @@ class LLExperienceCache: public LLSingleton < LLExperienceCache >
     friend class LLSingleton < LLExperienceCache > ;
 
 public:
+    typedef boost::function<std::string(const std::string &)> CapabilityQuery_t;
     typedef boost::function<void(const LLSD &)> Callback_t;
 
+    void cleanup();
+
     void erase(const LLUUID& key);
     bool fetch(const LLUUID& key, bool refresh = false);
     void insert(const LLSD& experience_data);
@@ -54,7 +59,39 @@ public:
     // If name information is in cache, callback will be called immediately.
     void get(const LLUUID& key, Callback_t slot);
 
+    bool isRequestPending(const LLUUID& public_key);
+
+    void setCapabilityQuery(CapabilityQuery_t queryfn);
+
+    static const std::string NAME;			// "name"
+    static const std::string EXPERIENCE_ID;	// "public_id"
+    static const std::string AGENT_ID;      // "agent_id"
+    static const std::string GROUP_ID;      // "group_id"
+    static const std::string PROPERTIES;	// "properties"
+    static const std::string EXPIRES;		// "expiration"  
+    static const std::string DESCRIPTION;	// "description"
+    static const std::string QUOTA;         // "quota"
+    static const std::string MATURITY;      // "maturity"
+    static const std::string METADATA;      // "extended_metadata"
+    static const std::string SLURL;         // "slurl"
+
+    static const std::string MISSING;       // "DoesNotExist"
+
+    // should be in sync with experience-api/experiences/models.py
+    static const int PROPERTY_INVALID;		// 1 << 0
+    static const int PROPERTY_PRIVILEGED;	// 1 << 3
+    static const int PROPERTY_GRID;			// 1 << 4
+    static const int PROPERTY_PRIVATE;		// 1 << 5
+    static const int PROPERTY_DISABLED;		// 1 << 6  
+    static const int PROPERTY_SUSPENDED;	// 1 << 7
+
 private:
+    LLExperienceCache();
+    virtual ~LLExperienceCache();
+
+    virtual void initSingleton();
+
+
     // Callback types for get() 
     typedef boost::signals2::signal < void(const LLSD &) > callback_signal_t;
 	typedef boost::shared_ptr<callback_signal_t> signal_ptr;
@@ -64,63 +101,43 @@ private:
 	typedef std::map<LLUUID, signal_ptr> signal_map_t;
 	typedef std::map<LLUUID, LLSD> cache_t;
 	
-	typedef std::set<LLUUID> ask_queue_t;
-	
-	
+	typedef std::set<LLUUID> RequestQueue_t;
+    typedef std::map<LLUUID, F64> PendingQueue_t;
+
 	//--------------------------------------------
 	static const std::string PRIVATE_KEY;	// "private_id"
-	static const std::string MISSING;       // "DoesNotExist"
-	
-	static const std::string AGENT_ID;      // "agent_id"
-	static const std::string GROUP_ID;      // "group_id"
-	static const std::string EXPERIENCE_ID;	// "public_id"
-	static const std::string NAME;			// "name"
-	static const std::string PROPERTIES;	// "properties"
-	static const std::string EXPIRES;		// "expiration"  
-	static const std::string DESCRIPTION;	// "description"
-	static const std::string QUOTA;         // "quota"
-	static const std::string MATURITY;      // "maturity"
-	static const std::string METADATA;      // "extended_metadata"
-	static const std::string SLURL;         // "slurl"
-	
-	// should be in sync with experience-api/experiences/models.py
-	static const int PROPERTY_INVALID;		// 1 << 0
-	static const int PROPERTY_PRIVILEGED;	// 1 << 3
-	static const int PROPERTY_GRID;			// 1 << 4
-	static const int PROPERTY_PRIVATE;		// 1 << 5
-	static const int PROPERTY_DISABLED;		// 1 << 6  
-	static const int PROPERTY_SUSPENDED;	// 1 << 7
 	
 	// default values
 	static const F64 DEFAULT_EXPIRATION; 	// 600.0
 	static const S32 DEFAULT_QUOTA; 		// 128 this is megabytes
 	
 //--------------------------------------------
-    LLExperienceCache();
-    virtual ~LLExperienceCache();
-
-    void exportFile(std::ostream& ostr) const;
-    void importFile(std::istream& istr);
-
-//--------------------------------------------
     void processExperience(const LLUUID& public_key, const LLSD& experience);
 
 //--------------------------------------------
-	cache_t			sCache;
-	signal_map_t	sSignalMap;	
-	ask_queue_t		sAskQueue;
-	
+	cache_t			mCache;
+	signal_map_t	mSignalMap;	
+	RequestQueue_t	mRequestQueue;
+    PendingQueue_t  mPendingQueue;
+
+    LLFrameTimer    mRequestTimer;
+    LLFrameTimer    mEraseExpiredTimer;    // Periodically clean out expired entries from the cache
+    CapabilityQuery_t mCapability;
+    std::string     mCacheFileName;
+    bool            mShutdown;
+
+    void idleCoro();
 	void eraseExpired();
-	
-	void setLookupURL(const std::string& lookup_url);
-	bool hasLookupURL();
+    void requestExperiencesCoro(LLCoreHttpUtil::HttpCoroutineAdapter::ptr_t &, std::string, RequestQueue_t);
+    void requestExperiences();
 
 	void setMaximumLookups(int maximumLookups);
 
-	void idle();
-	void bootstrap(const LLSD& legacyKeys, int initialExpiration);
-	
+    void bootstrap(const LLSD& legacyKeys, int initialExpiration);
+    void exportFile(std::ostream& ostr) const;
+    void importFile(std::istream& istr);
 
+    // 
 	const cache_t& getCached();
 
 	// maps an experience private key to the experience id