diff options
Diffstat (limited to 'indra/llmessage')
43 files changed, 1827 insertions, 767 deletions
diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index 0f40a670fa..d98781e9e6 100644 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -254,6 +254,14 @@ if (LL_TESTS) "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_llsdmessage_peer.py" ) + LL_ADD_INTEGRATION_TEST( + llhttpclient + "llhttpclient.cpp" + "${test_libs}" + ${PYTHON_EXECUTABLE} + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_llsdmessage_peer.py" + ) + LL_ADD_INTEGRATION_TEST(llavatarnamecache "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llhost "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llpartdata "" "${test_libs}") diff --git a/indra/llmessage/llavatarnamecache.cpp b/indra/llmessage/llavatarnamecache.cpp index 97f2792686..8f91d2a23f 100644 --- a/indra/llmessage/llavatarnamecache.cpp +++ b/indra/llmessage/llavatarnamecache.cpp @@ -43,26 +43,26 @@ namespace LLAvatarNameCache { use_display_name_signal_t mUseDisplayNamesSignal; - // Manual override for display names - can disable even if the region - // supports it. - bool sUseDisplayNames = true; - // Cache starts in a paused state until we can determine if the // current region supports display names. bool sRunning = false; + // Use the People API (modern) for fetching name if true. Use the old legacy protocol if false. + // For testing, there's a UsePeopleAPI setting that can be flipped (must restart viewer). + bool sUsePeopleAPI = true; + // Base lookup URL for name service. // On simulator, loaded from indra.xml // On viewer, usually a simulator capability (at People API team's request) // Includes the trailing slash, like "http://pdp60.lindenlab.com:8000/agents/" std::string sNameLookupURL; - // accumulated agent IDs for next query against service + // Accumulated agent IDs for next query against service typedef std::set<LLUUID> ask_queue_t; ask_queue_t sAskQueue; - // agent IDs that have been requested, but with no reply - // maps agent ID to frame time request was made + // Agent IDs that have been requested, but with no reply. + // Maps agent ID to frame time request was made. typedef std::map<LLUUID, F64> pending_queue_t; pending_queue_t sPendingQueue; @@ -73,45 +73,43 @@ namespace LLAvatarNameCache typedef std::map<LLUUID, callback_signal_t*> signal_map_t; signal_map_t sSignalMap; - // names we know about + // The cache at last, i.e. avatar names we know about. typedef std::map<LLUUID, LLAvatarName> cache_t; cache_t sCache; - // Send bulk lookup requests a few times a second at most - // only need per-frame timing resolution + // Send bulk lookup requests a few times a second at most. + // Only need per-frame timing resolution. LLFrameTimer sRequestTimer; - /// Maximum time an unrefreshed cache entry is allowed + // Maximum time an unrefreshed cache entry is allowed. const F64 MAX_UNREFRESHED_TIME = 20.0 * 60.0; - /// Time when unrefreshed cached names were checked last + // Time when unrefreshed cached names were checked last. static F64 sLastExpireCheck; + // Time-to-live for a temp cache entry. + const F64 TEMP_CACHE_ENTRY_LIFETIME = 60.0; + //----------------------------------------------------------------------- // Internal methods //----------------------------------------------------------------------- // Handle name response off network. - // Optionally skip adding to cache, used when this is a fallback to the - // legacy name system. void processName(const LLUUID& agent_id, - const LLAvatarName& av_name, - bool add_to_cache); + const LLAvatarName& av_name); void requestNamesViaCapability(); - // Legacy name system callback + // Legacy name system callbacks void legacyNameCallback(const LLUUID& agent_id, const std::string& full_name, - bool is_group - ); - + bool is_group); + void legacyNameFetch(const LLUUID& agent_id, + const std::string& full_name, + bool is_group); + void requestNamesViaLegacy(); - // Fill in an LLAvatarName with the legacy name data - void buildLegacyName(const std::string& full_name, - LLAvatarName* av_name); - // Do a single callback to a given slot void fireSignal(const LLUUID& agent_id, const callback_slot_t& slot, @@ -206,20 +204,11 @@ public: // Use expiration time from header av_name.mExpires = expires; - // Some avatars don't have explicit display names set - if (av_name.mDisplayName.empty()) - { - av_name.mDisplayName = av_name.mUsername; - } - - LL_DEBUGS("AvNameCache") << "LLAvatarNameResponder::result for " << agent_id << " " - << "user '" << av_name.mUsername << "' " - << "display '" << av_name.mDisplayName << "' " - << "expires in " << expires - now << " seconds" - << LL_ENDL; + LL_DEBUGS("AvNameCache") << "LLAvatarNameResponder::result for " << agent_id << LL_ENDL; + av_name.dump(); // cache it and fire signals - LLAvatarNameCache::processName(agent_id, av_name, true); + LLAvatarNameCache::processName(agent_id, av_name); } // Same logic as error response case @@ -274,41 +263,36 @@ void LLAvatarNameCache::handleAgentError(const LLUUID& agent_id) { // there is no existing cache entry, so make a temporary name from legacy LL_WARNS("AvNameCache") << "LLAvatarNameCache get legacy for agent " - << agent_id << LL_ENDL; + << agent_id << LL_ENDL; gCacheName->get(agent_id, false, // legacy compatibility - boost::bind(&LLAvatarNameCache::legacyNameCallback, - _1, _2, _3)); + boost::bind(&LLAvatarNameCache::legacyNameFetch, _1, _2, _3)); } else { - // we have a chached (but probably expired) entry - since that would have + // we have a cached (but probably expired) entry - since that would have // been returned by the get method, there is no need to signal anyone // Clear this agent from the pending list LLAvatarNameCache::sPendingQueue.erase(agent_id); - const LLAvatarName& av_name = existing->second; - LL_DEBUGS("AvNameCache") << "LLAvatarNameCache use cache for agent " - << agent_id - << "user '" << av_name.mUsername << "' " - << "display '" << av_name.mDisplayName << "' " - << "expires in " << av_name.mExpires - LLFrameTimer::getTotalSeconds() << " seconds" - << LL_ENDL; + LLAvatarName& av_name = existing->second; + LL_DEBUGS("AvNameCache") << "LLAvatarNameCache use cache for agent " << agent_id << LL_ENDL; + av_name.dump(); + + // Reset expiry time so we don't constantly rerequest. + av_name.setExpires(TEMP_CACHE_ENTRY_LIFETIME); } } -void LLAvatarNameCache::processName(const LLUUID& agent_id, - const LLAvatarName& av_name, - bool add_to_cache) +void LLAvatarNameCache::processName(const LLUUID& agent_id, const LLAvatarName& av_name) { - if (add_to_cache) - { - sCache[agent_id] = av_name; - } + // Add to the cache + sCache[agent_id] = av_name; + // Suppress request from the queue sPendingQueue.erase(agent_id); - // signal everyone waiting on this name + // Signal everyone waiting on this name signal_map_t::iterator sig_it = sSignalMap.find(agent_id); if (sig_it != sSignalMap.end()) { @@ -330,8 +314,9 @@ void LLAvatarNameCache::requestNamesViaCapability() // http://pdp60.lindenlab.com:8000/agents/?ids=3941037e-78ab-45f0-b421-bd6e77c1804d&ids=0012809d-7d2d-4c24-9609-af1230a37715&ids=0019aaba-24af-4f0a-aa72-6457953cf7f0 // // Apache can handle URLs of 4096 chars, but let's be conservative - const U32 NAME_URL_MAX = 4096; - const U32 NAME_URL_SEND_THRESHOLD = 3000; + static const U32 NAME_URL_MAX = 4096; + static const U32 NAME_URL_SEND_THRESHOLD = 3500; + std::string url; url.reserve(NAME_URL_MAX); @@ -339,10 +324,12 @@ void LLAvatarNameCache::requestNamesViaCapability() agent_ids.reserve(128); U32 ids = 0; - ask_queue_t::const_iterator it = sAskQueue.begin(); - for ( ; it != sAskQueue.end(); ++it) + ask_queue_t::const_iterator it; + while(!sAskQueue.empty()) { - const LLUUID& agent_id = *it; + it = sAskQueue.begin(); + LLUUID agent_id = *it; + sAskQueue.erase(it); if (url.empty()) { @@ -365,57 +352,63 @@ void LLAvatarNameCache::requestNamesViaCapability() if (url.size() > NAME_URL_SEND_THRESHOLD) { - LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::requestNamesViaCapability first " - << ids << " ids" - << LL_ENDL; - LLHTTPClient::get(url, new LLAvatarNameResponder(agent_ids)); - url.clear(); - agent_ids.clear(); + break; } } if (!url.empty()) { - LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::requestNamesViaCapability all " + LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::requestNamesViaCapability requested " << ids << " ids" << LL_ENDL; LLHTTPClient::get(url, new LLAvatarNameResponder(agent_ids)); - url.clear(); - agent_ids.clear(); } - - // We've moved all asks to the pending request queue - sAskQueue.clear(); } void LLAvatarNameCache::legacyNameCallback(const LLUUID& agent_id, const std::string& full_name, bool is_group) { - // Construct a dummy record for this name. By convention, SLID is blank - // Never expires, but not written to disk, so lasts until end of session. - LLAvatarName av_name; - LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::legacyNameCallback " - << "agent " << agent_id << " " + // Put the received data in the cache + legacyNameFetch(agent_id, full_name, is_group); + + // Retrieve the name and set it to never (or almost never...) expire: when we are using the legacy + // protocol, we do not get an expiration date for each name and there's no reason to ask the + // data again and again so we set the expiration time to the largest value admissible. + std::map<LLUUID,LLAvatarName>::iterator av_record = sCache.find(agent_id); + LLAvatarName& av_name = av_record->second; + av_name.setExpires(MAX_UNREFRESHED_TIME); +} + +void LLAvatarNameCache::legacyNameFetch(const LLUUID& agent_id, + const std::string& full_name, + bool is_group) +{ + LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::legacyNameFetch " + << "agent " << agent_id << " " << "full name '" << full_name << "'" - << ( is_group ? " [group]" : "" ) - << LL_ENDL; - buildLegacyName(full_name, &av_name); - - // Don't add to cache, the data already exists in the legacy name system - // cache and we don't want or need duplicate storage, because keeping the - // two copies in sync is complex. - processName(agent_id, av_name, false); + << ( is_group ? " [group]" : "" ) + << LL_ENDL; + + // Construct an av_name record from this name. + LLAvatarName av_name; + av_name.fromString(full_name); + + // Add to cache: we're still using the new cache even if we're using the old (legacy) protocol. + processName(agent_id, av_name); } void LLAvatarNameCache::requestNamesViaLegacy() { + static const S32 MAX_REQUESTS = 100; F64 now = LLFrameTimer::getTotalSeconds(); std::string full_name; - ask_queue_t::const_iterator it = sAskQueue.begin(); - for (; it != sAskQueue.end(); ++it) + ask_queue_t::const_iterator it; + for (S32 requests = 0; !sAskQueue.empty() && requests < MAX_REQUESTS; ++requests) { - const LLUUID& agent_id = *it; + it = sAskQueue.begin(); + LLUUID agent_id = *it; + sAskQueue.erase(it); // Mark as pending first, just in case the callback is immediately // invoked below. This should never happen in practice. @@ -424,22 +417,19 @@ void LLAvatarNameCache::requestNamesViaLegacy() LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::requestNamesViaLegacy agent " << agent_id << LL_ENDL; gCacheName->get(agent_id, false, // legacy compatibility - boost::bind(&LLAvatarNameCache::legacyNameCallback, - _1, _2, _3)); + boost::bind(&LLAvatarNameCache::legacyNameCallback, _1, _2, _3)); } - - // We've either answered immediately or moved all asks to the - // pending queue - sAskQueue.clear(); } -void LLAvatarNameCache::initClass(bool running) +void LLAvatarNameCache::initClass(bool running, bool usePeopleAPI) { sRunning = running; + sUsePeopleAPI = usePeopleAPI; } void LLAvatarNameCache::cleanupClass() { + sCache.clear(); } void LLAvatarNameCache::importFile(std::istream& istr) @@ -478,7 +468,7 @@ void LLAvatarNameCache::exportFile(std::ostream& ostr) const LLUUID& agent_id = it->first; const LLAvatarName& av_name = it->second; // Do not write temporary or expired entries to the stored cache - if (!av_name.mIsTemporaryName && av_name.mExpires >= max_unrefreshed) + if (av_name.isValidName(max_unrefreshed)) { // key must be a string agents[agent_id.asString()] = av_name.asLLSD(); @@ -499,6 +489,11 @@ bool LLAvatarNameCache::hasNameLookupURL() return !sNameLookupURL.empty(); } +bool LLAvatarNameCache::usePeopleAPI() +{ + return hasNameLookupURL() && sUsePeopleAPI; +} + void LLAvatarNameCache::idle() { // By convention, start running at first idle() call @@ -507,25 +502,30 @@ void LLAvatarNameCache::idle() // *TODO: Possibly re-enabled this based on People API load measurements // 100 ms is the threshold for "user speed" operations, so we can // stall for about that long to batch up requests. - //const F32 SECS_BETWEEN_REQUESTS = 0.1f; - //if (!sRequestTimer.checkExpirationAndReset(SECS_BETWEEN_REQUESTS)) - //{ - // return; - //} + const F32 SECS_BETWEEN_REQUESTS = 0.1f; + if (!sRequestTimer.hasExpired()) + { + return; + } if (!sAskQueue.empty()) { - if (useDisplayNames()) + if (usePeopleAPI()) { requestNamesViaCapability(); } else { - // ...fall back to legacy name cache system requestNamesViaLegacy(); } } + if (sAskQueue.empty()) + { + // cleared the list, reset the request timer. + sRequestTimer.resetWithExpiry(SECS_BETWEEN_REQUESTS); + } + // erase anything that has not been refreshed for more than MAX_UNREFRESHED_TIME eraseUnrefreshed(); } @@ -559,9 +559,8 @@ void LLAvatarNameCache::eraseUnrefreshed() const LLAvatarName& av_name = it->second; if (av_name.mExpires < max_unrefreshed) { - const LLUUID& agent_id = it->first; - LL_DEBUGS("AvNameCache") << agent_id - << " user '" << av_name.mUsername << "' " + LL_DEBUGS("AvNameCache") << it->first + << " user '" << av_name.getAccountName() << "' " << "expired " << now - av_name.mExpires << " secs ago" << LL_ENDL; sCache.erase(it++); @@ -575,20 +574,6 @@ void LLAvatarNameCache::eraseUnrefreshed() } } -void LLAvatarNameCache::buildLegacyName(const std::string& full_name, - LLAvatarName* av_name) -{ - llassert(av_name); - av_name->mUsername = ""; - av_name->mDisplayName = full_name; - av_name->mIsDisplayNameDefault = true; - av_name->mIsTemporaryName = true; - av_name->mExpires = F64_MAX; // not used because these are not cached - LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::buildLegacyName " - << full_name - << LL_ENDL; -} - // fills in av_name if it has it in the cache, even if expired (can check expiry time) // returns bool specifying if av_name was filled, false otherwise bool LLAvatarNameCache::get(const LLUUID& agent_id, LLAvatarName *av_name) @@ -596,38 +581,24 @@ bool LLAvatarNameCache::get(const LLUUID& agent_id, LLAvatarName *av_name) if (sRunning) { // ...only do immediate lookups when cache is running - if (useDisplayNames()) + std::map<LLUUID,LLAvatarName>::iterator it = sCache.find(agent_id); + if (it != sCache.end()) { - // ...use display names cache - std::map<LLUUID,LLAvatarName>::iterator it = sCache.find(agent_id); - if (it != sCache.end()) - { - *av_name = it->second; + *av_name = it->second; - // re-request name if entry is expired - if (av_name->mExpires < LLFrameTimer::getTotalSeconds()) + // re-request name if entry is expired + if (av_name->mExpires < LLFrameTimer::getTotalSeconds()) + { + if (!isRequestPending(agent_id)) { - if (!isRequestPending(agent_id)) - { - LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::get " - << "refresh agent " << agent_id - << LL_ENDL; - sAskQueue.insert(agent_id); - } + LL_DEBUGS("AvNameCache") << "LLAvatarNameCache::get " + << "refresh agent " << agent_id + << LL_ENDL; + sAskQueue.insert(agent_id); } - - return true; - } - } - else - { - // ...use legacy names cache - std::string full_name; - if (gCacheName->getFullName(agent_id, full_name)) - { - buildLegacyName(full_name, av_name); - return true; } + + return true; } } @@ -651,37 +622,23 @@ void LLAvatarNameCache::fireSignal(const LLUUID& agent_id, signal(agent_id, av_name); } -void LLAvatarNameCache::get(const LLUUID& agent_id, callback_slot_t slot) +LLAvatarNameCache::callback_connection_t LLAvatarNameCache::get(const LLUUID& agent_id, callback_slot_t slot) { + callback_connection_t connection; + if (sRunning) { // ...only do immediate lookups when cache is running - if (useDisplayNames()) + std::map<LLUUID,LLAvatarName>::iterator it = sCache.find(agent_id); + if (it != sCache.end()) { - // ...use new cache - std::map<LLUUID,LLAvatarName>::iterator it = sCache.find(agent_id); - if (it != sCache.end()) - { - const LLAvatarName& av_name = it->second; - - if (av_name.mExpires > LLFrameTimer::getTotalSeconds()) - { - // ...name already exists in cache, fire callback now - fireSignal(agent_id, slot, av_name); - return; - } - } - } - else - { - // ...use old name system - std::string full_name; - if (gCacheName->getFullName(agent_id, full_name)) + const LLAvatarName& av_name = it->second; + + if (av_name.mExpires > LLFrameTimer::getTotalSeconds()) { - LLAvatarName av_name; - buildLegacyName(full_name, &av_name); + // ...name already exists in cache, fire callback now fireSignal(agent_id, slot, av_name); - return; + return connection; } } } @@ -698,47 +655,34 @@ void LLAvatarNameCache::get(const LLUUID& agent_id, callback_slot_t slot) { // ...new callback for this id callback_signal_t* signal = new callback_signal_t(); - signal->connect(slot); + connection = signal->connect(slot); sSignalMap[agent_id] = signal; } else { // ...existing callback, bind additional slot callback_signal_t* signal = sig_it->second; - signal->connect(slot); + connection = signal->connect(slot); } + + return connection; } void LLAvatarNameCache::setUseDisplayNames(bool use) { - if (use != sUseDisplayNames) + if (use != LLAvatarName::useDisplayNames()) { - sUseDisplayNames = use; - // flush our cache - sCache.clear(); - + LLAvatarName::setUseDisplayNames(use); mUseDisplayNamesSignal(); } } -bool LLAvatarNameCache::useDisplayNames() -{ - // Must be both manually set on and able to look up names. - return sUseDisplayNames && !sNameLookupURL.empty(); -} - void LLAvatarNameCache::erase(const LLUUID& agent_id) { sCache.erase(agent_id); } -void LLAvatarNameCache::fetch(const LLUUID& agent_id) -{ - // re-request, even if request is already pending - sAskQueue.insert(agent_id); -} - void LLAvatarNameCache::insert(const LLUUID& agent_id, const LLAvatarName& av_name) { // *TODO: update timestamp if zero? diff --git a/indra/llmessage/llavatarnamecache.h b/indra/llmessage/llavatarnamecache.h index 59c1329ffa..2a8eb46187 100644 --- a/indra/llmessage/llavatarnamecache.h +++ b/indra/llmessage/llavatarnamecache.h @@ -37,33 +37,33 @@ class LLUUID; namespace LLAvatarNameCache { - typedef boost::signals2::signal<void (void)> use_display_name_signal_t; // Until the cache is set running, immediate lookups will fail and // async lookups will be queued. This allows us to block requests // until we know if the first region supports display names. - void initClass(bool running); + void initClass(bool running, bool usePeopleAPI); void cleanupClass(); + // Import/export the name cache to file. void importFile(std::istream& istr); void exportFile(std::ostream& ostr); - // On the viewer, usually a simulator capabilitity - // If empty, name cache will fall back to using legacy name - // lookup system + // On the viewer, usually a simulator capabilitity. + // If empty, name cache will fall back to using legacy name lookup system. void setNameLookupURL(const std::string& name_lookup_url); - // Do we have a valid lookup URL, hence are we trying to use the - // new display name lookup system? + // Do we have a valid lookup URL, i.e. are we trying to use the + // more recent display name lookup system? bool hasNameLookupURL(); + bool usePeopleAPI(); // Periodically makes a batch request for display names not already in - // cache. Call once per frame. + // cache. Called once per frame. void idle(); // If name is in cache, returns true and fills in provided LLAvatarName - // otherwise returns false + // otherwise returns false. bool get(const LLUUID& agent_id, LLAvatarName *av_name); // Callback types for get() below @@ -71,26 +71,21 @@ namespace LLAvatarNameCache void (const LLUUID& agent_id, const LLAvatarName& av_name)> callback_signal_t; typedef callback_signal_t::slot_type callback_slot_t; + typedef boost::signals2::connection callback_connection_t; - // Fetches name information and calls callback. - // If name information is in cache, callback will be called immediately. - void get(const LLUUID& agent_id, callback_slot_t slot); + // Fetches name information and calls callbacks. + // If name information is in cache, callbacks will be called immediately. + callback_connection_t get(const LLUUID& agent_id, callback_slot_t slot); - // Allow display names to be explicitly disabled for testing. + // Set display name: flips the switch and triggers the callbacks. void setUseDisplayNames(bool use); - bool useDisplayNames(); - + + void insert(const LLUUID& agent_id, const LLAvatarName& av_name); void erase(const LLUUID& agent_id); - /// Provide some fallback for agents that return errors + /// Provide some fallback for agents that return errors. void handleAgentError(const LLUUID& agent_id); - // Force a re-fetch of the most recent data, but keep the current - // data in cache - void fetch(const LLUUID& agent_id); - - void insert(const LLUUID& agent_id, const LLAvatarName& av_name); - // Compute name expiration time from HTTP Cache-Control header, // or return default value, in seconds from epoch. F64 nameExpirationFromHeaders(LLSD headers); diff --git a/indra/llmessage/llbuffer.cpp b/indra/llmessage/llbuffer.cpp index 0316797f00..01da20f060 100644 --- a/indra/llmessage/llbuffer.cpp +++ b/indra/llmessage/llbuffer.cpp @@ -30,8 +30,10 @@ #include "llbuffer.h" #include "llmath.h" -#include "llmemtype.h" #include "llstl.h" +#include "llthread.h" + +#define ASSERT_LLBUFFERARRAY_MUTEX_LOCKED llassert(!mMutexp || mMutexp->isSelfLocked()); /** * LLSegment @@ -41,7 +43,6 @@ LLSegment::LLSegment() : mData(NULL), mSize(0) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } LLSegment::LLSegment(S32 channel, U8* data, S32 data_len) : @@ -49,12 +50,10 @@ LLSegment::LLSegment(S32 channel, U8* data, S32 data_len) : mData(data), mSize(data_len) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } LLSegment::~LLSegment() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } bool LLSegment::isOnChannel(S32 channel) const @@ -101,7 +100,6 @@ LLHeapBuffer::LLHeapBuffer() : mNextFree(NULL), mReclaimedBytes(0) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); const S32 DEFAULT_HEAP_BUFFER_SIZE = 16384; allocate(DEFAULT_HEAP_BUFFER_SIZE); } @@ -112,7 +110,6 @@ LLHeapBuffer::LLHeapBuffer(S32 size) : mNextFree(NULL), mReclaimedBytes(0) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); allocate(size); } @@ -122,7 +119,6 @@ LLHeapBuffer::LLHeapBuffer(const U8* src, S32 len) : mNextFree(NULL), mReclaimedBytes(0) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); if((len > 0) && src) { allocate(len); @@ -136,7 +132,6 @@ LLHeapBuffer::LLHeapBuffer(const U8* src, S32 len) : // virtual LLHeapBuffer::~LLHeapBuffer() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); delete[] mBuffer; mBuffer = NULL; mSize = 0; @@ -154,7 +149,6 @@ bool LLHeapBuffer::createSegment( S32 size, LLSegment& segment) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); // get actual size of the segment. S32 actual_size = llmin(size, (mSize - S32(mNextFree - mBuffer))); @@ -209,7 +203,6 @@ bool LLHeapBuffer::containsSegment(const LLSegment& segment) const void LLHeapBuffer::allocate(S32 size) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); mReclaimedBytes = 0; mBuffer = new U8[size]; if(mBuffer) @@ -224,15 +217,16 @@ void LLHeapBuffer::allocate(S32 size) * LLBufferArray */ LLBufferArray::LLBufferArray() : - mNextBaseChannel(0) + mNextBaseChannel(0), + mMutexp(NULL) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } LLBufferArray::~LLBufferArray() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); std::for_each(mBuffers.begin(), mBuffers.end(), DeletePointer()); + + delete mMutexp; } // static @@ -243,14 +237,57 @@ LLChannelDescriptors LLBufferArray::makeChannelConsumer( return rv; } +void LLBufferArray::lock() +{ + if(mMutexp) + { + mMutexp->lock() ; + } +} + +void LLBufferArray::unlock() +{ + if(mMutexp) + { + mMutexp->unlock() ; + } +} + +LLMutex* LLBufferArray::getMutex() +{ + return mMutexp ; +} + +void LLBufferArray::setThreaded(bool threaded) +{ + if(threaded) + { + if(!mMutexp) + { + mMutexp = new LLMutex(NULL); + } + } + else + { + if(mMutexp) + { + delete mMutexp ; + mMutexp = NULL ; + } + } +} + LLChannelDescriptors LLBufferArray::nextChannel() { LLChannelDescriptors rv(mNextBaseChannel++); return rv; } +//mMutexp should be locked before calling this. S32 LLBufferArray::capacity() const { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED + S32 total = 0; const_buffer_iterator_t iter = mBuffers.begin(); const_buffer_iterator_t end = mBuffers.end(); @@ -263,7 +300,8 @@ S32 LLBufferArray::capacity() const bool LLBufferArray::append(S32 channel, const U8* src, S32 len) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + LLMutexLock lock(mMutexp) ; + std::vector<LLSegment> segments; if(copyIntoBuffers(channel, src, len, segments)) { @@ -273,9 +311,11 @@ bool LLBufferArray::append(S32 channel, const U8* src, S32 len) return false; } +//mMutexp should be locked before calling this. bool LLBufferArray::prepend(S32 channel, const U8* src, S32 len) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED + std::vector<LLSegment> segments; if(copyIntoBuffers(channel, src, len, segments)) { @@ -291,8 +331,9 @@ bool LLBufferArray::insertAfter( const U8* src, S32 len) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); std::vector<LLSegment> segments; + + LLMutexLock lock(mMutexp) ; if(mSegments.end() != segment) { ++segment; @@ -305,9 +346,11 @@ bool LLBufferArray::insertAfter( return false; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::splitAfter(U8* address) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED + segment_iterator_t end = mSegments.end(); segment_iterator_t it = getSegment(address); if(it == end) @@ -335,21 +378,26 @@ LLBufferArray::segment_iterator_t LLBufferArray::splitAfter(U8* address) return rv; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::beginSegment() { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED return mSegments.begin(); } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::endSegment() { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED return mSegments.end(); } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::constructSegmentAfter( U8* address, LLSegment& segment) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED segment_iterator_t rv = mSegments.begin(); segment_iterator_t end = mSegments.end(); if(!address) @@ -395,8 +443,10 @@ LLBufferArray::segment_iterator_t LLBufferArray::constructSegmentAfter( return rv; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::getSegment(U8* address) { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED segment_iterator_t end = mSegments.end(); if(!address) { @@ -414,9 +464,11 @@ LLBufferArray::segment_iterator_t LLBufferArray::getSegment(U8* address) return end; } +//mMutexp should be locked before calling this. LLBufferArray::const_segment_iterator_t LLBufferArray::getSegment( U8* address) const { + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED const_segment_iterator_t end = mSegments.end(); if(!address) { @@ -466,6 +518,8 @@ S32 LLBufferArray::countAfter(S32 channel, U8* start) const S32 count = 0; S32 offset = 0; const_segment_iterator_t it; + + LLMutexLock lock(mMutexp) ; const_segment_iterator_t end = mSegments.end(); if(start) { @@ -507,7 +561,6 @@ U8* LLBufferArray::readAfter( U8* dest, S32& len) const { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); U8* rv = start; if(!dest || len <= 0) { @@ -517,6 +570,8 @@ U8* LLBufferArray::readAfter( len = 0; S32 bytes_to_copy = 0; const_segment_iterator_t it; + + LLMutexLock lock(mMutexp) ; const_segment_iterator_t end = mSegments.end(); if(start) { @@ -568,7 +623,7 @@ U8* LLBufferArray::seek( U8* start, S32 delta) const { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED const_segment_iterator_t it; const_segment_iterator_t end = mSegments.end(); U8* rv = start; @@ -709,9 +764,12 @@ U8* LLBufferArray::seek( return rv; } +//test use only bool LLBufferArray::takeContents(LLBufferArray& source) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + LLMutexLock lock(mMutexp); + source.lock(); + std::copy( source.mBuffers.begin(), source.mBuffers.end(), @@ -723,14 +781,17 @@ bool LLBufferArray::takeContents(LLBufferArray& source) std::back_insert_iterator<segment_list_t>(mSegments)); source.mSegments.clear(); source.mNextBaseChannel = 0; + source.unlock(); + return true; } +//mMutexp should be locked before calling this. LLBufferArray::segment_iterator_t LLBufferArray::makeSegment( S32 channel, S32 len) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED // start at the end of the buffers, because it is the most likely // to have free space. LLSegment segment; @@ -765,9 +826,10 @@ LLBufferArray::segment_iterator_t LLBufferArray::makeSegment( return send; } +//mMutexp should be locked before calling this. bool LLBufferArray::eraseSegment(const segment_iterator_t& erase_iter) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED // Find out which buffer contains the segment, and if it is found, // ask it to reclaim the memory. @@ -792,14 +854,14 @@ bool LLBufferArray::eraseSegment(const segment_iterator_t& erase_iter) return rv; } - +//mMutexp should be locked before calling this. bool LLBufferArray::copyIntoBuffers( S32 channel, const U8* src, S32 len, std::vector<LLSegment>& segments) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); + ASSERT_LLBUFFERARRAY_MUTEX_LOCKED if(!src || !len) return false; S32 copied = 0; LLSegment segment; diff --git a/indra/llmessage/llbuffer.h b/indra/llmessage/llbuffer.h index 1c42b6fbc6..ccdb9fa7ee 100644 --- a/indra/llmessage/llbuffer.h +++ b/indra/llmessage/llbuffer.h @@ -39,6 +39,7 @@ #include <list> #include <vector> +class LLMutex; /** * @class LLChannelDescriptors * @brief A way simple interface to accesss channels inside a buffer @@ -564,6 +565,29 @@ public: * @return Returns true on success. */ bool eraseSegment(const segment_iterator_t& iter); + + /** + * @brief Lock the mutex if it exists + * This method locks mMutexp to make accessing LLBufferArray thread-safe + */ + void lock(); + + /** + * @brief Unlock the mutex if it exists + */ + void unlock(); + + /** + * @brief Return mMutexp + */ + LLMutex* getMutex(); + + /** + * @brief Set LLBufferArray to be shared across threads or not + * This method is to create mMutexp if is threaded. + * @param threaded Indicates this LLBufferArray instance is shared across threads if true. + */ + void setThreaded(bool threaded); //@} protected: @@ -595,6 +619,7 @@ protected: S32 mNextBaseChannel; buffer_list_t mBuffers; segment_list_t mSegments; + LLMutex* mMutexp; }; #endif // LL_LLBUFFER_H diff --git a/indra/llmessage/llbufferstream.cpp b/indra/llmessage/llbufferstream.cpp index 6257983c43..a51a48edc3 100644 --- a/indra/llmessage/llbufferstream.cpp +++ b/indra/llmessage/llbufferstream.cpp @@ -30,7 +30,7 @@ #include "llbufferstream.h" #include "llbuffer.h" -#include "llmemtype.h" +#include "llthread.h" static const S32 DEFAULT_OUTPUT_SEGMENT_SIZE = 1024 * 4; @@ -43,25 +43,23 @@ LLBufferStreamBuf::LLBufferStreamBuf( mChannels(channels), mBuffer(buffer) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } LLBufferStreamBuf::~LLBufferStreamBuf() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); sync(); } // virtual int LLBufferStreamBuf::underflow() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); //lldebugs << "LLBufferStreamBuf::underflow()" << llendl; if(!mBuffer) { return EOF; } + LLMutexLock lock(mBuffer->getMutex()); LLBufferArray::segment_iterator_t iter; LLBufferArray::segment_iterator_t end = mBuffer->endSegment(); U8* last_pos = (U8*)gptr(); @@ -127,7 +125,6 @@ int LLBufferStreamBuf::underflow() // virtual int LLBufferStreamBuf::overflow(int c) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); if(!mBuffer) { return EOF; @@ -149,6 +146,7 @@ int LLBufferStreamBuf::overflow(int c) // since we got here, we have a buffer, and we have a character to // put on it. LLBufferArray::segment_iterator_t it; + LLMutexLock lock(mBuffer->getMutex()); it = mBuffer->makeSegment(mChannels.out(), DEFAULT_OUTPUT_SEGMENT_SIZE); if(it != mBuffer->endSegment()) { @@ -166,7 +164,6 @@ int LLBufferStreamBuf::overflow(int c) // virtual int LLBufferStreamBuf::sync() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); int return_value = -1; if(!mBuffer) { @@ -210,6 +207,7 @@ int LLBufferStreamBuf::sync() // *NOTE: I bet we could just --address if address is not NULL. // Need to think about that. + LLMutexLock lock(mBuffer->getMutex()); address = mBuffer->seek(mChannels.out(), address, -1); if(address) { @@ -247,7 +245,6 @@ streampos LLBufferStreamBuf::seekoff( std::ios::openmode which) #endif { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); if(!mBuffer || ((way == std::ios::beg) && (off < 0)) || ((way == std::ios::end) && (off > 0))) @@ -273,6 +270,8 @@ streampos LLBufferStreamBuf::seekoff( // NULL is fine break; } + + LLMutexLock lock(mBuffer->getMutex()); address = mBuffer->seek(mChannels.in(), base_addr, off); if(address) { @@ -304,6 +303,8 @@ streampos LLBufferStreamBuf::seekoff( // NULL is fine break; } + + LLMutexLock lock(mBuffer->getMutex()); address = mBuffer->seek(mChannels.out(), base_addr, off); if(address) { @@ -335,10 +336,8 @@ LLBufferStream::LLBufferStream( std::iostream(&mStreamBuf), mStreamBuf(channels, buffer) { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } LLBufferStream::~LLBufferStream() { - LLMemType m1(LLMemType::MTYPE_IO_BUFFER); } diff --git a/indra/llmessage/llcachename.cpp b/indra/llmessage/llcachename.cpp index 479efabb5f..3fb36eecf0 100644 --- a/indra/llmessage/llcachename.cpp +++ b/indra/llmessage/llcachename.cpp @@ -36,7 +36,6 @@ #include "llsdserialize.h" #include "lluuid.h" #include "message.h" -#include "llmemtype.h" #include <boost/regex.hpp> @@ -524,6 +523,7 @@ std::string LLCacheName::cleanFullName(const std::string& full_name) } //static +// Transform hard-coded name provided by server to a more legible username std::string LLCacheName::buildUsername(const std::string& full_name) { // rare, but handle hard-coded error names returned from server @@ -549,8 +549,9 @@ std::string LLCacheName::buildUsername(const std::string& full_name) return username; } - // if the input wasn't a correctly formatted legacy name just return it unchanged - return full_name; + // if the input wasn't a correctly formatted legacy name, just return it + // cleaned up from a potential terminal "Resident" + return cleanFullName(full_name); } //static @@ -663,7 +664,6 @@ boost::signals2::connection LLCacheName::get(const LLUUID& id, bool is_group, ol void LLCacheName::processPending() { - LLMemType mt_pp(LLMemType::MTYPE_CACHE_PROCESS_PENDING); const F32 SECS_BETWEEN_PROCESS = 0.1f; if(!impl.mProcessTimer.checkExpirationAndReset(SECS_BETWEEN_PROCESS)) { @@ -769,7 +769,6 @@ std::string LLCacheName::getDefaultLastName() void LLCacheName::Impl::processPendingAsks() { - LLMemType mt_ppa(LLMemType::MTYPE_CACHE_PROCESS_PENDING_ASKS); sendRequest(_PREHASH_UUIDNameRequest, mAskNameQueue); sendRequest(_PREHASH_UUIDGroupNameRequest, mAskGroupQueue); mAskNameQueue.clear(); @@ -778,7 +777,6 @@ void LLCacheName::Impl::processPendingAsks() void LLCacheName::Impl::processPendingReplies() { - LLMemType mt_ppr(LLMemType::MTYPE_CACHE_PROCESS_PENDING_REPLIES); // First call all the callbacks, because they might send messages. for(ReplyQueue::iterator it = mReplyQueue.begin(); it != mReplyQueue.end(); ++it) { diff --git a/indra/llmessage/llcachename.h b/indra/llmessage/llcachename.h index b108e37157..d238c3a247 100644 --- a/indra/llmessage/llcachename.h +++ b/indra/llmessage/llcachename.h @@ -40,7 +40,7 @@ typedef boost::signals2::signal<void (const LLUUID& id, bool is_group)> LLCacheNameSignal; typedef LLCacheNameSignal::slot_type LLCacheNameCallback; -// Old callback with user data for compatability +// Old callback with user data for compatibility typedef void (*old_callback_t)(const LLUUID&, const std::string&, bool, void*); // Here's the theory: diff --git a/indra/llmessage/llcircuit.cpp b/indra/llmessage/llcircuit.cpp index e0410906fb..0c2d4b823d 100644 --- a/indra/llmessage/llcircuit.cpp +++ b/indra/llmessage/llcircuit.cpp @@ -679,7 +679,6 @@ void LLCircuitData::checkPacketInID(TPACKETID id, BOOL receive_resent) setPacketInID((id + 1) % LL_MAX_OUT_PACKET_ID); mLastPacketGap = 0; - mOutOfOrderRate.count(0); return; } @@ -775,7 +774,6 @@ void LLCircuitData::checkPacketInID(TPACKETID id, BOOL receive_resent) } } - mOutOfOrderRate.count(gap); mLastPacketGap = gap; } diff --git a/indra/llmessage/llcircuit.h b/indra/llmessage/llcircuit.h index d1c400c6a2..430d6358f7 100644 --- a/indra/llmessage/llcircuit.h +++ b/indra/llmessage/llcircuit.h @@ -40,7 +40,6 @@ #include "llpacketack.h" #include "lluuid.h" #include "llthrottle.h" -#include "llstat.h" // // Constants @@ -126,8 +125,6 @@ public: S32 getUnackedPacketCount() const { return mUnackedPacketCount; } S32 getUnackedPacketBytes() const { return mUnackedPacketBytes; } F64 getNextPingSendTime() const { return mNextPingSendTime; } - F32 getOutOfOrderRate(LLStatAccum::TimeScale scale = LLStatAccum::SCALE_MINUTE) - { return mOutOfOrderRate.meanValue(scale); } U32 getLastPacketGap() const { return mLastPacketGap; } LLHost getHost() const { return mHost; } F64 getLastPacketInTime() const { return mLastPacketInTime; } @@ -275,7 +272,6 @@ protected: LLTimer mExistenceTimer; // initialized when circuit created, used to track bandwidth numbers S32 mCurrentResendCount; // Number of resent packets since last spam - LLStatRate mOutOfOrderRate; // Rate of out of order packets coming in. U32 mLastPacketGap; // Gap in sequence number of last packet. const F32 mHeartbeatInterval; diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index 330028c926..8ffa8e4271 100644 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -72,10 +72,9 @@ static const U32 EASY_HANDLE_POOL_SIZE = 5; static const S32 MULTI_PERFORM_CALL_REPEAT = 5; -static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds +static const S32 CURL_REQUEST_TIMEOUT = 30; // seconds per operation static const S32 MAX_ACTIVE_REQUEST_COUNT = 100; -static // DEBUG // S32 gCurlEasyCount = 0; S32 gCurlMultiCount = 0; @@ -86,9 +85,12 @@ S32 gCurlMultiCount = 0; std::vector<LLMutex*> LLCurl::sSSLMutex; std::string LLCurl::sCAPath; std::string LLCurl::sCAFile; - -bool LLCurl::sMultiThreaded = false; -static U32 sMainThreadID = 0; +LLCurlThread* LLCurl::sCurlThread = NULL ; +LLMutex* LLCurl::sHandleMutexp = NULL ; +S32 LLCurl::sTotalHandles = 0 ; +bool LLCurl::sNotQuitting = true; +F32 LLCurl::sCurlRequestTimeOut = 120.f; //seonds +S32 LLCurl::sMaxHandles = 256; //max number of handles, (multi handles and easy handles combined). void check_curl_code(CURLcode code) { @@ -131,12 +133,12 @@ std::string LLCurl::getVersionString() ////////////////////////////////////////////////////////////////////////////// LLCurl::Responder::Responder() - : mReferenceCount(0) { } LLCurl::Responder::~Responder() { + LL_CHECK_MEMORY } // virtual @@ -200,38 +202,24 @@ void LLCurl::Responder::completedHeader(U32 status, const std::string& reason, c } -namespace boost -{ - void intrusive_ptr_add_ref(LLCurl::Responder* p) - { - ++p->mReferenceCount; - } - - void intrusive_ptr_release(LLCurl::Responder* p) - { - if (p && 0 == --p->mReferenceCount) - { - delete p; - } - } -}; - - ////////////////////////////////////////////////////////////////////////////// std::set<CURL*> LLCurl::Easy::sFreeHandles; std::set<CURL*> LLCurl::Easy::sActiveHandles; -LLMutex* LLCurl::Easy::sHandleMutex = NULL; -LLMutex* LLCurl::Easy::sMultiMutex = NULL; +LLMutex* LLCurl::Easy::sHandleMutexp = NULL ; //static CURL* LLCurl::Easy::allocEasyHandle() { + llassert(LLCurl::getCurlThread()) ; + CURL* ret = NULL; - LLMutexLock lock(sHandleMutex); + + LLMutexLock lock(sHandleMutexp) ; + if (sFreeHandles.empty()) { - ret = curl_easy_init(); + ret = LLCurl::newEasyHandle(); } else { @@ -251,17 +239,30 @@ CURL* LLCurl::Easy::allocEasyHandle() //static void LLCurl::Easy::releaseEasyHandle(CURL* handle) { + static const S32 MAX_NUM_FREE_HANDLES = 32 ; + if (!handle) { - llerrs << "handle cannot be NULL!" << llendl; + return ; //handle allocation failed. + //llerrs << "handle cannot be NULL!" << llendl; } - LLMutexLock lock(sHandleMutex); - + LLMutexLock lock(sHandleMutexp) ; if (sActiveHandles.find(handle) != sActiveHandles.end()) { + LL_CHECK_MEMORY sActiveHandles.erase(handle); - sFreeHandles.insert(handle); + LL_CHECK_MEMORY + if(sFreeHandles.size() < MAX_NUM_FREE_HANDLES) + { + sFreeHandles.insert(handle); + LL_CHECK_MEMORY + } + else + { + LLCurl::deleteEasyHandle(handle) ; + LL_CHECK_MEMORY + } } else { @@ -293,6 +294,8 @@ LLCurl::Easy* LLCurl::Easy::getEasy() // multi handles cache if they are added to one. CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); check_curl_code(result); + result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + check_curl_code(result); ++gCurlEasyCount; return easy; @@ -303,7 +306,17 @@ LLCurl::Easy::~Easy() releaseEasyHandle(mCurlEasyHandle); --gCurlEasyCount; curl_slist_free_all(mHeaders); + LL_CHECK_MEMORY for_each(mStrings.begin(), mStrings.end(), DeletePointerArray()); + LL_CHECK_MEMORY + if (mResponder && LLCurl::sNotQuitting) //aborted + { + std::string reason("Request timeout, aborted.") ; + mResponder->completedRaw(408, //HTTP_REQUEST_TIME_OUT, timeout, abort + reason, mChannels, mOutput); + LL_CHECK_MEMORY + } + mResponder = NULL; } void LLCurl::Easy::resetState() @@ -430,9 +443,9 @@ size_t curlReadCallback(char* data, size_t size, size_t nmemb, void* user_data) LLCurl::Easy* easy = (LLCurl::Easy*)user_data; S32 n = size * nmemb; - S32 startpos = easy->getInput().tellg(); + S32 startpos = (S32)easy->getInput().tellg(); easy->getInput().seekg(0, std::ios::end); - S32 endpos = easy->getInput().tellg(); + S32 endpos = (S32)easy->getInput().tellg(); easy->getInput().seekg(startpos, std::ios::beg); S32 maxn = endpos - startpos; n = llmin(n, maxn); @@ -471,11 +484,13 @@ void LLCurl::Easy::prepRequest(const std::string& url, //setopt(CURLOPT_VERBOSE, 1); // useful for debugging setopt(CURLOPT_NOSIGNAL, 1); - + setopt(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + // Set the CURL options for either Socks or HTTP proxy LLProxy::getInstance()->applyProxySettings(this); mOutput.reset(new LLBufferArray); + mOutput->setThreaded(true); setopt(CURLOPT_WRITEFUNCTION, (void*)&curlWriteCallback); setopt(CURLOPT_WRITEDATA, (void*)this); @@ -519,133 +534,261 @@ void LLCurl::Easy::prepRequest(const std::string& url, } //////////////////////////////////////////////////////////////////////////// - -LLCurl::Multi::Multi() - : LLThread("Curl Multi"), - mQueued(0), +LLCurl::Multi::Multi(F32 idle_time_out) + : mQueued(0), mErrorCount(0), - mPerformState(PERFORM_STATE_READY) -{ - mQuitting = false; - - mThreaded = LLCurl::sMultiThreaded && LLThread::currentID() == sMainThreadID; - if (mThreaded) - { - mSignal = new LLCondition(NULL); - } - else - { - mSignal = NULL; - } - - mCurlMultiHandle = curl_multi_init(); + mState(STATE_READY), + mDead(FALSE), + mValid(TRUE), + mMutexp(NULL), + mDeletionMutexp(NULL), + mEasyMutexp(NULL) +{ + mCurlMultiHandle = LLCurl::newMultiHandle(); if (!mCurlMultiHandle) { llwarns << "curl_multi_init() returned NULL! Easy handles: " << gCurlEasyCount << " Multi handles: " << gCurlMultiCount << llendl; - mCurlMultiHandle = curl_multi_init(); + mCurlMultiHandle = LLCurl::newMultiHandle(); } - llassert_always(mCurlMultiHandle); + //llassert_always(mCurlMultiHandle); + + if(mCurlMultiHandle) + { + if(LLCurl::getCurlThread()->getThreaded()) + { + mMutexp = new LLMutex(NULL) ; + mDeletionMutexp = new LLMutex(NULL) ; + mEasyMutexp = new LLMutex(NULL) ; + } + LLCurl::getCurlThread()->addMulti(this) ; + + mIdleTimeOut = idle_time_out ; + if(mIdleTimeOut < LLCurl::sCurlRequestTimeOut) + { + mIdleTimeOut = LLCurl::sCurlRequestTimeOut ; + } + ++gCurlMultiCount; } +} LLCurl::Multi::~Multi() { - llassert(isStopped()); + cleanup(true) ; + + delete mDeletionMutexp ; + mDeletionMutexp = NULL ; +} - if (LLCurl::sMultiThreaded) +void LLCurl::Multi::cleanup(bool deleted) +{ + if(!mCurlMultiHandle) { - LLCurl::Easy::sMultiMutex->lock(); + return ; //nothing to clean. } + llassert_always(deleted || !mValid) ; + + LLMutexLock lock(mDeletionMutexp); - delete mSignal; - mSignal = NULL; // Clean up active for(easy_active_list_t::iterator iter = mEasyActiveList.begin(); iter != mEasyActiveList.end(); ++iter) { Easy* easy = *iter; + LL_CHECK_MEMORY check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); + LL_CHECK_MEMORY + if(deleted) + { + easy->mResponder = NULL ; //avoid triggering mResponder. + LL_CHECK_MEMORY + } delete easy; + LL_CHECK_MEMORY } mEasyActiveList.clear(); mEasyActiveMap.clear(); - // Clean up freed + LL_CHECK_MEMORY + + // Clean up freed for_each(mEasyFreeList.begin(), mEasyFreeList.end(), DeletePointer()); mEasyFreeList.clear(); + + LL_CHECK_MEMORY + + check_curl_multi_code(LLCurl::deleteMultiHandle(mCurlMultiHandle)); + mCurlMultiHandle = NULL ; + + LL_CHECK_MEMORY + + delete mMutexp ; + mMutexp = NULL ; + + LL_CHECK_MEMORY + + delete mEasyMutexp ; + mEasyMutexp = NULL ; + + LL_CHECK_MEMORY - check_curl_multi_code(curl_multi_cleanup(mCurlMultiHandle)); + mQueued = 0 ; + mState = STATE_COMPLETED; + --gCurlMultiCount; - if (LLCurl::sMultiThreaded) + return ; +} + +void LLCurl::Multi::lock() +{ + if(mMutexp) { - LLCurl::Easy::sMultiMutex->unlock(); + mMutexp->lock() ; } } -CURLMsg* LLCurl::Multi::info_read(S32* msgs_in_queue) +void LLCurl::Multi::unlock() { - CURLMsg* curlmsg = curl_multi_info_read(mCurlMultiHandle, msgs_in_queue); - return curlmsg; + if(mMutexp) + { + mMutexp->unlock() ; + } } -void LLCurl::Multi::perform() +void LLCurl::Multi::markDead() { - if (mThreaded) { - if (mPerformState == PERFORM_STATE_READY) + LLMutexLock lock(mDeletionMutexp) ; + + if(mCurlMultiHandle != NULL) { - mSignal->signal(); + mDead = TRUE ; + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_URGENT) ; + + return; } } - else + + //not valid, delete it. + delete this; +} + +void LLCurl::Multi::setState(LLCurl::Multi::ePerformState state) +{ + lock() ; + mState = state ; + unlock() ; + + if(mState == STATE_READY) { - doPerform(); - } + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_NORMAL) ; + } } -void LLCurl::Multi::run() +LLCurl::Multi::ePerformState LLCurl::Multi::getState() { - llassert(mThreaded); + return mState; +} + +bool LLCurl::Multi::isCompleted() +{ + return STATE_COMPLETED == getState() ; +} - while (!mQuitting) +bool LLCurl::Multi::waitToComplete() +{ + if(!isValid()) { - mSignal->wait(); - mPerformState = PERFORM_STATE_PERFORMING; - if (!mQuitting) - { - LLMutexLock lock(LLCurl::Easy::sMultiMutex); - doPerform(); - } + return true ; + } + + if(!mMutexp) //not threaded + { + doPerform() ; + return true ; + } + + bool completed = (STATE_COMPLETED == mState) ; + if(!completed) + { + LLCurl::getCurlThread()->setPriority(mHandle, LLQueuedThread::PRIORITY_HIGH) ; } + + return completed; +} + +CURLMsg* LLCurl::Multi::info_read(S32* msgs_in_queue) +{ + LLMutexLock lock(mMutexp) ; + + CURLMsg* curlmsg = curl_multi_info_read(mCurlMultiHandle, msgs_in_queue); + return curlmsg; } -void LLCurl::Multi::doPerform() +//return true if dead +bool LLCurl::Multi::doPerform() { - S32 q = 0; - for (S32 call_count = 0; - call_count < MULTI_PERFORM_CALL_REPEAT; - call_count += 1) + LLMutexLock lock(mDeletionMutexp) ; + + bool dead = mDead ; + + if(mDead) { - CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); - if (CURLM_CALL_MULTI_PERFORM != code || q == 0) + setState(STATE_COMPLETED); + mQueued = 0 ; + } + else if(getState() != STATE_COMPLETED) + { + setState(STATE_PERFORMING); + + S32 q = 0; + for (S32 call_count = 0; + call_count < MULTI_PERFORM_CALL_REPEAT; + call_count++) { - check_curl_multi_code(code); - break; + LLMutexLock lock(mMutexp) ; + + //WARNING: curl_multi_perform will block for many hundreds of milliseconds + // NEVER call this from the main thread, and NEVER allow the main thread to + // wait on a mutex held by this thread while curl_multi_perform is executing + CURLMcode code = curl_multi_perform(mCurlMultiHandle, &q); + if (CURLM_CALL_MULTI_PERFORM != code || q == 0) + { + check_curl_multi_code(code); + + break; + } } - + + mQueued = q; + setState(STATE_COMPLETED) ; + mIdleTimer.reset() ; + } + else if(!mValid && mIdleTimer.getElapsedTimeF32() > mIdleTimeOut) //idle for too long, remove it. + { + dead = true ; } - mQueued = q; - mPerformState = PERFORM_STATE_COMPLETED; + else if(mValid && mIdleTimer.getElapsedTimeF32() > mIdleTimeOut - 1.f) //idle for too long, mark it invalid. + { + mValid = FALSE ; + } + + return dead ; } S32 LLCurl::Multi::process() { - perform(); + if(!isValid()) + { + return 0 ; + } + + waitToComplete() ; - if (mPerformState != PERFORM_STATE_COMPLETED) + if (getState() != STATE_COMPLETED) { return 0; } @@ -660,10 +803,19 @@ S32 LLCurl::Multi::process() if (msg->msg == CURLMSG_DONE) { U32 response = 0; - easy_active_map_t::iterator iter = mEasyActiveMap.find(msg->easy_handle); - if (iter != mEasyActiveMap.end()) + Easy* easy = NULL ; + + { + LLMutexLock lock(mEasyMutexp) ; + easy_active_map_t::iterator iter = mEasyActiveMap.find(msg->easy_handle); + if (iter != mEasyActiveMap.end()) + { + easy = iter->second; + } + } + + if(easy) { - Easy* easy = iter->second; response = easy->report(msg->data.result); removeEasy(easy); } @@ -681,25 +833,28 @@ S32 LLCurl::Multi::process() } } - mPerformState = PERFORM_STATE_READY; + setState(STATE_READY); + return processed; } LLCurl::Easy* LLCurl::Multi::allocEasy() { - Easy* easy = 0; + Easy* easy = 0; if (mEasyFreeList.empty()) - { + { easy = Easy::getEasy(); } else { + LLMutexLock lock(mEasyMutexp) ; easy = *(mEasyFreeList.begin()); mEasyFreeList.erase(easy); } if (easy) { + LLMutexLock lock(mEasyMutexp) ; mEasyActiveList.insert(easy); mEasyActiveMap[easy->getCurlHandle()] = easy; } @@ -708,6 +863,7 @@ LLCurl::Easy* LLCurl::Multi::allocEasy() bool LLCurl::Multi::addEasy(Easy* easy) { + LLMutexLock lock(mMutexp) ; CURLMcode mcode = curl_multi_add_handle(mCurlMultiHandle, easy->getCurlHandle()); check_curl_multi_code(mcode); //if (mcode != CURLM_OK) @@ -720,25 +876,154 @@ bool LLCurl::Multi::addEasy(Easy* easy) void LLCurl::Multi::easyFree(Easy* easy) { + if(mEasyMutexp) + { + mEasyMutexp->lock() ; + } + mEasyActiveList.erase(easy); mEasyActiveMap.erase(easy->getCurlHandle()); + if (mEasyFreeList.size() < EASY_HANDLE_POOL_SIZE) - { - easy->resetState(); + { mEasyFreeList.insert(easy); + + if(mEasyMutexp) + { + mEasyMutexp->unlock() ; + } + + easy->resetState(); } else { + if(mEasyMutexp) + { + mEasyMutexp->unlock() ; + } delete easy; } } void LLCurl::Multi::removeEasy(Easy* easy) { - check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); + { + LLMutexLock lock(mMutexp) ; + check_curl_multi_code(curl_multi_remove_handle(mCurlMultiHandle, easy->getCurlHandle())); + } easyFree(easy); } +//------------------------------------------------------------ +//LLCurlThread +LLCurlThread::CurlRequest::CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread) : + LLQueuedThread::QueuedRequest(handle, LLQueuedThread::PRIORITY_NORMAL, FLAG_AUTO_COMPLETE), + mMulti(multi), + mCurlThread(curl_thread) +{ +} + +LLCurlThread::CurlRequest::~CurlRequest() +{ + if(mMulti) + { + mCurlThread->deleteMulti(mMulti) ; + mMulti = NULL ; + } +} + +bool LLCurlThread::CurlRequest::processRequest() +{ + bool completed = true ; + if(mMulti) + { + completed = mCurlThread->doMultiPerform(mMulti) ; + + if(!completed) + { + setPriority(LLQueuedThread::PRIORITY_LOW) ; + } + } + + return completed ; +} + +void LLCurlThread::CurlRequest::finishRequest(bool completed) +{ + if(mMulti->isDead()) + { + mCurlThread->deleteMulti(mMulti) ; + } + else + { + mCurlThread->cleanupMulti(mMulti) ; //being idle too long, remove the request. + } + + mMulti = NULL ; +} + +LLCurlThread::LLCurlThread(bool threaded) : + LLQueuedThread("curlthread", threaded) +{ +} + +//virtual +LLCurlThread::~LLCurlThread() +{ +} + +S32 LLCurlThread::update(F32 max_time_ms) +{ + return LLQueuedThread::update(max_time_ms); +} + +void LLCurlThread::addMulti(LLCurl::Multi* multi) +{ + multi->mHandle = generateHandle() ; + + CurlRequest* req = new CurlRequest(multi->mHandle, multi, this) ; + + if (!addRequest(req)) + { + llwarns << "curl request added when the thread is quitted" << llendl; + } +} + +void LLCurlThread::killMulti(LLCurl::Multi* multi) +{ + if(!multi) + { + return ; + } + + + multi->markDead() ; +} + +//private +bool LLCurlThread::doMultiPerform(LLCurl::Multi* multi) +{ + return multi->doPerform() ; +} + +//private +void LLCurlThread::deleteMulti(LLCurl::Multi* multi) +{ + delete multi ; +} + +//private +void LLCurlThread::cleanupMulti(LLCurl::Multi* multi) +{ + multi->cleanup() ; + if(multi->isDead()) //check if marked dead during cleaning up. + { + deleteMulti(multi) ; + } +} + +//------------------------------------------------------------ + //static std::string LLCurl::strerror(CURLcode errorcode) { @@ -753,39 +1038,30 @@ LLCurlRequest::LLCurlRequest() : mActiveMulti(NULL), mActiveRequestCount(0) { - mThreadID = LLThread::currentID(); mProcessing = FALSE; } LLCurlRequest::~LLCurlRequest() { - llassert_always(mThreadID == LLThread::currentID()); - //stop all Multi handle background threads for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ++iter) { - LLCurl::Multi* multi = *iter; - multi->mQuitting = true; - if (multi->mThreaded) - { - while (!multi->isStopped()) - { - multi->mSignal->signal(); - apr_sleep(1000); - } - } + LLCurl::getCurlThread()->killMulti(*iter) ; } - for_each(mMultiSet.begin(), mMultiSet.end(), DeletePointer()); + mMultiSet.clear() ; } void LLCurlRequest::addMulti() { - llassert_always(mThreadID == LLThread::currentID()); LLCurl::Multi* multi = new LLCurl::Multi(); - if (multi->mThreaded) + if(!multi->isValid()) { - multi->start(); + LLCurl::getCurlThread()->killMulti(multi) ; + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + return; } + mMultiSet.insert(multi); mActiveMulti = multi; mActiveRequestCount = 0; @@ -799,7 +1075,12 @@ LLCurl::Easy* LLCurlRequest::allocEasy() { addMulti(); } - llassert_always(mActiveMulti); + if(!mActiveMulti) + { + return NULL ; + } + + //llassert_always(mActiveMulti); ++mActiveRequestCount; LLCurl::Easy* easy = mActiveMulti->allocEasy(); return easy; @@ -821,12 +1102,15 @@ void LLCurlRequest::get(const std::string& url, LLCurl::ResponderPtr responder) { getByteRange(url, headers_t(), 0, -1, responder); } - + +// Note: (length==0) is interpreted as "the rest of the file", i.e. the whole file if (offset==0) or +// the remainder of the file if not. bool LLCurlRequest::getByteRange(const std::string& url, const headers_t& headers, S32 offset, S32 length, LLCurl::ResponderPtr responder) { + llassert(LLCurl::sNotQuitting); LLCurl::Easy* easy = allocEasy(); if (!easy) { @@ -839,6 +1123,11 @@ bool LLCurlRequest::getByteRange(const std::string& url, std::string range = llformat("Range: bytes=%d-%d", offset,offset+length-1); easy->slist_append(range.c_str()); } + else if (offset > 0) + { + std::string range = llformat("Range: bytes=%d-", offset); + easy->slist_append(range.c_str()); + } easy->setHeaders(); bool res = addEasy(easy); return res; @@ -849,6 +1138,7 @@ bool LLCurlRequest::post(const std::string& url, const LLSD& data, LLCurl::ResponderPtr responder, S32 time_out) { + llassert(LLCurl::sNotQuitting); LLCurl::Easy* easy = allocEasy(); if (!easy) { @@ -876,6 +1166,7 @@ bool LLCurlRequest::post(const std::string& url, const std::string& data, LLCurl::ResponderPtr responder, S32 time_out) { + llassert(LLCurl::sNotQuitting); LLCurl::Easy* easy = allocEasy(); if (!easy) { @@ -901,7 +1192,6 @@ bool LLCurlRequest::post(const std::string& url, // Note: call once per frame S32 LLCurlRequest::process() { - llassert_always(mThreadID == LLThread::currentID()); S32 res = 0; mProcessing = TRUE; @@ -910,22 +1200,25 @@ S32 LLCurlRequest::process() { curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; + + if(!multi->isValid()) + { + if(multi == mActiveMulti) + { + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + } + mMultiSet.erase(curiter) ; + LLCurl::getCurlThread()->killMulti(multi) ; + continue ; + } + S32 tres = multi->process(); res += tres; if (multi != mActiveMulti && tres == 0 && multi->mQueued == 0) { mMultiSet.erase(curiter); - multi->mQuitting = true; - if (multi->mThreaded) - { - while (!multi->isStopped()) - { - multi->mSignal->signal(); - apr_sleep(1000); - } - } - - delete multi; + LLCurl::getCurlThread()->killMulti(multi); } } mProcessing = FALSE; @@ -934,15 +1227,27 @@ S32 LLCurlRequest::process() S32 LLCurlRequest::getQueued() { - llassert_always(mThreadID == LLThread::currentID()); S32 queued = 0; for (curlmulti_set_t::iterator iter = mMultiSet.begin(); iter != mMultiSet.end(); ) { curlmulti_set_t::iterator curiter = iter++; LLCurl::Multi* multi = *curiter; + + if(!multi->isValid()) + { + if(multi == mActiveMulti) + { + mActiveMulti = NULL ; + mActiveRequestCount = 0 ; + } + LLCurl::getCurlThread()->killMulti(multi); + mMultiSet.erase(curiter) ; + continue ; + } + queued += multi->mQueued; - if (multi->mPerformState != LLCurl::Multi::PERFORM_STATE_READY) + if (multi->getState() != LLCurl::Multi::STATE_READY) { ++queued; } @@ -950,6 +1255,208 @@ S32 LLCurlRequest::getQueued() return queued; } +LLCurlTextureRequest::LLCurlTextureRequest(S32 concurrency) : + LLCurlRequest(), + mConcurrency(concurrency), + mInQueue(0), + mMutex(NULL), + mHandleCounter(1), + mTotalIssuedRequests(0), + mTotalReceivedBits(0) +{ + mGlobalTimer.reset(); +} + +LLCurlTextureRequest::~LLCurlTextureRequest() +{ + mRequestMap.clear(); + + for(req_queue_t::iterator iter = mCachedRequests.begin(); iter != mCachedRequests.end(); ++iter) + { + delete *iter; + } + mCachedRequests.clear(); +} + +//return 0: success +// > 0: cached handle +U32 LLCurlTextureRequest::getByteRange(const std::string& url, + const headers_t& headers, + S32 offset, S32 length, U32 pri, + LLCurl::ResponderPtr responder, F32 delay_time) +{ + U32 ret_val = 0; + bool success = false; + + if(mInQueue < mConcurrency && delay_time < 0.f) + { + success = LLCurlRequest::getByteRange(url, headers, offset, length, responder); + } + + LLMutexLock lock(&mMutex); + + if(success) + { + mInQueue++; + mTotalIssuedRequests++; + } + else + { + request_t* request = new request_t(mHandleCounter, url, headers, offset, length, pri, responder); + if(delay_time > 0.f) + { + request->mStartTime = mGlobalTimer.getElapsedTimeF32() + delay_time; + } + + mCachedRequests.insert(request); + mRequestMap[mHandleCounter] = request; + ret_val = mHandleCounter; + mHandleCounter++; + + if(!mHandleCounter) + { + mHandleCounter = 1; + } + } + + return ret_val; +} + +void LLCurlTextureRequest::completeRequest(S32 received_bytes) +{ + LLMutexLock lock(&mMutex); + + llassert_always(mInQueue > 0); + + mInQueue--; + mTotalReceivedBits += received_bytes * 8; +} + +void LLCurlTextureRequest::nextRequests() +{ + if(mCachedRequests.empty() || mInQueue >= mConcurrency) + { + return; + } + + F32 cur_time = mGlobalTimer.getElapsedTimeF32(); + + req_queue_t::iterator iter; + { + LLMutexLock lock(&mMutex); + iter = mCachedRequests.begin(); + } + while(1) + { + request_t* request = *iter; + if(request->mStartTime < cur_time) + { + if(!LLCurlRequest::getByteRange(request->mUrl, request->mHeaders, request->mOffset, request->mLength, request->mResponder)) + { + break; + } + + LLMutexLock lock(&mMutex); + ++iter; + mInQueue++; + mTotalIssuedRequests++; + mCachedRequests.erase(request); + mRequestMap.erase(request->mHandle); + delete request; + + if(iter == mCachedRequests.end() || mInQueue >= mConcurrency) + { + break; + } + } + else + { + LLMutexLock lock(&mMutex); + ++iter; + if(iter == mCachedRequests.end() || mInQueue >= mConcurrency) + { + break; + } + } + } + + return; +} + +void LLCurlTextureRequest::updatePriority(U32 handle, U32 pri) +{ + if(!handle) + { + return; + } + + LLMutexLock lock(&mMutex); + + std::map<S32, request_t*>::iterator iter = mRequestMap.find(handle); + if(iter != mRequestMap.end()) + { + request_t* req = iter->second; + + if(req->mPriority != pri) + { + mCachedRequests.erase(req); + req->mPriority = pri; + mCachedRequests.insert(req); + } + } +} + +void LLCurlTextureRequest::removeRequest(U32 handle) +{ + if(!handle) + { + return; + } + + LLMutexLock lock(&mMutex); + + std::map<S32, request_t*>::iterator iter = mRequestMap.find(handle); + if(iter != mRequestMap.end()) + { + request_t* req = iter->second; + mRequestMap.erase(iter); + mCachedRequests.erase(req); + delete req; + } +} + +bool LLCurlTextureRequest::isWaiting(U32 handle) +{ + if(!handle) + { + return false; + } + + LLMutexLock lock(&mMutex); + return mRequestMap.find(handle) != mRequestMap.end(); +} + +U32 LLCurlTextureRequest::getTotalReceivedBits() +{ + LLMutexLock lock(&mMutex); + + U32 bits = mTotalReceivedBits; + mTotalReceivedBits = 0; + return bits; +} + +U32 LLCurlTextureRequest::getTotalIssuedRequests() +{ + LLMutexLock lock(&mMutex); + return mTotalIssuedRequests; +} + +S32 LLCurlTextureRequest::getNumRequests() +{ + LLMutexLock lock(&mMutex); + return mInQueue; +} + //////////////////////////////////////////////////////////////////////////// // For generating one easy request // associated with a single multi request @@ -959,10 +1466,9 @@ LLCurlEasyRequest::LLCurlEasyRequest() mResultReturned(false) { mMulti = new LLCurl::Multi(); - if (mMulti->mThreaded) + + if(mMulti->isValid()) { - mMulti->start(); - } mEasy = mMulti->allocEasy(); if (mEasy) { @@ -972,24 +1478,22 @@ LLCurlEasyRequest::LLCurlEasyRequest() LLProxy::getInstance()->applyProxySettings(mEasy); } } + else + { + LLCurl::getCurlThread()->killMulti(mMulti) ; + mEasy = NULL ; + mMulti = NULL ; + } +} LLCurlEasyRequest::~LLCurlEasyRequest() { - mMulti->mQuitting = true; - if (mMulti->mThreaded) - { - while (!mMulti->isStopped()) - { - mMulti->mSignal->signal(); - apr_sleep(1000); - } - } - delete mMulti; + LLCurl::getCurlThread()->killMulti(mMulti) ; } void LLCurlEasyRequest::setopt(CURLoption option, S32 value) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(option, value); } @@ -997,7 +1501,7 @@ void LLCurlEasyRequest::setopt(CURLoption option, S32 value) void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setoptString(option, value); } @@ -1005,7 +1509,7 @@ void LLCurlEasyRequest::setoptString(CURLoption option, const std::string& value void LLCurlEasyRequest::setPost(char* postdata, S32 size) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_POST, 1); mEasy->setopt(CURLOPT_POSTFIELDS, postdata); @@ -1015,7 +1519,7 @@ void LLCurlEasyRequest::setPost(char* postdata, S32 size) void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_HEADERFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_HEADERDATA, userdata); // aka CURLOPT_WRITEHEADER @@ -1024,7 +1528,7 @@ void LLCurlEasyRequest::setHeaderCallback(curl_header_callback callback, void* u void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_WRITEFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_WRITEDATA, userdata); @@ -1033,7 +1537,7 @@ void LLCurlEasyRequest::setWriteCallback(curl_write_callback callback, void* use void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_READFUNCTION, (void*)callback); mEasy->setopt(CURLOPT_READDATA, userdata); @@ -1042,7 +1546,7 @@ void LLCurlEasyRequest::setReadCallback(curl_read_callback callback, void* userd void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* userdata) { - if (mEasy) + if (isValid() && mEasy) { mEasy->setopt(CURLOPT_SSL_CTX_FUNCTION, (void*)callback); mEasy->setopt(CURLOPT_SSL_CTX_DATA, userdata); @@ -1051,7 +1555,7 @@ void LLCurlEasyRequest::setSSLCtxCallback(curl_ssl_ctx_callback callback, void* void LLCurlEasyRequest::slist_append(const char* str) { - if (mEasy) + if (isValid() && mEasy) { mEasy->slist_append(str); } @@ -1062,7 +1566,7 @@ void LLCurlEasyRequest::sendRequest(const std::string& url) llassert_always(!mRequestSent); mRequestSent = true; lldebugs << url << llendl; - if (mEasy) + if (isValid() && mEasy) { mEasy->setHeaders(); mEasy->setoptString(CURLOPT_URL, url); @@ -1074,25 +1578,24 @@ void LLCurlEasyRequest::requestComplete() { llassert_always(mRequestSent); mRequestSent = false; - if (mEasy) + if (isValid() && mEasy) { mMulti->removeEasy(mEasy); } } -void LLCurlEasyRequest::perform() -{ - mMulti->perform(); -} - // Usage: Call getRestult until it returns false (no more messages) bool LLCurlEasyRequest::getResult(CURLcode* result, LLCurl::TransferInfo* info) { - if (mMulti->mPerformState != LLCurl::Multi::PERFORM_STATE_COMPLETED) + if(!isValid()) + { + return false ; + } + if (!mMulti->isCompleted()) { //we're busy, try again later return false; } - mMulti->mPerformState = LLCurl::Multi::PERFORM_STATE_READY; + mMulti->setState(LLCurl::Multi::STATE_READY) ; if (!mEasy) { @@ -1152,7 +1655,7 @@ CURLMsg* LLCurlEasyRequest::info_read(S32* q, LLCurl::TransferInfo* info) std::string LLCurlEasyRequest::getErrorString() { - return mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); + return isValid() && mEasy ? std::string(mEasy->getErrorBuffer()) : std::string(); } //////////////////////////////////////////////////////////////////////////// @@ -1178,10 +1681,11 @@ unsigned long LLCurl::ssl_thread_id(void) } #endif -void LLCurl::initClass(bool multi_threaded) +void LLCurl::initClass(F32 curl_reuest_timeout, S32 max_number_handles, bool multi_threaded) { - sMainThreadID = LLThread::currentID(); - sMultiThreaded = multi_threaded; + sCurlRequestTimeOut = curl_reuest_timeout ; //seconds + sMaxHandles = max_number_handles ; //max number of handles, (multi handles and easy handles combined). + // Do not change this "unless you are familiar with and mean to control // internal operations of libcurl" // - http://curl.haxx.se/libcurl/c/curl_global_init.html @@ -1189,9 +1693,6 @@ void LLCurl::initClass(bool multi_threaded) check_curl_code(code); - Easy::sHandleMutex = new LLMutex(NULL); - Easy::sMultiMutex = new LLMutex(NULL); - #if SAFE_SSL S32 mutex_count = CRYPTO_num_locks(); for (S32 i=0; i<mutex_count; i++) @@ -1201,29 +1702,135 @@ void LLCurl::initClass(bool multi_threaded) CRYPTO_set_id_callback(&LLCurl::ssl_thread_id); CRYPTO_set_locking_callback(&LLCurl::ssl_locking_callback); #endif + + sCurlThread = new LLCurlThread(multi_threaded) ; + if(multi_threaded) + { + sHandleMutexp = new LLMutex(NULL) ; + Easy::sHandleMutexp = new LLMutex(NULL) ; + } } void LLCurl::cleanupClass() { + sNotQuitting = false; //set quitting + + //shut down curl thread + while(1) + { + if(!sCurlThread->update(1)) //finish all tasks + { + break ; + } + } + LL_CHECK_MEMORY + sCurlThread->shutdown() ; + LL_CHECK_MEMORY + delete sCurlThread ; + sCurlThread = NULL ; + LL_CHECK_MEMORY + #if SAFE_SSL CRYPTO_set_locking_callback(NULL); for_each(sSSLMutex.begin(), sSSLMutex.end(), DeletePointer()); #endif - - delete Easy::sHandleMutex; - Easy::sHandleMutex = NULL; - delete Easy::sMultiMutex; - Easy::sMultiMutex = NULL; + + LL_CHECK_MEMORY for (std::set<CURL*>::iterator iter = Easy::sFreeHandles.begin(); iter != Easy::sFreeHandles.end(); ++iter) { CURL* curl = *iter; - curl_easy_cleanup(curl); + LLCurl::deleteEasyHandle(curl); } + + LL_CHECK_MEMORY Easy::sFreeHandles.clear(); - llassert(Easy::sActiveHandles.empty()); + LL_CHECK_MEMORY + + delete Easy::sHandleMutexp ; + Easy::sHandleMutexp = NULL ; + + LL_CHECK_MEMORY + + delete sHandleMutexp ; + sHandleMutexp = NULL ; + + LL_CHECK_MEMORY + + // removed as per https://jira.secondlife.com/browse/SH-3115 + //llassert(Easy::sActiveHandles.empty()); +} + +//static +CURLM* LLCurl::newMultiHandle() +{ + llassert(sNotQuitting); + + LLMutexLock lock(sHandleMutexp) ; + + if(sTotalHandles + 1 > sMaxHandles) + { + llwarns << "no more handles available." << llendl ; + return NULL ; //failed + } + sTotalHandles++; + + CURLM* ret = curl_multi_init() ; + if(!ret) + { + llwarns << "curl_multi_init failed." << llendl ; + } + + return ret ; +} + +//static +CURLMcode LLCurl::deleteMultiHandle(CURLM* handle) +{ + if(handle) + { + LLMutexLock lock(sHandleMutexp) ; + sTotalHandles-- ; + return curl_multi_cleanup(handle) ; + } + return CURLM_OK ; +} + +//static +CURL* LLCurl::newEasyHandle() +{ + llassert(sNotQuitting); + LLMutexLock lock(sHandleMutexp) ; + + if(sTotalHandles + 1 > sMaxHandles) + { + llwarns << "no more handles available." << llendl ; + return NULL ; //failed + } + sTotalHandles++; + + CURL* ret = curl_easy_init() ; + if(!ret) + { + llwarns << "curl_easy_init failed." << llendl ; + } + + return ret ; +} + +//static +void LLCurl::deleteEasyHandle(CURL* handle) +{ + if(handle) + { + LLMutexLock lock(sHandleMutexp) ; + LL_CHECK_MEMORY + curl_easy_cleanup(handle) ; + LL_CHECK_MEMORY + sTotalHandles-- ; + } } const unsigned int LLCurl::MAX_REDIRECTS = 5; diff --git a/indra/llmessage/llcurl.h b/indra/llmessage/llcurl.h index 87de202717..7bcf61e233 100644 --- a/indra/llmessage/llcurl.h +++ b/indra/llmessage/llcurl.h @@ -42,8 +42,13 @@ #include "lliopipe.h" #include "llsd.h" #include "llthread.h" +#include "llqueuedthread.h" +#include "llframetimer.h" +#include "llpointer.h" + class LLMutex; +class LLCurlThread; // For whatever reason, this is not typedef'd in curl.h typedef size_t (*curl_header_callback)(void *ptr, size_t size, size_t nmemb, void *stream); @@ -56,8 +61,6 @@ public: class Easy; class Multi; - static bool sMultiThreaded; - struct TransferInfo { TransferInfo() : mSizeDownload(0.0), mTotalTime(0.0), mSpeedDownload(0.0) {} @@ -66,7 +69,7 @@ public: F64 mSpeedDownload; }; - class Responder + class Responder : public LLThreadSafeRefCount { //LOG_CLASS(Responder); public: @@ -125,13 +128,10 @@ public: return false; } - public: /* but not really -- don't touch this */ - U32 mReferenceCount; - private: std::string mURL; }; - typedef boost::intrusive_ptr<Responder> ResponderPtr; + typedef LLPointer<Responder> ResponderPtr; /** @@ -162,7 +162,7 @@ public: /** * @ brief Initialize LLCurl class */ - static void initClass(bool multi_threaded = false); + static void initClass(F32 curl_reuest_timeout = 120.f, S32 max_number_handles = 256, bool multi_threaded = false); /** * @ brief Cleanup LLCurl class @@ -181,10 +181,25 @@ public: static void ssl_locking_callback(int mode, int type, const char *file, int line); static unsigned long ssl_thread_id(void); + static LLCurlThread* getCurlThread() { return sCurlThread ;} + + static CURLM* newMultiHandle() ; + static CURLMcode deleteMultiHandle(CURLM* handle) ; + static CURL* newEasyHandle() ; + static void deleteEasyHandle(CURL* handle) ; + private: static std::string sCAPath; static std::string sCAFile; static const unsigned int MAX_REDIRECTS; + static LLCurlThread* sCurlThread; + + static LLMutex* sHandleMutexp ; + static S32 sTotalHandles ; + static S32 sMaxHandles; +public: + static bool sNotQuitting; + static F32 sCurlRequestTimeOut; }; class LLCurl::Easy @@ -216,7 +231,7 @@ public: U32 report(CURLcode); void getTransferInfo(LLCurl::TransferInfo* info); - void prepRequest(const std::string& url, const std::vector<std::string>& headers, ResponderPtr, S32 time_out = 0, bool post = false); + void prepRequest(const std::string& url, const std::vector<std::string>& headers, LLCurl::ResponderPtr, S32 time_out = 0, bool post = false); const char* getErrorBuffer(); @@ -247,69 +262,120 @@ private: // Note: char*'s not strings since we pass pointers to curl std::vector<char*> mStrings; - ResponderPtr mResponder; + LLCurl::ResponderPtr mResponder; static std::set<CURL*> sFreeHandles; static std::set<CURL*> sActiveHandles; - static LLMutex* sHandleMutex; - static LLMutex* sMultiMutex; + static LLMutex* sHandleMutexp ; }; -class LLCurl::Multi : public LLThread +class LLCurl::Multi { LOG_CLASS(Multi); + + friend class LLCurlThread ; + +private: + ~Multi(); + + void markDead() ; + bool doPerform(); + public: typedef enum { - PERFORM_STATE_READY=0, - PERFORM_STATE_PERFORMING=1, - PERFORM_STATE_COMPLETED=2 + STATE_READY=0, + STATE_PERFORMING=1, + STATE_COMPLETED=2 } ePerformState; - Multi(); - ~Multi(); + Multi(F32 idle_time_out = 0.f); + + LLCurl::Easy* allocEasy(); + bool addEasy(LLCurl::Easy* easy); + void removeEasy(LLCurl::Easy* easy); + + void lock() ; + void unlock() ; - Easy* allocEasy(); - bool addEasy(Easy* easy); + void setState(ePerformState state) ; + ePerformState getState() ; - void removeEasy(Easy* easy); + bool isCompleted() ; + bool isValid() {return mCurlMultiHandle != NULL && mValid;} + bool isDead() {return mDead;} + + bool waitToComplete() ; S32 process(); - void perform(); - void doPerform(); - virtual void run(); - CURLMsg* info_read(S32* msgs_in_queue); S32 mQueued; S32 mErrorCount; - S32 mPerformState; - - LLCondition* mSignal; - bool mQuitting; - bool mThreaded; - private: - void easyFree(Easy*); + void easyFree(LLCurl::Easy*); + void cleanup(bool deleted = false) ; CURLM* mCurlMultiHandle; - typedef std::set<Easy*> easy_active_list_t; + typedef std::set<LLCurl::Easy*> easy_active_list_t; easy_active_list_t mEasyActiveList; - typedef std::map<CURL*, Easy*> easy_active_map_t; + typedef std::map<CURL*, LLCurl::Easy*> easy_active_map_t; easy_active_map_t mEasyActiveMap; - typedef std::set<Easy*> easy_free_list_t; + typedef std::set<LLCurl::Easy*> easy_free_list_t; easy_free_list_t mEasyFreeList; + + LLQueuedThread::handle_t mHandle ; + ePerformState mState; + + BOOL mDead ; + BOOL mValid ; + LLMutex* mMutexp ; + LLMutex* mDeletionMutexp ; + LLMutex* mEasyMutexp ; + LLFrameTimer mIdleTimer ; + F32 mIdleTimeOut; }; -namespace boost +class LLCurlThread : public LLQueuedThread { - void intrusive_ptr_add_ref(LLCurl::Responder* p); - void intrusive_ptr_release(LLCurl::Responder* p); -}; +public: + + class CurlRequest : public LLQueuedThread::QueuedRequest + { + protected: + virtual ~CurlRequest(); // use deleteRequest() + + public: + CurlRequest(handle_t handle, LLCurl::Multi* multi, LLCurlThread* curl_thread); + + /*virtual*/ bool processRequest(); + /*virtual*/ void finishRequest(bool completed); + + private: + // input + LLCurl::Multi* mMulti; + LLCurlThread* mCurlThread; + }; + friend class CurlRequest; + +public: + LLCurlThread(bool threaded = true) ; + virtual ~LLCurlThread() ; + + S32 update(F32 max_time_ms); + + void addMulti(LLCurl::Multi* multi) ; + void killMulti(LLCurl::Multi* multi) ; + +private: + bool doMultiPerform(LLCurl::Multi* multi) ; + void deleteMulti(LLCurl::Multi* multi) ; + void cleanupMulti(LLCurl::Multi* multi) ; +} ; class LLCurlRequest @@ -339,7 +405,71 @@ private: LLCurl::Multi* mActiveMulti; S32 mActiveRequestCount; BOOL mProcessing; - U32 mThreadID; // debug +}; + +//for texture fetch only +class LLCurlTextureRequest : public LLCurlRequest +{ +public: + LLCurlTextureRequest(S32 concurrency); + ~LLCurlTextureRequest(); + + U32 getByteRange(const std::string& url, const headers_t& headers, S32 offset, S32 length, U32 pri, LLCurl::ResponderPtr responder, F32 delay_time = -1.f); + void nextRequests(); + void completeRequest(S32 received_bytes); + + void updatePriority(U32 handle, U32 pri); + void removeRequest(U32 handle); + + U32 getTotalReceivedBits(); + U32 getTotalIssuedRequests(); + S32 getNumRequests(); + bool isWaiting(U32 handle); + +private: + LLMutex mMutex; + S32 mConcurrency; + S32 mInQueue; //request currently in queue. + U32 mHandleCounter; + U32 mTotalIssuedRequests; + U32 mTotalReceivedBits; + + typedef struct _request_t + { + _request_t(U32 handle, const std::string& url, const headers_t& headers, S32 offset, S32 length, U32 pri, LLCurl::ResponderPtr responder) : + mHandle(handle), mUrl(url), mHeaders(headers), mOffset(offset), mLength(length), mPriority(pri), mResponder(responder), mStartTime(0.f) + {} + + U32 mHandle; + std::string mUrl; + LLCurlRequest::headers_t mHeaders; + S32 mOffset; + S32 mLength; + LLCurl::ResponderPtr mResponder; + U32 mPriority; + F32 mStartTime; //start time to issue this request + } request_t; + + struct request_compare + { + bool operator()(const request_t* lhs, const request_t* rhs) const + { + if(lhs->mPriority != rhs->mPriority) + { + return lhs->mPriority > rhs->mPriority; // higher priority in front of queue (set) + } + else + { + return (U32)lhs < (U32)rhs; + } + } + }; + + typedef std::set<request_t*, request_compare> req_queue_t; + req_queue_t mCachedRequests; + std::map<S32, request_t*> mRequestMap; + + LLFrameTimer mGlobalTimer; }; class LLCurlEasyRequest @@ -357,9 +487,11 @@ public: void slist_append(const char* str); void sendRequest(const std::string& url); void requestComplete(); - void perform(); bool getResult(CURLcode* result, LLCurl::TransferInfo* info = NULL); std::string getErrorString(); + bool isCompleted() {return mMulti->isCompleted() ;} + bool wait() { return mMulti->waitToComplete(); } + bool isValid() {return mMulti && mMulti->isValid(); } LLCurl::Easy* getEasy() const { return mEasy; } diff --git a/indra/llmessage/lldbstrings.h b/indra/llmessage/lldbstrings.h index 9bf1b3eda4..e23d17d5b6 100644 --- a/indra/llmessage/lldbstrings.h +++ b/indra/llmessage/lldbstrings.h @@ -156,18 +156,6 @@ const S32 DB_USER_SKILLS_BUF_SIZE = 255; const S32 DB_NV_NAME_STR_LEN = 128; const S32 DB_NV_NAME_BUF_SIZE = 129; -// votes.vote_text varchar(254) -const S32 DB_VOTE_TEXT_STR_LEN = 254; -const S32 DB_VOTE_TEXT_BUF_SIZE = 255; - -// vpte type text varchar(9) -const S32 DB_VOTE_TYPE_STR_LEN = 9; -const S32 DB_VOTE_TYPE_BUF_SIZE = 10; - -// vote result text -const S32 DB_VOTE_RESULT_BUF_LEN = 8; -const S32 DB_VOTE_RESULT_BUF_SIZE = 9; - // user_start_location.location_name varchar(254) const S32 DB_START_LOCATION_STR_LEN = 254; const S32 DB_START_LOCATION_BUF_SIZE = 255; diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index 2bca517e97..d6ed08055e 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -232,7 +232,8 @@ LLSD LLHTTPAssetRequest::getFullDetails() const void LLHTTPAssetRequest::setupCurlHandle() { // *NOTE: Similar code exists in mapserver/llcurlutil.cpp JC - mCurlHandle = curl_easy_init(); + mCurlHandle = LLCurl::newEasyHandle(); + llassert_always(mCurlHandle != NULL) ; // Apply proxy settings if configured to do so LLProxy::getInstance()->applyProxySettings(mCurlHandle); @@ -278,7 +279,7 @@ void LLHTTPAssetRequest::setupCurlHandle() void LLHTTPAssetRequest::cleanupCurlHandle() { - curl_easy_cleanup(mCurlHandle); + LLCurl::deleteEasyHandle(mCurlHandle); if (mAssetStoragep) { // Terminating a request. Thus upload or download is no longer pending. @@ -429,12 +430,13 @@ void LLHTTPAssetStorage::_init(const std::string& web_host, const std::string& l // curl_global_init moved to LLCurl::initClass() - mCurlMultiHandle = curl_multi_init(); + mCurlMultiHandle = LLCurl::newMultiHandle() ; + llassert_always(mCurlMultiHandle != NULL) ; } LLHTTPAssetStorage::~LLHTTPAssetStorage() { - curl_multi_cleanup(mCurlMultiHandle); + LLCurl::deleteMultiHandle(mCurlMultiHandle); mCurlMultiHandle = NULL; // curl_global_cleanup moved to LLCurl::initClass() @@ -747,7 +749,7 @@ LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list request_list_t::iterator pending_iter = pending.begin(); request_list_t::iterator pending_end = pending.end(); // Loop over all pending requests until we miss finding it in the running list. - for (; pending_iter != pending.end(); ++pending_iter) + for (; pending_iter != pending_end; ++pending_iter) { LLAssetRequest* req = *pending_iter; // Look for this pending request in the running list. diff --git a/indra/llmessage/llhttpclient.cpp b/indra/llmessage/llhttpclient.cpp index dd4e3a6300..0c325a68aa 100644 --- a/indra/llmessage/llhttpclient.cpp +++ b/indra/llmessage/llhttpclient.cpp @@ -158,7 +158,7 @@ namespace if(fstream.is_open()) { fstream.seekg(0, std::ios::end); - U32 fileSize = fstream.tellg(); + U32 fileSize = (U32)fstream.tellg(); fstream.seekg(0, std::ios::beg); std::vector<char> fileBuffer(fileSize); fstream.read(&fileBuffer[0], fileSize); @@ -228,6 +228,12 @@ static void request( LLPumpIO::chain_t chain; LLURLRequest* req = new LLURLRequest(method, url); + if(!req->isValid())//failed + { + delete req ; + return ; + } + req->setSSLVerifyCallback(LLHTTPClient::getCertVerifyCallback(), (void *)req); @@ -423,7 +429,9 @@ static LLSD blocking_request( { lldebugs << "blockingRequest of " << url << llendl; char curl_error_buffer[CURL_ERROR_SIZE] = "\0"; - CURL* curlp = curl_easy_init(); + CURL* curlp = LLCurl::newEasyHandle(); + llassert_always(curlp != NULL) ; + LLHTTPBuffer http_buffer; std::string body_str; @@ -517,7 +525,7 @@ static LLSD blocking_request( } // * Cleanup - curl_easy_cleanup(curlp); + LLCurl::deleteEasyHandle(curlp); return response; } diff --git a/indra/llmessage/llinstantmessage.cpp b/indra/llmessage/llinstantmessage.cpp index d68e0c423e..b0275c161b 100644 --- a/indra/llmessage/llinstantmessage.cpp +++ b/indra/llmessage/llinstantmessage.cpp @@ -43,14 +43,6 @@ const U8 IM_ONLINE = 0; const U8 IM_OFFLINE = 1; -const S32 VOTE_YES = 1; -const S32 VOTE_NO = 0; -const S32 VOTE_ABSTAIN = -1; - -const S32 VOTE_MAJORITY = 0; -const S32 VOTE_SUPER_MAJORITY = 1; -const S32 VOTE_UNANIMOUS = 2; - const char EMPTY_BINARY_BUCKET[] = ""; const S32 EMPTY_BINARY_BUCKET_SIZE = 1; const U32 NO_TIMESTAMP = 0; @@ -69,7 +61,6 @@ LLIMInfo::LLIMInfo() : mViewerThinksToIsOnline(false), mIMType(IM_NOTHING_SPECIAL), mTimeStamp(0), - mSource(IM_FROM_SIM), mTTL(IM_TTL) { } @@ -88,7 +79,6 @@ LLIMInfo::LLIMInfo( LLSD data, U8 offline, U32 timestamp, - EIMSource source, S32 ttl) : mFromID(from_id), mFromGroup(from_group), @@ -104,14 +94,12 @@ LLIMInfo::LLIMInfo( mName(name), mMessage(message), mData(data), - mSource(source), mTTL(ttl) { } -LLIMInfo::LLIMInfo(LLMessageSystem* msg, EIMSource source, S32 ttl) : +LLIMInfo::LLIMInfo(LLMessageSystem* msg, S32 ttl) : mViewerThinksToIsOnline(false), - mSource(source), mTTL(ttl) { unpackMessageBlock(msg); @@ -326,7 +314,6 @@ LLSD im_info_to_llsd(LLPointer<LLIMInfo> im_info) param_message["region_id"] = im_info->mRegionID; param_message["position"] = ll_sd_from_vector3(im_info->mPosition); param_message["data"] = im_info->mData; - param_message["source"]= im_info->mSource; param_message["ttl"] = im_info->mTTL; LLSD param_agent; @@ -359,7 +346,6 @@ LLPointer<LLIMInfo> llsd_to_im_info(const LLSD& im_info_sd) param_message["data"], (U8) param_message["offline"].asInteger(), (U32) param_message["timestamp"].asInteger(), - (EIMSource)param_message["source"].asInteger(), param_message["ttl"].asInteger()); return im_info; @@ -381,7 +367,6 @@ LLPointer<LLIMInfo> LLIMInfo::clone() mData, mOffline, mTimeStamp, - mSource, mTTL); } diff --git a/indra/llmessage/llinstantmessage.h b/indra/llmessage/llinstantmessage.h index e0dae376b4..db4a38ea9e 100644 --- a/indra/llmessage/llinstantmessage.h +++ b/indra/llmessage/llinstantmessage.h @@ -115,8 +115,8 @@ enum EInstantMessage // viewer, since you can't IM an object yet. IM_FROM_TASK = 19, - // sent an IM to a busy user, this is the auto response - IM_BUSY_AUTO_RESPONSE = 20, + // sent an IM to a do not disturb user, this is the auto response + IM_DO_NOT_DISTURB_AUTO_RESPONSE = 20, // Shows the message in the console and chat history IM_CONSOLE_AND_CHAT_HISTORY = 21, @@ -164,57 +164,9 @@ enum EInstantMessage }; -// Hooks for quickly hacking in experimental admin debug messages -// without needing to recompile the viewer -// *NOTE: This functionality has been moved to be a string based -// operation so that we don't even have to do a full recompile. This -// enumeration will be phased out soon. -enum EGodlikeRequest -{ - GOD_WANTS_NOTHING, - - // for requesting physics information about an object - GOD_WANTS_PHYSICS_INFO, - - // two unused requests that can be appropriated for debug - // purposes (no viewer recompile necessary) - GOD_WANTS_FOO, - GOD_WANTS_BAR, - - // to dump simulator terrain data to terrain.raw file - GOD_WANTS_TERRAIN_SAVE, - // to load simulator terrain data from terrain.raw file - GOD_WANTS_TERRAIN_LOAD, - - GOD_WANTS_TOGGLE_AVATAR_GEOMETRY, // HACK for testing new avatar geom - - // real-time telehub operations - GOD_WANTS_TELEHUB_INFO, - GOD_WANTS_CONNECT_TELEHUB, - GOD_WANTS_DELETE_TELEHUB, - GOD_WANTS_ADD_TELEHUB_SPAWNPOINT, - GOD_WANTS_REMOVE_TELEHUB_SPAWNPOINT, - -}; - -enum EIMSource -{ - IM_FROM_VIEWER, - IM_FROM_DATASERVER, - IM_FROM_SIM -}; - extern const U8 IM_ONLINE; extern const U8 IM_OFFLINE; -extern const S32 VOTE_YES; -extern const S32 VOTE_NO; -extern const S32 VOTE_ABSTAIN; - -extern const S32 VOTE_MAJORITY; -extern const S32 VOTE_SUPER_MAJORITY; -extern const S32 VOTE_UNANIMOUS; - extern const char EMPTY_BINARY_BUCKET[]; extern const S32 EMPTY_BINARY_BUCKET_SIZE; @@ -234,7 +186,6 @@ protected: public: LLIMInfo(LLMessageSystem* msg, - EIMSource source = IM_FROM_SIM, S32 ttl = IM_TTL); LLIMInfo( @@ -251,7 +202,6 @@ public: LLSD data, U8 offline, U32 timestamp, - EIMSource source, S32 ttl = IM_TTL); void packInstantMessage(LLMessageSystem* msg) const; @@ -274,7 +224,6 @@ public: std::string mMessage; LLSD mData; - EIMSource mSource; S32 mTTL; }; diff --git a/indra/llmessage/lliohttpserver.cpp b/indra/llmessage/lliohttpserver.cpp index 73e8a69085..1236fc8b71 100644 --- a/indra/llmessage/lliohttpserver.cpp +++ b/indra/llmessage/lliohttpserver.cpp @@ -37,12 +37,10 @@ #include "lliopipe.h" #include "lliosocket.h" #include "llioutil.h" -#include "llmemtype.h" #include "llmemorystream.h" #include "llpumpio.h" #include "llsd.h" #include "llsdserialize_xml.h" -#include "llstat.h" #include "llstl.h" #include "lltimer.h" @@ -141,6 +139,11 @@ private: }; static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_PIPE("HTTP Pipe"); +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_GET("HTTP Get"); +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_PUT("HTTP Put"); +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_POST("HTTP Post"); +static LLFastTimer::DeclareTimer FTM_PROCESS_HTTP_DELETE("HTTP Delete"); + LLIOPipe::EStatus LLHTTPPipe::process_impl( const LLChannelDescriptors& channels, buffer_ptr_t& buffer, @@ -177,12 +180,12 @@ LLIOPipe::EStatus LLHTTPPipe::process_impl( std::string verb = context[CONTEXT_REQUEST][CONTEXT_VERB]; if(verb == HTTP_VERB_GET) { - LLPerfBlock getblock("http_get"); + LLFastTimer _(FTM_PROCESS_HTTP_GET); mNode.get(LLHTTPNode::ResponsePtr(mResponse), context); } else if(verb == HTTP_VERB_PUT) { - LLPerfBlock putblock("http_put"); + LLFastTimer _(FTM_PROCESS_HTTP_PUT); LLSD input; if (mNode.getContentType() == LLHTTPNode::CONTENT_TYPE_LLSD) { @@ -198,7 +201,7 @@ LLIOPipe::EStatus LLHTTPPipe::process_impl( } else if(verb == HTTP_VERB_POST) { - LLPerfBlock postblock("http_post"); + LLFastTimer _(FTM_PROCESS_HTTP_POST); LLSD input; if (mNode.getContentType() == LLHTTPNode::CONTENT_TYPE_LLSD) { @@ -214,7 +217,7 @@ LLIOPipe::EStatus LLHTTPPipe::process_impl( } else if(verb == HTTP_VERB_DELETE) { - LLPerfBlock delblock("http_delete"); + LLFastTimer _(FTM_PROCESS_HTTP_DELETE); mNode.del(LLHTTPNode::ResponsePtr(mResponse), context); } else if(verb == HTTP_VERB_OPTIONS) @@ -443,7 +446,6 @@ LLIOPipe::EStatus LLHTTPResponseHeader::process_impl( { LLFastTimer t(FTM_PROCESS_HTTP_HEADER); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); if(eos) { PUMP_DEBUG; @@ -587,13 +589,11 @@ LLHTTPResponder::LLHTTPResponder(const LLHTTPNode& tree, const LLSD& ctx) : mContentLength(0), mRootNode(tree) { - LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); } // virtual LLHTTPResponder::~LLHTTPResponder() { - LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); //lldebugs << "destroying LLHTTPResponder" << llendl; } @@ -603,7 +603,6 @@ bool LLHTTPResponder::readHeaderLine( U8* dest, S32& len) { - LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); --len; U8* last = buffer->readAfter(channels.in(), mLastRead, dest, len); dest[len] = '\0'; @@ -628,7 +627,6 @@ void LLHTTPResponder::markBad( const LLChannelDescriptors& channels, buffer_ptr_t buffer) { - LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); mState = STATE_SHORT_CIRCUIT; LLBufferStream out(channels, buffer.get()); out << HTTP_VERSION_STR << " 400 Bad Request\r\n\r\n<html>\n" @@ -648,7 +646,6 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( { LLFastTimer t(FTM_PROCESS_HTTP_RESPONDER); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_HTTP_SERVER); LLIOPipe::EStatus status = STATUS_OK; // parsing headers @@ -818,6 +815,8 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( // Copy everything after mLast read to the out. LLBufferArray::segment_iterator_t seg_iter; + + buffer->lock(); seg_iter = buffer->splitAfter(mLastRead); if(seg_iter != buffer->endSegment()) { @@ -838,7 +837,7 @@ LLIOPipe::EStatus LLHTTPResponder::process_impl( } #endif } - + buffer->unlock(); // // *FIX: get rid of extra bytes off the end // diff --git a/indra/llmessage/lliopipe.cpp b/indra/llmessage/lliopipe.cpp index 6e4eec74a6..8f827f7a30 100644 --- a/indra/llmessage/lliopipe.cpp +++ b/indra/llmessage/lliopipe.cpp @@ -75,6 +75,12 @@ LLIOPipe::~LLIOPipe() //lldebugs << "destroying LLIOPipe" << llendl; } +//virtual +bool LLIOPipe::isValid() +{ + return true ; +} + // static std::string LLIOPipe::lookupStatusString(EStatus status) { diff --git a/indra/llmessage/lliopipe.h b/indra/llmessage/lliopipe.h index 8e656b6da1..cbd17b5a3d 100644 --- a/indra/llmessage/lliopipe.h +++ b/indra/llmessage/lliopipe.h @@ -231,6 +231,8 @@ public: */ virtual ~LLIOPipe(); + virtual bool isValid() ; + protected: /** * @brief Base Constructor. diff --git a/indra/llmessage/lliosocket.cpp b/indra/llmessage/lliosocket.cpp index 54ceab3422..0287026659 100644 --- a/indra/llmessage/lliosocket.cpp +++ b/indra/llmessage/lliosocket.cpp @@ -33,7 +33,6 @@ #include "llbuffer.h" #include "llhost.h" -#include "llmemtype.h" #include "llpumpio.h" // @@ -100,7 +99,6 @@ void ll_debug_socket(const char* msg, apr_socket_t* apr_sock) // static LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) { - LLMemType m1(LLMemType::MTYPE_IO_TCP); LLSocket::ptr_t rv; apr_socket_t* socket = NULL; apr_pool_t* new_pool = NULL; @@ -198,7 +196,6 @@ LLSocket::ptr_t LLSocket::create(apr_pool_t* pool, EType type, U16 port) // static LLSocket::ptr_t LLSocket::create(apr_socket_t* socket, apr_pool_t* pool) { - LLMemType m1(LLMemType::MTYPE_IO_TCP); LLSocket::ptr_t rv; if(!socket) { @@ -240,12 +237,10 @@ LLSocket::LLSocket(apr_socket_t* socket, apr_pool_t* pool) : mPort(PORT_INVALID) { ll_debug_socket("Constructing wholely formed socket", mSocket); - LLMemType m1(LLMemType::MTYPE_IO_TCP); } LLSocket::~LLSocket() { - LLMemType m1(LLMemType::MTYPE_IO_TCP); // *FIX: clean up memory we are holding. if(mSocket) { @@ -265,7 +260,6 @@ LLSocket::~LLSocket() void LLSocket::setBlocking(S32 timeout) { - LLMemType m1(LLMemType::MTYPE_IO_TCP); // set up the socket options ll_apr_warn_status(apr_socket_timeout_set(mSocket, timeout)); ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_NONBLOCK, 0)); @@ -276,7 +270,6 @@ void LLSocket::setBlocking(S32 timeout) void LLSocket::setNonBlocking() { - LLMemType m1(LLMemType::MTYPE_IO_TCP); // set up the socket options ll_apr_warn_status(apr_socket_timeout_set(mSocket, 0)); ll_apr_warn_status(apr_socket_opt_set(mSocket, APR_SO_NONBLOCK, 1)); @@ -293,12 +286,10 @@ LLIOSocketReader::LLIOSocketReader(LLSocket::ptr_t socket) : mSource(socket), mInitialized(false) { - LLMemType m1(LLMemType::MTYPE_IO_TCP); } LLIOSocketReader::~LLIOSocketReader() { - LLMemType m1(LLMemType::MTYPE_IO_TCP); //lldebugs << "Destroying LLIOSocketReader" << llendl; } @@ -314,7 +305,6 @@ LLIOPipe::EStatus LLIOSocketReader::process_impl( { LLFastTimer t(FTM_PROCESS_SOCKET_READER); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!mSource) return STATUS_PRECONDITION_NOT_MET; if(!mInitialized) { @@ -396,12 +386,10 @@ LLIOSocketWriter::LLIOSocketWriter(LLSocket::ptr_t socket) : mLastWritten(NULL), mInitialized(false) { - LLMemType m1(LLMemType::MTYPE_IO_TCP); } LLIOSocketWriter::~LLIOSocketWriter() { - LLMemType m1(LLMemType::MTYPE_IO_TCP); //lldebugs << "Destroying LLIOSocketWriter" << llendl; } @@ -416,7 +404,6 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( { LLFastTimer t(FTM_PROCESS_SOCKET_WRITER); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!mDestination) return STATUS_PRECONDITION_NOT_MET; if(!mInitialized) { @@ -445,6 +432,7 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( // efficient - not only because writev() is better, but also // because we won't have to do as much work to find the start // address. + buffer->lock(); LLBufferArray::segment_iterator_t it; LLBufferArray::segment_iterator_t end = buffer->endSegment(); LLSegment segment; @@ -524,6 +512,8 @@ LLIOPipe::EStatus LLIOSocketWriter::process_impl( } } + buffer->unlock(); + PUMP_DEBUG; if(done && eos) { @@ -547,12 +537,10 @@ LLIOServerSocket::LLIOServerSocket( mInitialized(false), mResponseTimeout(DEFAULT_CHAIN_EXPIRY_SECS) { - LLMemType m1(LLMemType::MTYPE_IO_TCP); } LLIOServerSocket::~LLIOServerSocket() { - LLMemType m1(LLMemType::MTYPE_IO_TCP); //lldebugs << "Destroying LLIOServerSocket" << llendl; } @@ -572,7 +560,6 @@ LLIOPipe::EStatus LLIOServerSocket::process_impl( { LLFastTimer t(FTM_PROCESS_SERVER_SOCKET); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_TCP); if(!pump) { llwarns << "Need a pump for server socket." << llendl; diff --git a/indra/llmessage/llmessagetemplate.h b/indra/llmessage/llmessagetemplate.h index 16d825d33b..ae8e0087c1 100644 --- a/indra/llmessage/llmessagetemplate.h +++ b/indra/llmessage/llmessagetemplate.h @@ -29,7 +29,6 @@ #include "lldarray.h" #include "message.h" // TODO: babbage: Remove... -#include "llstat.h" #include "llstl.h" class LLMsgVarData @@ -263,6 +262,7 @@ enum EMsgDeprecation MD_DEPRECATED }; + class LLMessageTemplate { public: @@ -364,7 +364,6 @@ public: { if (mHandlerFunc) { - LLPerfBlock msg_cb_time("msg_cb", mName); mHandlerFunc(msgsystem, mUserData); return TRUE; } diff --git a/indra/llmessage/llmime.cpp b/indra/llmessage/llmime.cpp index 943a734927..9d9c4ebd68 100644 --- a/indra/llmessage/llmime.cpp +++ b/indra/llmessage/llmime.cpp @@ -388,7 +388,7 @@ bool LLMimeParser::Impl::parseHeaders( // not to read past limit when we get() the newline. S32 max_get = llmin((S32)LINE_BUFFER_LENGTH, limit - mScanCount - 1); istr.getline(mBuffer, max_get, '\r'); - mScanCount += istr.gcount(); + mScanCount += (S32)istr.gcount(); int c = istr.get(); if(EOF == c) { @@ -496,7 +496,7 @@ void LLMimeParser::Impl::scanPastSeparator( // past limit when we get() the newline. S32 max_get = llmin((S32)LINE_BUFFER_LENGTH, limit - mScanCount - 1); istr.getline(mBuffer, max_get, '\r'); - mScanCount += istr.gcount(); + mScanCount += (S32)istr.gcount(); if(istr.gcount() >= LINE_BUFFER_LENGTH - 1) { // that's way too long to be a separator, so ignore it. diff --git a/indra/llmessage/llpumpio.cpp b/indra/llmessage/llpumpio.cpp index a8d2a0a224..0623e99f0a 100644 --- a/indra/llmessage/llpumpio.cpp +++ b/indra/llmessage/llpumpio.cpp @@ -34,9 +34,7 @@ #include "apr_poll.h" #include "llapr.h" -#include "llmemtype.h" #include "llstl.h" -#include "llstat.h" // These should not be enabled in production, but they can be // intensely useful during development for finding certain kinds of @@ -153,7 +151,6 @@ struct ll_delete_apr_pollset_fd_client_data typedef std::pair<LLIOPipe::ptr_t, apr_pollfd_t> pipe_conditional_t; void operator()(const pipe_conditional_t& conditional) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); S32* client_id = (S32*)conditional.second.client_data; delete client_id; } @@ -177,35 +174,33 @@ LLPumpIO::LLPumpIO(apr_pool_t* pool) : { mCurrentChain = mRunningChains.end(); - LLMemType m1(LLMemType::MTYPE_IO_PUMP); initialize(pool); } LLPumpIO::~LLPumpIO() { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); cleanup(); } bool LLPumpIO::prime(apr_pool_t* pool) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); cleanup(); initialize(pool); return ((pool == NULL) ? false : true); } -bool LLPumpIO::addChain(const chain_t& chain, F32 timeout) +bool LLPumpIO::addChain(const chain_t& chain, F32 timeout, bool has_curl_request) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(chain.empty()) return false; #if LL_THREADS_APR LLScopedLock lock(mChainsMutex); #endif LLChainInfo info; + info.mHasCurlRequest = has_curl_request; info.setTimeoutSeconds(timeout); info.mData = LLIOPipe::buffer_ptr_t(new LLBufferArray); + info.mData->setThreaded(has_curl_request); LLLinkInfo link; #if LL_DEBUG_PIPE_TYPE_IN_PUMP lldebugs << "LLPumpIO::addChain() " << chain[0] << " '" @@ -231,7 +226,6 @@ bool LLPumpIO::addChain( LLSD context, F32 timeout) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); // remember that if the caller is providing a full link // description, we need to have that description matched to a @@ -309,7 +303,6 @@ static std::string events_2_string(apr_int16_t events) bool LLPumpIO::setConditional(LLIOPipe* pipe, const apr_pollfd_t* poll) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(!pipe) return false; ll_debug_poll_fd("Set conditional", poll); @@ -421,7 +414,6 @@ bool LLPumpIO::sleepChain(F64 seconds) bool LLPumpIO::copyCurrentLinkInfo(links_t& links) const { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(mRunningChains.end() == mCurrentChain) { return false; @@ -439,11 +431,20 @@ void LLPumpIO::pump() } static LLFastTimer::DeclareTimer FTM_PUMP_IO("Pump IO"); +static LLFastTimer::DeclareTimer FTM_PUMP_POLL("Pump Poll"); + +LLPumpIO::current_chain_t LLPumpIO::removeRunningChain(LLPumpIO::current_chain_t& run_chain) +{ + std::for_each( + (*run_chain).mDescriptors.begin(), + (*run_chain).mDescriptors.end(), + ll_delete_apr_pollset_fd_client_data()); + return mRunningChains.erase(run_chain); +} //timeout is in microseconds void LLPumpIO::pump(const S32& poll_timeout) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); LLFastTimer t1(FTM_PUMP_IO); //llinfos << "LLPumpIO::pump()" << llendl; @@ -525,7 +526,7 @@ void LLPumpIO::pump(const S32& poll_timeout) S32 count = 0; S32 client_id = 0; { - LLPerfBlock polltime("pump_poll"); + LLFastTimer _(FTM_PUMP_POLL); apr_pollset_poll(mPollset, poll_timeout, &count, &poll_fd); } PUMP_DEBUG; @@ -585,10 +586,16 @@ void LLPumpIO::pump(const S32& poll_timeout) // << (*run_chain).mChainLinks[0].mPipe // << " because we reached the end." << llendl; #endif - run_chain = mRunningChains.erase(run_chain); + run_chain = removeRunningChain(run_chain); continue; } } + else if(isChainExpired(*run_chain)) + { + run_chain = removeRunningChain(run_chain); + continue; + } + PUMP_DEBUG; if((*run_chain).mLock) { @@ -696,11 +703,7 @@ void LLPumpIO::pump(const S32& poll_timeout) PUMP_DEBUG; // This chain is done. Clean up any allocated memory and // erase the chain info. - std::for_each( - (*run_chain).mDescriptors.begin(), - (*run_chain).mDescriptors.end(), - ll_delete_apr_pollset_fd_client_data()); - run_chain = mRunningChains.erase(run_chain); + run_chain = removeRunningChain(run_chain); // *NOTE: may not always need to rebuild the pollset. mRebuildPollset = true; @@ -734,7 +737,6 @@ void LLPumpIO::pump(const S32& poll_timeout) bool LLPumpIO::respond(LLIOPipe* pipe) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(NULL == pipe) return false; #if LL_THREADS_APR @@ -753,7 +755,6 @@ bool LLPumpIO::respond( LLIOPipe::buffer_ptr_t data, LLSD context) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); // if the caller is providing a full link description, we need to // have that description matched to a particular buffer. if(!data) return false; @@ -776,7 +777,6 @@ static LLFastTimer::DeclareTimer FTM_PUMP_CALLBACK_CHAIN("Chain"); void LLPumpIO::callback() { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); //llinfos << "LLPumpIO::callback()" << llendl; if(true) { @@ -827,7 +827,6 @@ void LLPumpIO::control(LLPumpIO::EControl op) void LLPumpIO::initialize(apr_pool_t* pool) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(!pool) return; #if LL_THREADS_APR // SJB: Windows defaults to NESTED and OSX defaults to UNNESTED, so use UNNESTED explicitly. @@ -839,7 +838,6 @@ void LLPumpIO::initialize(apr_pool_t* pool) void LLPumpIO::cleanup() { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); #if LL_THREADS_APR if(mChainsMutex) apr_thread_mutex_destroy(mChainsMutex); if(mCallbackMutex) apr_thread_mutex_destroy(mCallbackMutex); @@ -862,7 +860,6 @@ void LLPumpIO::cleanup() void LLPumpIO::rebuildPollset() { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); // lldebugs << "LLPumpIO::rebuildPollset()" << llendl; if(mPollset) { @@ -915,7 +912,6 @@ void LLPumpIO::rebuildPollset() void LLPumpIO::processChain(LLChainInfo& chain) { PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_PUMP); LLIOPipe::EStatus status = LLIOPipe::STATUS_OK; links_t::iterator it = chain.mHead; links_t::iterator end = chain.mChainLinks.end(); @@ -1095,11 +1091,28 @@ void LLPumpIO::processChain(LLChainInfo& chain) PUMP_DEBUG; } +bool LLPumpIO::isChainExpired(LLChainInfo& chain) +{ + if(!chain.mHasCurlRequest) + { + return false ; + } + + for(links_t::iterator iter = chain.mChainLinks.begin(); iter != chain.mChainLinks.end(); ++iter) + { + if(!(*iter).mPipe->isValid()) + { + return true ; + } + } + + return false ; +} + bool LLPumpIO::handleChainError( LLChainInfo& chain, LLIOPipe::EStatus error) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); links_t::reverse_iterator rit; if(chain.mHead == chain.mChainLinks.end()) { @@ -1136,6 +1149,9 @@ bool LLPumpIO::handleChainError( #endif keep_going = false; break; + case LLIOPipe::STATUS_EXPIRED: + keep_going = false; + break ; default: if(LLIOPipe::isSuccess(error)) { @@ -1157,15 +1173,14 @@ bool LLPumpIO::handleChainError( LLPumpIO::LLChainInfo::LLChainInfo() : mInit(false), mLock(0), - mEOS(false) + mEOS(false), + mHasCurlRequest(false) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); mTimer.setTimerExpirySec(DEFAULT_CHAIN_EXPIRY_SECS); } void LLPumpIO::LLChainInfo::setTimeoutSeconds(F32 timeout) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(timeout > 0.0f) { mTimer.start(); @@ -1180,7 +1195,6 @@ void LLPumpIO::LLChainInfo::setTimeoutSeconds(F32 timeout) void LLPumpIO::LLChainInfo::adjustTimeoutSeconds(F32 delta) { - LLMemType m1(LLMemType::MTYPE_IO_PUMP); if(mTimer.getStarted()) { F64 expiry = mTimer.expiresAt(); diff --git a/indra/llmessage/llpumpio.h b/indra/llmessage/llpumpio.h index 9303c9d7fc..d2c5d37571 100644 --- a/indra/llmessage/llpumpio.h +++ b/indra/llmessage/llpumpio.h @@ -111,9 +111,10 @@ public: * @param chain The pipes for the chain * @param timeout The number of seconds in the future to * expire. Pass in 0.0f to never expire. + * @param has_curl_request The chain contains LLURLRequest if true. * @return Returns true if anything was added to the pump. */ - bool addChain(const chain_t& chain, F32 timeout); + bool addChain(const chain_t& chain, F32 timeout, bool has_curl_request = false); /** * @brief Struct to associate a pipe with it's buffer io indexes. @@ -356,12 +357,13 @@ protected: // basic member data bool mInit; + bool mEOS; + bool mHasCurlRequest; S32 mLock; LLFrameTimer mTimer; links_t::iterator mHead; links_t mChainLinks; - LLIOPipe::buffer_ptr_t mData; - bool mEOS; + LLIOPipe::buffer_ptr_t mData; LLSD mContext; // tracking inside the pump @@ -402,7 +404,7 @@ protected: protected: void initialize(apr_pool_t* pool); void cleanup(); - + current_chain_t removeRunningChain(current_chain_t& chain) ; /** * @brief Given the internal state of the chains, rebuild the pollset * @see setConditional() @@ -429,6 +431,9 @@ protected: */ bool handleChainError(LLChainInfo& chain, LLIOPipe::EStatus error); + //if the chain is expired, remove it + bool isChainExpired(LLChainInfo& chain) ; + public: /** * @brief Return number of running chains. diff --git a/indra/llmessage/llsdmessage.cpp b/indra/llmessage/llsdmessage.cpp index 9148c9dd15..1c93c12d99 100644 --- a/indra/llmessage/llsdmessage.cpp +++ b/indra/llmessage/llsdmessage.cpp @@ -88,7 +88,7 @@ bool LLSDMessage::httpListener(const LLSD& request) request, url, "POST", reply, error), LLSD(), // headers - timeout); + (F32)timeout); return false; } diff --git a/indra/llmessage/llsdmessagebuilder.cpp b/indra/llmessage/llsdmessagebuilder.cpp index 2698a271ee..615221e0ad 100644 --- a/indra/llmessage/llsdmessagebuilder.cpp +++ b/indra/llmessage/llsdmessagebuilder.cpp @@ -317,7 +317,7 @@ void LLSDMessageBuilder::copyFromMessageData(const LLMsgData& data) // S64 not supported in LLSD so we just truncate it case MVT_S64: - addS32(varname, *(S64*)mvci.getData()); + addS32(varname, (S32)*(S64*)mvci.getData()); break; case MVT_F32: diff --git a/indra/llmessage/llsdmessagereader.cpp b/indra/llmessage/llsdmessagereader.cpp index 304a692cdf..a6fccd2a56 100644 --- a/indra/llmessage/llsdmessagereader.cpp +++ b/indra/llmessage/llsdmessagereader.cpp @@ -276,7 +276,7 @@ S32 getElementSize(const LLSD& llsd) case LLSD::TypeReal: return sizeof(F64); case LLSD::TypeString: - return llsd.asString().size(); + return llsd.size(); case LLSD::TypeUUID: return sizeof(LLUUID); case LLSD::TypeDate: @@ -291,9 +291,10 @@ S32 getElementSize(const LLSD& llsd) case LLSD::TypeMap: case LLSD::TypeArray: case LLSD::TypeUndefined: + default: // TypeLLSDTypeEnd, TypeLLSDNumTypes, etc. return 0; } - return 0; + //return 0; } //virtual diff --git a/indra/llmessage/llsdrpcclient.cpp b/indra/llmessage/llsdrpcclient.cpp index 91fd070f07..fcda0e81a3 100644 --- a/indra/llmessage/llsdrpcclient.cpp +++ b/indra/llmessage/llsdrpcclient.cpp @@ -31,7 +31,6 @@ #include "llbufferstream.h" #include "llfiltersd2xmlrpc.h" -#include "llmemtype.h" #include "llpumpio.h" #include "llsd.h" #include "llsdserialize.h" @@ -50,18 +49,15 @@ LLSDRPCResponse::LLSDRPCResponse() : mIsError(false), mIsFault(false) { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); } // virtual LLSDRPCResponse::~LLSDRPCResponse() { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); } bool LLSDRPCResponse::extractResponse(const LLSD& sd) { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); bool rv = true; if(sd.has(LLSDRPC_RESPONSE_NAME)) { @@ -94,7 +90,6 @@ LLIOPipe::EStatus LLSDRPCResponse::process_impl( { LLFastTimer t(FTM_SDRPC_RESPONSE); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); if(mIsError) { error(pump); @@ -119,13 +114,11 @@ LLSDRPCClient::LLSDRPCClient() : mState(STATE_NONE), mQueue(EPBQ_PROCESS) { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); } // virtual LLSDRPCClient::~LLSDRPCClient() { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); } bool LLSDRPCClient::call( @@ -135,7 +128,6 @@ bool LLSDRPCClient::call( LLSDRPCResponse* response, EPassBackQueue queue) { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); //llinfos << "RPC: " << uri << "." << method << "(" << *parameter << ")" // << llendl; if(method.empty() || !response) @@ -162,7 +154,6 @@ bool LLSDRPCClient::call( LLSDRPCResponse* response, EPassBackQueue queue) { - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); //llinfos << "RPC: " << uri << "." << method << "(" << parameter << ")" // << llendl; if(method.empty() || parameter.empty() || !response) @@ -193,7 +184,6 @@ LLIOPipe::EStatus LLSDRPCClient::process_impl( { LLFastTimer t(FTM_PROCESS_SDRPC_CLIENT); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_SD_CLIENT); if((STATE_NONE == mState) || (!pump)) { // You should have called the call() method already. diff --git a/indra/llmessage/llsdrpcclient.h b/indra/llmessage/llsdrpcclient.h index 9fb49a5c33..0cecf4f688 100644 --- a/indra/llmessage/llsdrpcclient.h +++ b/indra/llmessage/llsdrpcclient.h @@ -240,9 +240,16 @@ public: virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const { lldebugs << "LLSDRPCClientFactory::build" << llendl; - LLIOPipe::ptr_t service(new Client); - chain.push_back(service); LLURLRequest* http(new LLURLRequest(LLURLRequest::HTTP_POST)); + if(!http->isValid()) + { + llwarns << "Creating LLURLRequest failed." << llendl ; + delete http; + return false; + } + + LLIOPipe::ptr_t service(new Client); + chain.push_back(service); LLIOPipe::ptr_t http_pipe(http); http->addHeader("Content-Type: text/llsd"); if(mURL.empty()) @@ -283,9 +290,16 @@ public: virtual bool build(LLPumpIO::chain_t& chain, LLSD context) const { lldebugs << "LLXMLSDRPCClientFactory::build" << llendl; - LLIOPipe::ptr_t service(new Client); - chain.push_back(service); + LLURLRequest* http(new LLURLRequest(LLURLRequest::HTTP_POST)); + if(!http->isValid()) + { + llwarns << "Creating LLURLRequest failed." << llendl ; + delete http; + return false ; + } + LLIOPipe::ptr_t service(new Client); + chain.push_back(service); LLIOPipe::ptr_t http_pipe(http); http->addHeader("Content-Type: text/xml"); if(mURL.empty()) diff --git a/indra/llmessage/llsdrpcserver.cpp b/indra/llmessage/llsdrpcserver.cpp index 9f776aca72..f26ee52f71 100644 --- a/indra/llmessage/llsdrpcserver.cpp +++ b/indra/llmessage/llsdrpcserver.cpp @@ -31,7 +31,6 @@ #include "llbuffer.h" #include "llbufferstream.h" -#include "llmemtype.h" #include "llpumpio.h" #include "llsdserialize.h" #include "llstl.h" @@ -58,12 +57,10 @@ LLSDRPCServer::LLSDRPCServer() : mPump(NULL), mLock(0) { - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); } LLSDRPCServer::~LLSDRPCServer() { - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); std::for_each( mMethods.begin(), mMethods.end(), @@ -109,7 +106,6 @@ LLIOPipe::EStatus LLSDRPCServer::process_impl( { LLFastTimer t(FTM_PROCESS_SDRPC_SERVER); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); // lldebugs << "LLSDRPCServer::process_impl" << llendl; // Once we have all the data, We need to read the sd on // the the in channel, and respond on the out channel @@ -253,7 +249,6 @@ ESDRPCSStatus LLSDRPCServer::callMethod( const LLChannelDescriptors& channels, LLBufferArray* response) { - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); // Try to find the method in the method table. ESDRPCSStatus rv = ESDRPCS_DONE; method_map_t::iterator it = mMethods.find(method); @@ -292,7 +287,6 @@ ESDRPCSStatus LLSDRPCServer::callbackMethod( const LLChannelDescriptors& channels, LLBufferArray* response) { - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); // Try to find the method in the callback method table. ESDRPCSStatus rv = ESDRPCS_DONE; method_map_t::iterator it = mCallbackMethods.find(method); @@ -320,7 +314,6 @@ void LLSDRPCServer::buildFault( S32 code, const std::string& msg) { - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); LLBufferStream ostr(channels, data); ostr << FAULT_PART_1 << code << FAULT_PART_2 << msg << FAULT_PART_3; llinfos << "LLSDRPCServer::buildFault: " << code << ", " << msg << llendl; @@ -332,7 +325,6 @@ void LLSDRPCServer::buildResponse( LLBufferArray* data, const LLSD& response) { - LLMemType m1(LLMemType::MTYPE_IO_SD_SERVER); LLBufferStream ostr(channels, data); ostr << RESPONSE_PART_1; LLSDSerialize::toNotation(response, ostr); diff --git a/indra/llmessage/llurlrequest.cpp b/indra/llmessage/llurlrequest.cpp index fa03bb7512..227efdb07a 100644 --- a/indra/llmessage/llurlrequest.cpp +++ b/indra/llmessage/llurlrequest.cpp @@ -34,7 +34,6 @@ #include <openssl/ssl.h> #include "llcurl.h" #include "llioutil.h" -#include "llmemtype.h" #include "llproxy.h" #include "llpumpio.h" #include "llsd.h" @@ -64,7 +63,7 @@ public: ~LLURLRequestDetail(); std::string mURL; LLCurlEasyRequest* mCurlRequest; - LLBufferArray* mResponseBuffer; + LLIOPipe::buffer_ptr_t mResponseBuffer; LLChannelDescriptors mChannels; U8* mLastRead; U32 mBodyLimit; @@ -75,22 +74,24 @@ public: LLURLRequestDetail::LLURLRequestDetail() : mCurlRequest(NULL), - mResponseBuffer(NULL), mLastRead(NULL), mBodyLimit(0), mByteAccumulator(0), mIsBodyLimitSet(false), mSSLVerifyCallback(NULL) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mCurlRequest = new LLCurlEasyRequest(); + + if(!mCurlRequest->isValid()) //failed. + { + delete mCurlRequest ; + mCurlRequest = NULL ; + } } LLURLRequestDetail::~LLURLRequestDetail() { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); delete mCurlRequest; - mResponseBuffer = NULL; mLastRead = NULL; } @@ -152,7 +153,6 @@ std::string LLURLRequest::actionAsVerb(LLURLRequest::ERequestAction action) LLURLRequest::LLURLRequest(LLURLRequest::ERequestAction action) : mAction(action) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); initialize(); } @@ -161,20 +161,18 @@ LLURLRequest::LLURLRequest( const std::string& url) : mAction(action) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); initialize(); setURL(url); } LLURLRequest::~LLURLRequest() { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); delete mDetail; + mDetail = NULL ; } void LLURLRequest::setURL(const std::string& url) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mDetail->mURL = url; } @@ -185,7 +183,6 @@ std::string LLURLRequest::getURL() const void LLURLRequest::addHeader(const char* header) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mDetail->mCurlRequest->slist_append(header); } @@ -197,7 +194,6 @@ void LLURLRequest::setBodyLimit(U32 size) void LLURLRequest::setCallback(LLURLRequestComplete* callback) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mCompletionCallback = callback; mDetail->mCurlRequest->setHeaderCallback(&headerCallback, (void*)callback); } @@ -251,12 +247,22 @@ void LLURLRequest::allowCookies() mDetail->mCurlRequest->setoptString(CURLOPT_COOKIEFILE, ""); } +//virtual +bool LLURLRequest::isValid() +{ + return mDetail->mCurlRequest && mDetail->mCurlRequest->isValid(); +} + // virtual LLIOPipe::EStatus LLURLRequest::handleError( LLIOPipe::EStatus status, LLPumpIO* pump) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); + if(!isValid()) + { + return STATUS_EXPIRED ; + } + if(mCompletionCallback && pump) { LLURLRequestComplete* complete = NULL; @@ -272,6 +278,8 @@ LLIOPipe::EStatus LLURLRequest::handleError( } static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST("URL Request"); +static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST_GET_RESULT("Get Result"); +static LLFastTimer::DeclareTimer FTM_URL_PERFORM("Perform"); // virtual LLIOPipe::EStatus LLURLRequest::process_impl( @@ -283,7 +291,6 @@ LLIOPipe::EStatus LLURLRequest::process_impl( { LLFastTimer t(FTM_PROCESS_URL_REQUEST); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); //llinfos << "LLURLRequest::process_impl()" << llendl; if (!buffer) return STATUS_ERROR; @@ -325,7 +332,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( // *FIX: bit of a hack, but it should work. The configure and // callback method expect this information to be ready. - mDetail->mResponseBuffer = buffer.get(); + mDetail->mResponseBuffer = buffer; mDetail->mChannels = channels; if(!configure()) { @@ -341,18 +348,19 @@ LLIOPipe::EStatus LLURLRequest::process_impl( { PUMP_DEBUG; LLIOPipe::EStatus status = STATUS_BREAK; - static LLFastTimer::DeclareTimer FTM_URL_PERFORM("Perform"); { LLFastTimer t(FTM_URL_PERFORM); - mDetail->mCurlRequest->perform(); + if(!mDetail->mCurlRequest->wait()) + { + return status ; + } } - while(1) + bool keep_looping = true; + while(keep_looping) { CURLcode result; - static LLFastTimer::DeclareTimer FTM_PROCESS_URL_REQUEST_GET_RESULT("Get Result"); - bool newmsg = false; { LLFastTimer t(FTM_PROCESS_URL_REQUEST_GET_RESULT); @@ -401,8 +409,9 @@ LLIOPipe::EStatus LLURLRequest::process_impl( case CURLE_FAILED_INIT: case CURLE_COULDNT_CONNECT: status = STATUS_NO_CONNECTION; + keep_looping = false; break; - default: + default: // CURLE_URL_MALFORMAT llwarns << "URLRequest Error: " << result << ", " << LLCurl::strerror(result) @@ -410,6 +419,7 @@ LLIOPipe::EStatus LLURLRequest::process_impl( << (mDetail->mURL.empty() ? "<EMPTY URL>" : mDetail->mURL) << llendl; status = STATUS_ERROR; + keep_looping = false; break; } } @@ -436,9 +446,14 @@ LLIOPipe::EStatus LLURLRequest::process_impl( void LLURLRequest::initialize() { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mState = STATE_INITIALIZED; mDetail = new LLURLRequestDetail; + + if(!isValid()) + { + return ; + } + mDetail->mCurlRequest->setopt(CURLOPT_NOSIGNAL, 1); mDetail->mCurlRequest->setWriteCallback(&downCallback, (void*)this); mDetail->mCurlRequest->setReadCallback(&upCallback, (void*)this); @@ -451,7 +466,6 @@ bool LLURLRequest::configure() { LLFastTimer t(FTM_URL_REQUEST_CONFIGURE); - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); bool rv = false; S32 bytes = mDetail->mResponseBuffer->countAfter( mDetail->mChannels.in(), @@ -531,7 +545,6 @@ size_t LLURLRequest::downCallback( size_t nmemb, void* user) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); LLURLRequest* req = (LLURLRequest*)user; if(STATE_WAITING_FOR_RESPONSE == req->mState) { @@ -567,7 +580,6 @@ size_t LLURLRequest::upCallback( size_t nmemb, void* user) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); LLURLRequest* req = (LLURLRequest*)user; S32 bytes = llmin( (S32)(size * nmemb), @@ -665,7 +677,6 @@ LLIOPipe::EStatus LLContextURLExtractor::process_impl( { LLFastTimer t(FTM_PROCESS_URL_EXTRACTOR); PUMP_DEBUG; - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); // The destination host is in the context. if(context.isUndefined() || !mRequest) { @@ -693,13 +704,11 @@ LLIOPipe::EStatus LLContextURLExtractor::process_impl( LLURLRequestComplete::LLURLRequestComplete() : mRequestStatus(LLIOPipe::STATUS_ERROR) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); } // virtual LLURLRequestComplete::~LLURLRequestComplete() { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); } //virtual @@ -738,7 +747,6 @@ void LLURLRequestComplete::noResponse() void LLURLRequestComplete::responseStatus(LLIOPipe::EStatus status) { - LLMemType m1(LLMemType::MTYPE_IO_URL_REQUEST); mRequestStatus = status; } diff --git a/indra/llmessage/llurlrequest.h b/indra/llmessage/llurlrequest.h index ec5c2c1941..44d358d906 100644 --- a/indra/llmessage/llurlrequest.h +++ b/indra/llmessage/llurlrequest.h @@ -188,6 +188,8 @@ public: */ void allowCookies(); + /*virtual*/ bool isValid() ; + public: /** * @brief Give this pipe a chance to handle a generated error diff --git a/indra/llmessage/llxfer.h b/indra/llmessage/llxfer.h index 989e8b2cab..f9348eb11f 100644 --- a/indra/llmessage/llxfer.h +++ b/indra/llmessage/llxfer.h @@ -29,6 +29,7 @@ #include "message.h" #include "lltimer.h" +#include "llextendedstatus.h" const S32 LL_XFER_LARGE_PAYLOAD = 7680; diff --git a/indra/llmessage/llxfermanager.cpp b/indra/llmessage/llxfermanager.cpp index b9cddc8e45..00b9d81611 100644 --- a/indra/llmessage/llxfermanager.cpp +++ b/indra/llmessage/llxfermanager.cpp @@ -886,8 +886,17 @@ void LLXferManager::processFileRequest (LLMessageSystem *mesgsys, void ** /*user return; } - - std::string expanded_filename = gDirUtilp->getExpandedFilename( local_path, local_filename ); + // If we want to use a special path (e.g. LL_PATH_CACHE), we want to make sure we create the + // proper expanded filename. + std::string expanded_filename; + if (local_path != LL_PATH_NONE) + { + expanded_filename = gDirUtilp->getExpandedFilename( local_path, local_filename ); + } + else + { + expanded_filename = local_filename; + } llinfos << "starting file transfer: " << expanded_filename << " to " << mesgsys->getSender() << llendl; BOOL delete_local_on_completion = FALSE; diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index d0b0e178b8..ae95087377 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -80,7 +80,6 @@ #include "v3math.h" #include "v4math.h" #include "lltransfertargetvfile.h" -#include "llmemtype.h" // Constants //const char* MESSAGE_LOG_FILENAME = "message.log"; @@ -793,7 +792,6 @@ S32 LLMessageSystem::getReceiveBytes() const void LLMessageSystem::processAcks() { - LLMemType mt_pa(LLMemType::MTYPE_MESSAGE_PROCESS_ACKS); F64 mt_sec = getMessageTimeSeconds(); { gTransferManager.updateTransfers(); @@ -3147,7 +3145,7 @@ bool LLMessageSystem::generateDigestForWindowAndUUIDs(char* digest, const S32 wi LL_ERRS("Messaging") << "Trying to generate complex digest on a machine without a shared secret!" << llendl; } - U32 now = time(NULL); + U32 now = (U32)time(NULL); now /= window; @@ -3167,7 +3165,7 @@ bool LLMessageSystem::isMatchingDigestForWindowAndUUIDs(const char* digest, cons } char our_digest[MD5HEX_STR_SIZE]; /* Flawfinder: ignore */ - U32 now = time(NULL); + U32 now = (U32)time(NULL); now /= window; @@ -3213,7 +3211,7 @@ bool LLMessageSystem::generateDigestForWindow(char* digest, const S32 window) co LL_ERRS("Messaging") << "Trying to generate simple digest on a machine without a shared secret!" << llendl; } - U32 now = time(NULL); + U32 now = (U32)time(NULL); now /= window; @@ -4020,7 +4018,6 @@ void LLMessageSystem::setTimeDecodesSpamThreshold( F32 seconds ) // TODO: babbage: move gServicePump in to LLMessageSystem? bool LLMessageSystem::checkAllMessages(S64 frame_count, LLPumpIO* http_pump) { - LLMemType mt_cam(LLMemType::MTYPE_MESSAGE_CHECK_ALL); if(checkMessages(frame_count)) { return true; diff --git a/indra/llmessage/message_prehash.cpp b/indra/llmessage/message_prehash.cpp index e71fb96540..d7658862da 100644 --- a/indra/llmessage/message_prehash.cpp +++ b/indra/llmessage/message_prehash.cpp @@ -943,7 +943,6 @@ char const* const _PREHASH_SysGPU = LLMessageStringTable::getInstance()->getStri char const* const _PREHASH_AvatarInterestsReply = LLMessageStringTable::getInstance()->getString("AvatarInterestsReply"); char const* const _PREHASH_StartLure = LLMessageStringTable::getInstance()->getString("StartLure"); char const* const _PREHASH_SysRAM = LLMessageStringTable::getInstance()->getString("SysRAM"); -char const* const _PREHASH_ObjectPosition = LLMessageStringTable::getInstance()->getString("ObjectPosition"); char const* const _PREHASH_SitPosition = LLMessageStringTable::getInstance()->getString("SitPosition"); char const* const _PREHASH_StartTime = LLMessageStringTable::getInstance()->getString("StartTime"); char const* const _PREHASH_BornOn = LLMessageStringTable::getInstance()->getString("BornOn"); @@ -999,7 +998,6 @@ char const* const _PREHASH_SnapshotID = LLMessageStringTable::getInstance()->get char const* const _PREHASH_Aspect = LLMessageStringTable::getInstance()->getString("Aspect"); char const* const _PREHASH_ParamSize = LLMessageStringTable::getInstance()->getString("ParamSize"); char const* const _PREHASH_VoteCast = LLMessageStringTable::getInstance()->getString("VoteCast"); -char const* const _PREHASH_CastsShadows = LLMessageStringTable::getInstance()->getString("CastsShadows"); char const* const _PREHASH_EveryoneMask = LLMessageStringTable::getInstance()->getString("EveryoneMask"); char const* const _PREHASH_ObjectSpinUpdate = LLMessageStringTable::getInstance()->getString("ObjectSpinUpdate"); char const* const _PREHASH_MaturePublish = LLMessageStringTable::getInstance()->getString("MaturePublish"); @@ -1048,7 +1046,6 @@ char const* const _PREHASH_SimIP = LLMessageStringTable::getInstance()->getStrin char const* const _PREHASH_GodID = LLMessageStringTable::getInstance()->getString("GodID"); char const* const _PREHASH_TeleportMinPrice = LLMessageStringTable::getInstance()->getString("TeleportMinPrice"); char const* const _PREHASH_VoteItem = LLMessageStringTable::getInstance()->getString("VoteItem"); -char const* const _PREHASH_ObjectRotation = LLMessageStringTable::getInstance()->getString("ObjectRotation"); char const* const _PREHASH_SitRotation = LLMessageStringTable::getInstance()->getString("SitRotation"); char const* const _PREHASH_SnapSelection = LLMessageStringTable::getInstance()->getString("SnapSelection"); char const* const _PREHASH_SoundTrigger = LLMessageStringTable::getInstance()->getString("SoundTrigger"); diff --git a/indra/llmessage/message_prehash.h b/indra/llmessage/message_prehash.h index dd2c2dbd64..da2b613f53 100644 --- a/indra/llmessage/message_prehash.h +++ b/indra/llmessage/message_prehash.h @@ -943,7 +943,6 @@ extern char const* const _PREHASH_SysGPU; extern char const* const _PREHASH_AvatarInterestsReply; extern char const* const _PREHASH_StartLure; extern char const* const _PREHASH_SysRAM; -extern char const* const _PREHASH_ObjectPosition; extern char const* const _PREHASH_SitPosition; extern char const* const _PREHASH_StartTime; extern char const* const _PREHASH_BornOn; @@ -999,7 +998,6 @@ extern char const* const _PREHASH_SnapshotID; extern char const* const _PREHASH_Aspect; extern char const* const _PREHASH_ParamSize; extern char const* const _PREHASH_VoteCast; -extern char const* const _PREHASH_CastsShadows; extern char const* const _PREHASH_EveryoneMask; extern char const* const _PREHASH_ObjectSpinUpdate; extern char const* const _PREHASH_MaturePublish; @@ -1048,7 +1046,6 @@ extern char const* const _PREHASH_SimIP; extern char const* const _PREHASH_GodID; extern char const* const _PREHASH_TeleportMinPrice; extern char const* const _PREHASH_VoteItem; -extern char const* const _PREHASH_ObjectRotation; extern char const* const _PREHASH_SitRotation; extern char const* const _PREHASH_SnapSelection; extern char const* const _PREHASH_SoundTrigger; diff --git a/indra/llmessage/tests/llcurl_stub.cpp b/indra/llmessage/tests/llcurl_stub.cpp index d84fe0a49f..9b298d0c04 100644 --- a/indra/llmessage/tests/llcurl_stub.cpp +++ b/indra/llmessage/tests/llcurl_stub.cpp @@ -28,7 +28,6 @@ #include "llcurl.h" LLCurl::Responder::Responder() - : mReferenceCount(0) { } @@ -77,19 +76,3 @@ void LLCurl::Responder::result(LLSD const&) { } -namespace boost -{ - void intrusive_ptr_add_ref(LLCurl::Responder* p) - { - ++p->mReferenceCount; - } - - void intrusive_ptr_release(LLCurl::Responder* p) - { - if(p && 0 == --p->mReferenceCount) - { - delete p; - } - } -}; - diff --git a/indra/llmessage/tests/llhttpclient_test.cpp b/indra/llmessage/tests/llhttpclient_test.cpp new file mode 100644 index 0000000000..7c3def6024 --- /dev/null +++ b/indra/llmessage/tests/llhttpclient_test.cpp @@ -0,0 +1,382 @@ +/** + * @file llhttpclient_test.cpp + * @brief Testing the HTTP client classes. + * + * $LicenseInfo:firstyear=2006&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +/** + * + * These classes test the HTTP client framework. + * + */ + +#include <tut/tut.hpp> +#include "linden_common.h" + +#include "lltut.h" +#include "llhttpclient.h" +#include "llformat.h" +#include "llpipeutil.h" +#include "llproxy.h" +#include "llpumpio.h" + +#include "llsdhttpserver.h" +#include "lliohttpserver.h" +#include "lliosocket.h" +#include "stringize.h" + +namespace tut +{ + LLSD storage; + + class LLSDStorageNode : public LLHTTPNode + { + public: + LLSD simpleGet() const { return storage; } + LLSD simplePut(const LLSD& value) const { storage = value; return LLSD(); } + }; + + class ErrorNode : public LLHTTPNode + { + public: + void get(ResponsePtr r, const LLSD& context) const + { r->status(599, "Intentional error"); } + void post(ResponsePtr r, const LLSD& context, const LLSD& input) const + { r->status(input["status"], input["reason"]); } + }; + + class TimeOutNode : public LLHTTPNode + { + public: + void get(ResponsePtr r, const LLSD& context) const + { + /* do nothing, the request will eventually time out */ + } + }; + + LLHTTPRegistration<LLSDStorageNode> gStorageNode("/test/storage"); + LLHTTPRegistration<ErrorNode> gErrorNode("/test/error"); + LLHTTPRegistration<TimeOutNode> gTimeOutNode("/test/timeout"); + + struct HTTPClientTestData + { + public: + HTTPClientTestData(): + PORT(getenv("PORT")), + // Turning NULL PORT into empty string doesn't make things work; + // that's just to keep this initializer from blowing up. We test + // PORT separately in the constructor body. + local_server(STRINGIZE("http://127.0.0.1:" << (PORT? PORT : "") << "/")) + { + ensure("Set environment variable PORT to local test server port", PORT); + apr_pool_create(&mPool, NULL); + LLCurl::initClass(false); + mServerPump = new LLPumpIO(mPool); + mClientPump = new LLPumpIO(mPool); + + LLHTTPClient::setPump(*mClientPump); + } + + ~HTTPClientTestData() + { + delete mServerPump; + delete mClientPump; + LLProxy::cleanupClass(); + apr_pool_destroy(mPool); + } + + void setupTheServer() + { + LLHTTPNode& root = LLIOHTTPServer::create(mPool, *mServerPump, 8888); + + LLHTTPStandardServices::useServices(); + LLHTTPRegistrar::buildAllServices(root); + } + + void runThePump(float timeout = 100.0f) + { + LLTimer timer; + timer.setTimerExpirySec(timeout); + + while(!mSawCompleted && !mSawCompletedHeader && !timer.hasExpired()) + { + if (mServerPump) + { + mServerPump->pump(); + mServerPump->callback(); + } + if (mClientPump) + { + mClientPump->pump(); + mClientPump->callback(); + } + } + } + + void killServer() + { + delete mServerPump; + mServerPump = NULL; + } + + const char* const PORT; + const std::string local_server; + + private: + apr_pool_t* mPool; + LLPumpIO* mServerPump; + LLPumpIO* mClientPump; + + protected: + void ensureStatusOK() + { + if (mSawError) + { + std::string msg = + llformat("error() called when not expected, status %d", + mStatus); + fail(msg); + } + } + + void ensureStatusError() + { + if (!mSawError) + { + fail("error() wasn't called"); + } + } + + LLSD getResult() + { + return mResult; + } + LLSD getHeader() + { + return mHeader; + } + + protected: + bool mSawError; + U32 mStatus; + std::string mReason; + bool mSawCompleted; + bool mSawCompletedHeader; + LLSD mResult; + LLSD mHeader; + bool mResultDeleted; + + class Result : public LLHTTPClient::Responder + { + protected: + Result(HTTPClientTestData& client) + : mClient(client) + { + } + + public: + static Result* build(HTTPClientTestData& client) + { + return new Result(client); + } + + ~Result() + { + mClient.mResultDeleted = true; + } + + virtual void error(U32 status, const std::string& reason) + { + mClient.mSawError = true; + mClient.mStatus = status; + mClient.mReason = reason; + } + + virtual void result(const LLSD& content) + { + mClient.mResult = content; + } + + virtual void completed( + U32 status, const std::string& reason, + const LLSD& content) + { + LLHTTPClient::Responder::completed(status, reason, content); + + mClient.mSawCompleted = true; + } + + virtual void completedHeader( + U32 status, const std::string& reason, + const LLSD& content) + { + mClient.mHeader = content; + mClient.mSawCompletedHeader = true; + } + + private: + HTTPClientTestData& mClient; + }; + + friend class Result; + + protected: + LLHTTPClient::ResponderPtr newResult() + { + mSawError = false; + mStatus = 0; + mSawCompleted = false; + mSawCompletedHeader = false; + mResult.clear(); + mHeader.clear(); + mResultDeleted = false; + + return Result::build(*this); + } + }; + + + typedef test_group<HTTPClientTestData> HTTPClientTestGroup; + typedef HTTPClientTestGroup::object HTTPClientTestObject; + HTTPClientTestGroup httpClientTestGroup("http_client"); + + template<> template<> + void HTTPClientTestObject::test<1>() + { + LLHTTPClient::get(local_server, newResult()); + runThePump(); + ensureStatusOK(); + ensure("result object wasn't destroyed", mResultDeleted); + } + + template<> template<> + void HTTPClientTestObject::test<2>() + { + // Please nobody listen on this particular port... + LLHTTPClient::get("http://127.0.0.1:7950", newResult()); + runThePump(); + ensureStatusError(); + } + + template<> template<> + void HTTPClientTestObject::test<3>() + { + LLSD sd; + + sd["list"][0]["one"] = 1; + sd["list"][0]["two"] = 2; + sd["list"][1]["three"] = 3; + sd["list"][1]["four"] = 4; + + setupTheServer(); + + LLHTTPClient::post("http://localhost:8888/web/echo", sd, newResult()); + runThePump(); + ensureStatusOK(); + ensure_equals("echoed result matches", getResult(), sd); + } + + template<> template<> + void HTTPClientTestObject::test<4>() + { + LLSD sd; + + sd["message"] = "This is my test message."; + + setupTheServer(); + LLHTTPClient::put("http://localhost:8888/test/storage", sd, newResult()); + runThePump(); + ensureStatusOK(); + + LLHTTPClient::get("http://localhost:8888/test/storage", newResult()); + runThePump(); + ensureStatusOK(); + ensure_equals("echoed result matches", getResult(), sd); + + } + + template<> template<> + void HTTPClientTestObject::test<5>() + { + LLSD sd; + sd["status"] = 543; + sd["reason"] = "error for testing"; + + setupTheServer(); + + LLHTTPClient::post("http://localhost:8888/test/error", sd, newResult()); + runThePump(); + ensureStatusError(); + ensure_contains("reason", mReason, sd["reason"]); + } + + template<> template<> + void HTTPClientTestObject::test<6>() + { + setupTheServer(); + + LLHTTPClient::get("http://localhost:8888/test/timeout", newResult()); + runThePump(1.0f); + killServer(); + runThePump(); + ensureStatusError(); + ensure_equals("reason", mReason, "STATUS_ERROR"); + } + + template<> template<> + void HTTPClientTestObject::test<7>() + { + // Can not use the little mini server. The blocking request + // won't ever let it run. Instead get from a known LLSD + // source and compare results with the non-blocking get which + // is tested against the mini server earlier. + LLHTTPClient::get(local_server, newResult()); + runThePump(); + ensureStatusOK(); + LLSD expected = getResult(); + + LLSD result; + result = LLHTTPClient::blockingGet(local_server); + LLSD body = result["body"]; + ensure_equals("echoed result matches", body.size(), expected.size()); + } + template<> template<> + void HTTPClientTestObject::test<8>() + { + // This is testing for the presence of the Header in the returned results + // from an HTTP::get call. + LLHTTPClient::get(local_server, newResult()); + runThePump(); + ensureStatusOK(); + LLSD header = getHeader(); + ensure("got a header", ! header.emptyMap().asBoolean()); + } + template<> template<> + void HTTPClientTestObject::test<9>() + { + LLHTTPClient::head(local_server, newResult()); + runThePump(); + ensureStatusOK(); + ensure("result object wasn't destroyed", mResultDeleted); + } +} diff --git a/indra/llmessage/tests/llsdmessage_test.cpp b/indra/llmessage/tests/llsdmessage_test.cpp index 0f2c069303..44b024a83f 100644 --- a/indra/llmessage/tests/llsdmessage_test.cpp +++ b/indra/llmessage/tests/llsdmessage_test.cpp @@ -42,6 +42,7 @@ // external library headers // other Linden headers #include "../test/lltut.h" +#include "../test/catch_and_store_what_in.h" #include "llsdserialize.h" #include "llevents.h" #include "stringize.h" @@ -72,43 +73,14 @@ namespace tut template<> template<> void llsdmessage_object::test<1>() { - bool threw = false; + std::string threw; // This should fail... try { LLSDMessage localListener; } - catch (const LLEventPump::DupPumpName&) - { - threw = true; - } - catch (const std::runtime_error& ex) - { - // This clause is because on Linux, on the viewer side, for this - // one test program (though not others!), the - // LLEventPump::DupPumpName exception isn't caught by the clause - // above. Warn the user... - std::cerr << "Failed to catch " << typeid(ex).name() << std::endl; - // But if the expected exception was thrown, allow the test to - // succeed anyway. Not sure how else to handle this odd case. - if (std::string(typeid(ex).name()) == typeid(LLEventPump::DupPumpName).name()) - { - threw = true; - } - else - { - // We don't even recognize this exception. Let it propagate - // out to TUT to fail the test. - throw; - } - } - catch (...) - { - std::cerr << "Utterly failed to catch expected exception!" << std::endl; - // This case is full of fail. We HAVE to address it. - throw; - } - ensure("second LLSDMessage should throw", threw); + CATCH_AND_STORE_WHAT_IN(threw, LLEventPump::DupPumpName) + ensure("second LLSDMessage should throw", ! threw.empty()); } template<> template<> @@ -143,7 +115,7 @@ namespace tut httpPump.post(request); ensure("got response", netio.pump()); ensure("success response", success); - ensure_equals(result.asString(), "success"); + ensure_equals(result["reply"].asString(), "success"); body["status"] = 499; body["reason"] = "custom error message"; diff --git a/indra/llmessage/tests/test_llsdmessage_peer.py b/indra/llmessage/tests/test_llsdmessage_peer.py index 22edd9dad8..fe4f3a8c01 100644 --- a/indra/llmessage/tests/test_llsdmessage_peer.py +++ b/indra/llmessage/tests/test_llsdmessage_peer.py @@ -78,25 +78,32 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): ## debug("root node tag %s", tree.getroot().tag) ## return llsd.to_python(tree.getroot()) - def do_GET(self): + def do_HEAD(self): + self.do_GET(withdata=False) + + def do_GET(self, withdata=True): # Of course, don't attempt to read data. - self.answer(dict(reply="success", status=500, - reason="Your GET operation requested failure")) + data = dict(reply="success", body="avatar", random=17) + self.answer(data, withdata=withdata) def do_POST(self): # Read the provided POST data. self.answer(self.read_xml()) - def answer(self, data): + def answer(self, data, withdata=True): debug("%s.answer(%s): self.path = %r", self.__class__.__name__, data, self.path) if "fail" not in self.path: - response = llsd.format_xml(data.get("reply", llsd.LLSD("success"))) + data = data.copy() # we're going to modify + # Ensure there's a "reply" key in data, even if there wasn't before + data["reply"] = data.get("reply", llsd.LLSD("success")) + response = llsd.format_xml(data) debug("success: %s", response) self.send_response(200) self.send_header("Content-type", "application/llsd+xml") self.send_header("Content-Length", str(len(response))) self.end_headers() - self.wfile.write(response) + if withdata: + self.wfile.write(response) else: # fail requested status = data.get("status", 500) # self.responses maps an int status to a (short, long) pair of diff --git a/indra/llmessage/tests/testrunner.py b/indra/llmessage/tests/testrunner.py index f2c841532a..5b9beb359b 100644 --- a/indra/llmessage/tests/testrunner.py +++ b/indra/llmessage/tests/testrunner.py @@ -35,7 +35,7 @@ import re import errno import socket -VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "1") # default to verbose +VERBOSE = os.environ.get("INTEGRATION_TEST_VERBOSE", "0") # default to quiet # Support usage such as INTEGRATION_TEST_VERBOSE=off -- distressing to user if # that construct actually turns on verbosity... VERBOSE = not re.match(r"(0|off|false|quiet)$", VERBOSE, re.IGNORECASE) |