diff options
Diffstat (limited to 'indra/llcommon')
74 files changed, 8129 insertions, 652 deletions
diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 9342a22d46..0a3eaec5c5 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -7,17 +7,16 @@ include(00-Common) include(LLCommon) include(Linking) include(Boost) -include(Pth) include(LLSharedLibs) include(GoogleBreakpad) include(GooglePerfTools) include(Copy3rdPartyLibs) +include(ZLIB) include_directories( ${EXPAT_INCLUDE_DIRS} ${LLCOMMON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS} - ${PTH_INCLUDE_DIRS} ) # add_executable(lltreeiterators lltreeiterators.cpp) @@ -62,6 +61,7 @@ set(llcommon_SOURCE_FILES llformat.cpp llframetimer.cpp llheartbeat.cpp + llinstancetracker.cpp llliveappconfig.cpp lllivefile.cpp lllog.cpp @@ -115,6 +115,7 @@ set(llcommon_HEADER_FILES indra_constants.h linden_common.h linked_lists.h + llaccountingcost.h llallocator.h llallocator_heap_profile.h llagentconstants.h @@ -266,6 +267,10 @@ if(LLCOMMON_LINK_SHARED) add_definitions(-fPIC) endif(WINDOWS) endif(NOT WORD_SIZE EQUAL 32) + if(WINDOWS) + # always generate llcommon.pdb, even for "Release" builds + set_target_properties(llcommon PROPERTIES LINK_FLAGS "/DEBUG") + endif(WINDOWS) ll_stage_sharedlib(llcommon) else(LLCOMMON_LINK_SHARED) add_library (llcommon ${llcommon_SOURCE_FILES}) @@ -281,10 +286,15 @@ target_link_libraries( ${WINDOWS_LIBRARIES} ${BOOST_PROGRAM_OPTIONS_LIBRARY} ${BOOST_REGEX_LIBRARY} - ${PTH_LIBRARIES} ${GOOGLE_PERFTOOLS_LIBRARIES} ) +if (DARWIN) + include(CMakeFindFrameworks) + find_library(CARBON_LIBRARY Carbon) + target_link_libraries(llcommon ${CARBON_LIBRARY}) +endif (DARWIN) + add_dependencies(llcommon stage_third_party_libs) if (LL_TESTS) @@ -307,12 +317,15 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(lllazy "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}") - LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}" + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/setpython.py") + LL_ADD_INTEGRATION_TEST(llsingleton "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llstring "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lltreeiterators "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lluri "" "${test_libs}") LL_ADD_INTEGRATION_TEST(reflection "" "${test_libs}") LL_ADD_INTEGRATION_TEST(stringize "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(lleventdispatcher "" "${test_libs}") # *TODO - reenable these once tcmalloc libs no longer break the build. #ADD_BUILD_TEST(llallocator llcommon) diff --git a/indra/llcommon/indra_constants.h b/indra/llcommon/indra_constants.h index 95cb606240..0745696ef3 100644 --- a/indra/llcommon/indra_constants.h +++ b/indra/llcommon/indra_constants.h @@ -312,6 +312,14 @@ const F32 CHAT_SHOUT_RADIUS = 100.f; const F32 CHAT_MAX_RADIUS = CHAT_SHOUT_RADIUS; const F32 CHAT_MAX_RADIUS_BY_TWO = CHAT_MAX_RADIUS / 2.f; +// squared editions of the above for distance checks +const F32 CHAT_WHISPER_RADIUS_SQUARED = CHAT_WHISPER_RADIUS * CHAT_WHISPER_RADIUS; +const F32 CHAT_NORMAL_RADIUS_SQUARED = CHAT_NORMAL_RADIUS * CHAT_NORMAL_RADIUS; +const F32 CHAT_SHOUT_RADIUS_SQUARED = CHAT_SHOUT_RADIUS * CHAT_SHOUT_RADIUS; +const F32 CHAT_MAX_RADIUS_SQUARED = CHAT_SHOUT_RADIUS_SQUARED; +const F32 CHAT_MAX_RADIUS_BY_TWO_SQUARED = CHAT_MAX_RADIUS_BY_TWO * CHAT_MAX_RADIUS_BY_TWO; + + // this times above gives barely audible radius const F32 CHAT_BARELY_AUDIBLE_FACTOR = 2.0f; @@ -379,8 +387,6 @@ const S32 MAP_SIM_RETURN_NULL_SIMS = 0x00010000; const S32 MAP_SIM_PRELUDE = 0x00020000; // Crash reporter behavior -const char* const CRASH_SETTINGS_FILE = "settings_crash_behavior.xml"; -const char* const CRASH_BEHAVIOR_SETTING = "CrashSubmitBehavior"; const S32 CRASH_BEHAVIOR_ASK = 0; const S32 CRASH_BEHAVIOR_ALWAYS_SEND = 1; const S32 CRASH_BEHAVIOR_NEVER_SEND = 2; diff --git a/indra/llcommon/linden_common.h b/indra/llcommon/linden_common.h index bdcc98e402..5cfcdab41c 100644 --- a/indra/llcommon/linden_common.h +++ b/indra/llcommon/linden_common.h @@ -52,34 +52,11 @@ #include <ctime> #include <iosfwd> -// Work around Microsoft compiler warnings in STL headers -#ifdef LL_WINDOWS -#pragma warning (disable : 4702) // unreachable code -#pragma warning (disable : 4244) // conversion from time_t to S32 -#endif // LL_WINDOWS - -// *TODO: Eliminate these, most library .cpp files don't need them. -// Add them to llviewerprecompiledheaders.h if necessary. -#include <list> -#include <map> -#include <vector> -#include <string> - -#ifdef LL_WINDOWS -// Reenable warnings we disabled above -#pragma warning (3 : 4702) // unreachable code, we like level 3, not 4 -// moved msvc warnings to llpreprocessor.h *TODO - delete this comment after merge conflicts are unlikely -brad -#endif // LL_WINDOWS - // Linden only libs in alpha-order other than stdtypes.h // *NOTE: Please keep includes here to a minimum, see above. #include "stdtypes.h" #include "lldefs.h" #include "llerror.h" -#include "llextendedstatus.h" -// Don't do this, adds 15K lines of header code to every library file. -//#include "llfasttimer.h" #include "llfile.h" -#include "llformat.h" #endif diff --git a/indra/llcommon/llaccountingcost.h b/indra/llcommon/llaccountingcost.h new file mode 100644 index 0000000000..0ef3b50c6d --- /dev/null +++ b/indra/llcommon/llaccountingcost.h @@ -0,0 +1,86 @@ +/** + * @file llaccountingcost.h + * @ + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_ACCOUNTINGQUOTA_H +#define LL_ACCOUNTINGQUOTA_H + +struct ParcelQuota +{ + ParcelQuota( F32 ownerRenderCost, F32 ownerPhysicsCost, F32 ownerNetworkCost, F32 ownerSimulationCost, + F32 groupRenderCost, F32 groupPhysicsCost, F32 groupNetworkCost, F32 groupSimulationCost, + F32 otherRenderCost, F32 otherPhysicsCost, F32 otherNetworkCost, F32 otherSimulationCost, + F32 tempRenderCost, F32 tempPhysicsCost, F32 tempNetworkCost, F32 tempSimulationCost, + F32 selectedRenderCost, F32 selectedPhysicsCost, F32 selectedNetworkCost, F32 selectedSimulationCost, + F32 parcelCapacity ) + : mOwnerRenderCost( ownerRenderCost ), mOwnerPhysicsCost( ownerPhysicsCost ) + , mOwnerNetworkCost( ownerNetworkCost ), mOwnerSimulationCost( ownerSimulationCost ) + , mGroupRenderCost( groupRenderCost ), mGroupPhysicsCost( groupPhysicsCost ) + , mGroupNetworkCost( groupNetworkCost ), mGroupSimulationCost( groupSimulationCost ) + , mOtherRenderCost( otherRenderCost ), mOtherPhysicsCost( otherPhysicsCost ) + , mOtherNetworkCost( otherNetworkCost ), mOtherSimulationCost( otherSimulationCost ) + , mTempRenderCost( tempRenderCost ), mTempPhysicsCost( tempPhysicsCost ) + , mTempNetworkCost( tempNetworkCost ), mTempSimulationCost( tempSimulationCost ) + , mSelectedRenderCost( tempRenderCost ), mSelectedPhysicsCost( tempPhysicsCost ) + , mSelectedNetworkCost( tempNetworkCost ), mSelectedSimulationCost( selectedSimulationCost ) + , mParcelCapacity( parcelCapacity ) + { + } + + ParcelQuota(){} + F32 mOwnerRenderCost, mOwnerPhysicsCost, mOwnerNetworkCost, mOwnerSimulationCost; + F32 mGroupRenderCost, mGroupPhysicsCost, mGroupNetworkCost, mGroupSimulationCost; + F32 mOtherRenderCost, mOtherPhysicsCost, mOtherNetworkCost, mOtherSimulationCost; + F32 mTempRenderCost, mTempPhysicsCost, mTempNetworkCost, mTempSimulationCost; + F32 mSelectedRenderCost, mSelectedPhysicsCost, mSelectedNetworkCost, mSelectedSimulationCost; + F32 mParcelCapacity; +}; + +//SelectionQuota atm does not require a id +struct SelectionCost +{ + SelectionCost( /*LLTransactionID transactionId, */ F32 physicsCost, F32 networkCost, F32 simulationCost ) + //: mTransactionId( transactionId) + : mPhysicsCost( physicsCost ) + , mNetworkCost( networkCost ) + , mSimulationCost( simulationCost ) + { + } + SelectionCost() + : mPhysicsCost( 0.0f ) + , mNetworkCost( 0.0f ) + , mSimulationCost( 0.0f ) + {} + + F32 mPhysicsCost, mNetworkCost, mSimulationCost; + //LLTransactionID mTransactionId; +}; + +typedef enum { Roots = 0 , Prims } eSelectionType; + +#endif + + + diff --git a/indra/llcommon/llapp.cpp b/indra/llcommon/llapp.cpp index 39daefd1ad..ed192a9975 100644 --- a/indra/llcommon/llapp.cpp +++ b/indra/llcommon/llapp.cpp @@ -24,6 +24,10 @@ * $/LicenseInfo$ */ +#include "linden_common.h" + +#include "llapp.h" + #include <cstdlib> #ifdef LL_DARWIN @@ -32,9 +36,6 @@ #include <sys/sysctl.h> #endif -#include "linden_common.h" -#include "llapp.h" - #include "llcommon.h" #include "llapr.h" #include "llerrorcontrol.h" diff --git a/indra/llcommon/llapr.cpp b/indra/llcommon/llapr.cpp index 66ec5bad2c..d1c44c9403 100644 --- a/indra/llcommon/llapr.cpp +++ b/indra/llcommon/llapr.cpp @@ -28,6 +28,7 @@ #include "linden_common.h" #include "llapr.h" +#include "apr_dso.h" apr_pool_t *gAPRPoolp = NULL; // Global APR memory pool LLVolatileAPRPool *LLAPRFile::sAPRFilePoolp = NULL ; //global volatile APR memory pool. @@ -279,14 +280,31 @@ bool ll_apr_warn_status(apr_status_t status) { if(APR_SUCCESS == status) return false; char buf[MAX_STRING]; /* Flawfinder: ignore */ - apr_strerror(status, buf, MAX_STRING); + apr_strerror(status, buf, sizeof(buf)); LL_WARNS("APR") << "APR: " << buf << LL_ENDL; return true; } +bool ll_apr_warn_status(apr_status_t status, apr_dso_handle_t *handle) +{ + bool result = ll_apr_warn_status(status); + // Despite observed truncation of actual Mac dylib load errors, increasing + // this buffer to more than MAX_STRING doesn't help: it appears that APR + // stores the output in a fixed 255-character internal buffer. (*sigh*) + char buf[MAX_STRING]; /* Flawfinder: ignore */ + apr_dso_error(handle, buf, sizeof(buf)); + LL_WARNS("APR") << "APR: " << buf << LL_ENDL; + return result; +} + void ll_apr_assert_status(apr_status_t status) { - llassert(ll_apr_warn_status(status) == false); + llassert(! ll_apr_warn_status(status)); +} + +void ll_apr_assert_status(apr_status_t status, apr_dso_handle_t *handle) +{ + llassert(! ll_apr_warn_status(status, handle)); } //--------------------------------------------------------------------- diff --git a/indra/llcommon/llapr.h b/indra/llcommon/llapr.h index 4930270af8..af33ce666f 100644 --- a/indra/llcommon/llapr.h +++ b/indra/llcommon/llapr.h @@ -53,6 +53,8 @@ extern LL_COMMON_API apr_thread_mutex_t* gLogMutexp; extern apr_thread_mutex_t* gCallStacksLogMutexp; +struct apr_dso_handle_t; + /** * @brief initialize the common apr constructs -- apr itself, the * global pool, and a mutex. @@ -259,8 +261,11 @@ public: * @return Returns <code>true</code> if status is an error condition. */ bool LL_COMMON_API ll_apr_warn_status(apr_status_t status); +/// There's a whole other APR error-message function if you pass a DSO handle. +bool LL_COMMON_API ll_apr_warn_status(apr_status_t status, apr_dso_handle_t* handle); void LL_COMMON_API ll_apr_assert_status(apr_status_t status); +void LL_COMMON_API ll_apr_assert_status(apr_status_t status, apr_dso_handle_t* handle); extern "C" LL_COMMON_API apr_pool_t* gAPRPoolp; // Global APR memory pool diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index eb610f625a..5e566d6c7c 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -93,8 +93,10 @@ LLAssetDictionary::LLAssetDictionary() addEntry(LLAssetType::AT_LINK, new AssetEntry("LINK", "link", "sym link", false, false, true)); addEntry(LLAssetType::AT_LINK_FOLDER, new AssetEntry("FOLDER_LINK", "link_f", "sym folder link", false, false, true)); + addEntry(LLAssetType::AT_MESH, new AssetEntry("MESH", "mesh", "mesh", false, false, false)); + addEntry(LLAssetType::AT_WIDGET, new AssetEntry("WIDGET", "widget", "widget", false, false, false)); + addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, FALSE, FALSE, FALSE)); - addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, false, false, false)); }; // static diff --git a/indra/llcommon/llassettype.h b/indra/llcommon/llassettype.h index c5ff2364cc..d538accbf7 100644 --- a/indra/llcommon/llassettype.h +++ b/indra/llcommon/llassettype.h @@ -109,7 +109,13 @@ public: AT_LINK_FOLDER = 25, // Inventory folder link - AT_COUNT = 26, + AT_WIDGET = 40, + // UI Widget: this is *not* an inventory asset type, only a viewer side asset (e.g. button, other ui items...) + + AT_MESH = 49, + // Mesh data in our proprietary SLM format + + AT_COUNT = 50, // +*********************************************************+ // | TO ADD AN ELEMENT TO THIS ENUM: | diff --git a/indra/llcommon/llavatarconstants.h b/indra/llcommon/llavatarconstants.h index 596b0643ef..f47f447b45 100644 --- a/indra/llcommon/llavatarconstants.h +++ b/indra/llcommon/llavatarconstants.h @@ -46,10 +46,10 @@ const U32 AVATAR_TRANSACTED = 0x1 << 3; // whether avatar has actively used p const U32 AVATAR_ONLINE = 0x1 << 4; // the online status of this avatar, if known. const U32 AVATAR_AGEVERIFIED = 0x1 << 5; // whether avatar has been age-verified -static const std::string VISIBILITY_DEFAULT("default"); -static const std::string VISIBILITY_HIDDEN("hidden"); -static const std::string VISIBILITY_VISIBLE("visible"); -static const std::string VISIBILITY_INVISIBLE("invisible"); +char const* const VISIBILITY_DEFAULT = "default"; +char const* const VISIBILITY_HIDDEN = "hidden"; +char const* const VISIBILITY_VISIBLE = "visible"; +char const* const VISIBILITY_INVISIBLE = "invisible"; #endif diff --git a/indra/llcommon/llavatarname.cpp b/indra/llcommon/llavatarname.cpp index b1ec9e9875..3206843bf4 100644 --- a/indra/llcommon/llavatarname.cpp +++ b/indra/llcommon/llavatarname.cpp @@ -48,7 +48,7 @@ LLAvatarName::LLAvatarName() mLegacyFirstName(), mLegacyLastName(), mIsDisplayNameDefault(false), - mIsDummy(false), + mIsTemporaryName(false), mExpires(F64_MAX), mNextUpdate(0.0) { } @@ -90,20 +90,27 @@ void LLAvatarName::fromLLSD(const LLSD& sd) std::string LLAvatarName::getCompleteName() const { std::string name; - if (!mUsername.empty()) + if (mUsername.empty() || mIsDisplayNameDefault) + // If the display name feature is off + // OR this particular display name is defaulted (i.e. based on user name), + // then display only the easier to read instance of the person's name. { - name = mDisplayName + " (" + mUsername + ")"; + name = mDisplayName; } else { - // ...display names are off, legacy name is in mDisplayName - name = mDisplayName; + name = mDisplayName + " (" + mUsername + ")"; } return name; } std::string LLAvatarName::getLegacyName() const { + if (mLegacyFirstName.empty() && mLegacyLastName.empty()) // display names disabled? + { + return mDisplayName; + } + std::string name; name.reserve( mLegacyFirstName.size() + 1 + mLegacyLastName.size() ); name = mLegacyFirstName; diff --git a/indra/llcommon/llavatarname.h b/indra/llcommon/llavatarname.h index 145aeccd35..ba258d6d52 100644 --- a/indra/llcommon/llavatarname.h +++ b/indra/llcommon/llavatarname.h @@ -79,7 +79,7 @@ public: // Under error conditions, we may insert "dummy" records with // names like "???" into caches as placeholders. These can be // shown in UI, but are not serialized. - bool mIsDummy; + bool mIsTemporaryName; // Names can change, so need to keep track of when name was // last checked. diff --git a/indra/llcommon/llchat.h b/indra/llcommon/llchat.h index 87c2d6775b..f5b242fdfc 100644 --- a/indra/llcommon/llchat.h +++ b/indra/llcommon/llchat.h @@ -49,7 +49,8 @@ typedef enum e_chat_type CHAT_TYPE_STOP = 5, CHAT_TYPE_DEBUG_MSG = 6, CHAT_TYPE_REGION = 7, - CHAT_TYPE_OWNER = 8 + CHAT_TYPE_OWNER = 8, + CHAT_TYPE_DIRECT = 9 // From llRegionSayTo() } EChatType; typedef enum e_chat_audible_level diff --git a/indra/llcommon/lldefs.h b/indra/llcommon/lldefs.h index 6b38de6500..5a4b8325f4 100644 --- a/indra/llcommon/lldefs.h +++ b/indra/llcommon/lldefs.h @@ -236,5 +236,13 @@ inline LLDATATYPE llclampb(const LLDATATYPE& a) return llmin(llmax(a, (LLDATATYPE)0), (LLDATATYPE)255); } +template <class LLDATATYPE> +inline void llswap(LLDATATYPE& lhs, LLDATATYPE& rhs) +{ + LLDATATYPE tmp = lhs; + lhs = rhs; + rhs = tmp; +} + #endif // LL_LLDEFS_H diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index bb64152407..e4381dbbd6 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -379,7 +379,7 @@ namespace { /* This pattern, of returning a reference to a static function variable, is to ensure that this global is constructed before - it is used, no matter what the global initializeation sequence + it is used, no matter what the global initialization sequence is. See C++ FAQ Lite, sections 10.12 through 10.14 */ @@ -617,6 +617,12 @@ namespace LLError s.defaultLevel = level; } + ELevel getDefaultLevel() + { + Settings& s = Settings::get(); + return s.defaultLevel; + } + void setFunctionLevel(const std::string& function_name, ELevel level) { Globals& g = Globals::get(); diff --git a/indra/llcommon/llerror.h b/indra/llcommon/llerror.h index 4a42241c4f..b3e604f8e8 100644 --- a/indra/llcommon/llerror.h +++ b/indra/llcommon/llerror.h @@ -39,7 +39,7 @@ Information for most users: - Code can log messages with constuctions like this: + Code can log messages with constructions like this: LL_INFOS("StringTag") << "request to fizzbip agent " << agent_id << " denied due to timeout" << LL_ENDL; @@ -47,9 +47,9 @@ Messages can be logged to one of four increasing levels of concern, using one of four "streams": - LL_DEBUGS("StringTag") - debug messages that are normally supressed - LL_INFOS("StringTag") - informational messages that are normall shown - LL_WARNS("StringTag") - warning messages that singal a problem + LL_DEBUGS("StringTag") - debug messages that are normally suppressed + LL_INFOS("StringTag") - informational messages that are normal shown + LL_WARNS("StringTag") - warning messages that signal a problem LL_ERRS("StringTag") - error messages that are major, unrecoverable failures The later (LL_ERRS("StringTag")) automatically crashes the process after the message @@ -90,7 +90,7 @@ WARN: LLFoo::doSomething: called with a big value for i: 283 - Which messages are logged and which are supressed can be controled at run + Which messages are logged and which are suppressed can be controlled at run time from the live file logcontrol.xml based on function, class and/or source file. See etc/logcontrol-dev.xml for details. @@ -106,7 +106,7 @@ namespace LLError enum ELevel { LEVEL_ALL = 0, - // used to indicate that all messagess should be logged + // used to indicate that all messages should be logged LEVEL_DEBUG = 0, LEVEL_INFO = 1, @@ -220,7 +220,7 @@ namespace LLError // See top of file for example of how to use this typedef LLError::NoClassInfo _LL_CLASS_TO_LOG; - // Outside a class declartion, or in class without LOG_CLASS(), this + // Outside a class declaration, or in class without LOG_CLASS(), this // typedef causes the messages to not be associated with any class. diff --git a/indra/llcommon/llerrorcontrol.h b/indra/llcommon/llerrorcontrol.h index fb75d45e2c..ed9de002f5 100644 --- a/indra/llcommon/llerrorcontrol.h +++ b/indra/llcommon/llerrorcontrol.h @@ -75,6 +75,7 @@ namespace LLError LL_COMMON_API void setPrintLocation(bool); LL_COMMON_API void setDefaultLevel(LLError::ELevel); + LL_COMMON_API ELevel getDefaultLevel(); LL_COMMON_API void setFunctionLevel(const std::string& function_name, LLError::ELevel); LL_COMMON_API void setClassLevel(const std::string& class_name, LLError::ELevel); LL_COMMON_API void setFileLevel(const std::string& file_name, LLError::ELevel); diff --git a/indra/llcommon/lleventapi.cpp b/indra/llcommon/lleventapi.cpp index 4270c8b511..ff5459c1eb 100644 --- a/indra/llcommon/lleventapi.cpp +++ b/indra/llcommon/lleventapi.cpp @@ -34,6 +34,7 @@ // std headers // external library headers // other Linden headers +#include "llerror.h" LLEventAPI::LLEventAPI(const std::string& name, const std::string& desc, const std::string& field): lbase(name, field), @@ -45,3 +46,32 @@ LLEventAPI::LLEventAPI(const std::string& name, const std::string& desc, const s LLEventAPI::~LLEventAPI() { } + +LLEventAPI::Response::Response(const LLSD& seed, const LLSD& request, const LLSD::String& replyKey): + mResp(seed), + mReq(request), + mKey(replyKey) +{} + +LLEventAPI::Response::~Response() +{ + // When you instantiate a stack Response object, if the original + // request requested a reply, send it when we leave this block, no + // matter how. + sendReply(mResp, mReq, mKey); +} + +void LLEventAPI::Response::warn(const std::string& warning) +{ + LL_WARNS("LLEventAPI::Response") << warning << LL_ENDL; + mResp["warnings"].append(warning); +} + +void LLEventAPI::Response::error(const std::string& error) +{ + // Use LL_WARNS rather than LL_ERROR: we don't want the viewer to shut + // down altogether. + LL_WARNS("LLEventAPI::Response") << error << LL_ENDL; + + mResp["error"] = error; +} diff --git a/indra/llcommon/lleventapi.h b/indra/llcommon/lleventapi.h index d75d521e8e..1a37d780b6 100644 --- a/indra/llcommon/lleventapi.h +++ b/indra/llcommon/lleventapi.h @@ -76,6 +76,89 @@ public: LLEventDispatcher::add(name, desc, callable, required); } + /** + * Instantiate a Response object in any LLEventAPI subclass method that + * wants to guarantee a reply (if requested) will be sent on exit from the + * method. The reply will be sent if request.has(@a replyKey), default + * "reply". If specified, the value of request[replyKey] is the name of + * the LLEventPump on which to send the reply. Conventionally you might + * code something like: + * + * @code + * void MyEventAPI::someMethod(const LLSD& request) + * { + * // Send a reply event as long as request.has("reply") + * Response response(LLSD(), request); + * // ... + * // will be sent in reply event + * response["somekey"] = some_data; + * } + * @endcode + */ + class LL_COMMON_API Response + { + public: + /** + * Instantiating a Response object in an LLEventAPI subclass method + * ensures that, if desired, a reply event will be sent. + * + * @a seed is the initial reply LLSD that will be further decorated before + * being sent as the reply + * + * @a request is the incoming request LLSD; we particularly care about + * [replyKey] and ["reqid"] + * + * @a replyKey [default "reply"] is the string name of the LLEventPump + * on which the caller wants a reply. If <tt>(! + * request.has(replyKey))</tt>, no reply will be sent. + */ + Response(const LLSD& seed, const LLSD& request, const LLSD::String& replyKey="reply"); + ~Response(); + + /** + * @code + * if (some condition) + * { + * response.warn("warnings are logged and collected in [\"warnings\"]"); + * } + * @endcode + */ + void warn(const std::string& warning); + /** + * @code + * if (some condition isn't met) + * { + * // In a function returning void, you can validly 'return + * // expression' if the expression is itself of type void. But + * // returning is up to you; response.error() has no effect on + * // flow of control. + * return response.error("error message, logged and also sent as [\"error\"]"); + * } + * @endcode + */ + void error(const std::string& error); + + /** + * set other keys... + * + * @code + * // set any attributes you want to be sent in the reply + * response["info"] = some_value; + * // ... + * response["ok"] = went_well; + * @endcode + */ + LLSD& operator[](const LLSD::String& key) { return mResp[key]; } + + /** + * set the response to the given data + */ + void setResponse(LLSD const & response){ mResp = response; } + + LLSD mResp, mReq; + LLSD::String mKey; + }; + private: std::string mDesc; }; diff --git a/indra/llcommon/lleventdispatcher.cpp b/indra/llcommon/lleventdispatcher.cpp index d6e820d793..5b6d4efbe9 100644 --- a/indra/llcommon/lleventdispatcher.cpp +++ b/indra/llcommon/lleventdispatcher.cpp @@ -41,6 +41,354 @@ #include "llevents.h" #include "llerror.h" #include "llsdutil.h" +#include "stringize.h" +#include <memory> // std::auto_ptr + +/***************************************************************************** +* LLSDArgsSource +*****************************************************************************/ +/** + * Store an LLSD array, producing its elements one at a time. Die with LL_ERRS + * if the consumer requests more elements than the array contains. + */ +class LL_COMMON_API LLSDArgsSource +{ +public: + LLSDArgsSource(const std::string function, const LLSD& args); + ~LLSDArgsSource(); + + LLSD next(); + + void done() const; + +private: + std::string _function; + LLSD _args; + LLSD::Integer _index; +}; + +LLSDArgsSource::LLSDArgsSource(const std::string function, const LLSD& args): + _function(function), + _args(args), + _index(0) +{ + if (! (_args.isUndefined() || _args.isArray())) + { + LL_ERRS("LLSDArgsSource") << _function << " needs an args array instead of " + << _args << LL_ENDL; + } +} + +LLSDArgsSource::~LLSDArgsSource() +{ + done(); +} + +LLSD LLSDArgsSource::next() +{ + if (_index >= _args.size()) + { + LL_ERRS("LLSDArgsSource") << _function << " requires more arguments than the " + << _args.size() << " provided: " << _args << LL_ENDL; + } + return _args[_index++]; +} + +void LLSDArgsSource::done() const +{ + if (_index < _args.size()) + { + LL_WARNS("LLSDArgsSource") << _function << " only consumed " << _index + << " of the " << _args.size() << " arguments provided: " + << _args << LL_ENDL; + } +} + +/***************************************************************************** +* LLSDArgsMapper +*****************************************************************************/ +/** + * From a formal parameters description and a map of arguments, construct an + * arguments array. + * + * That is, given: + * - an LLSD array of length n containing parameter-name strings, + * corresponding to the arguments of a function of interest + * - an LLSD collection specifying default parameter values, either: + * - an LLSD array of length m <= n, matching the rightmost m params, or + * - an LLSD map explicitly stating default name=value pairs + * - an LLSD map of parameter names and actual values for a particular + * function call + * construct an LLSD array of actual argument values for this function call. + * + * The parameter-names array and the defaults collection describe the function + * being called. The map might vary with every call, providing argument values + * for the described parameters. + * + * The array of parameter names must match the number of parameters expected + * by the function of interest. + * + * If you pass a map of default parameter values, it provides default values + * as you might expect. It is an error to specify a default value for a name + * not listed in the parameters array. + * + * If you pass an array of default parameter values, it is mapped to the + * rightmost m of the n parameter names. It is an error if the default-values + * array is longer than the parameter-names array. Consider the following + * parameter names: ["a", "b", "c", "d"]. + * + * - An empty array of default values (or an isUndefined() value) asserts that + * every one of the above parameter names is required. + * - An array of four default values [1, 2, 3, 4] asserts that every one of + * the above parameters is optional. If the current parameter map is empty, + * they will be passed to the function as [1, 2, 3, 4]. + * - An array of two default values [11, 12] asserts that parameters "a" and + * "b" are required, while "c" and "d" are optional, having default values + * "c"=11 and "d"=12. + * + * The arguments array is constructed as follows: + * + * - Arguments-map keys not found in the parameter-names array are ignored. + * - Entries from the map provide values for an improper subset of the + * parameters named in the parameter-names array. This results in a + * tentative values array with "holes." (size of map) + (number of holes) = + * (size of names array) + * - Holes are filled with the default values. + * - Any remaining holes constitute an error. + */ +class LL_COMMON_API LLSDArgsMapper +{ +public: + /// Accept description of function: function name, param names, param + /// default values + LLSDArgsMapper(const std::string& function, const LLSD& names, const LLSD& defaults); + + /// Given arguments map, return LLSD::Array of parameter values, or LL_ERRS. + LLSD map(const LLSD& argsmap) const; + +private: + static std::string formatlist(const LLSD&); + + // The function-name string is purely descriptive. We want error messages + // to be able to indicate which function's LLSDArgsMapper has the problem. + std::string _function; + // Store the names array pretty much as given. + LLSD _names; + // Though we're handed an array of name strings, it's more useful to us to + // store it as a map from name string to position index. Of course that's + // easy to generate from the incoming names array, but why do it more than + // once? + typedef std::map<LLSD::String, LLSD::Integer> IndexMap; + IndexMap _indexes; + // Generated array of default values, aligned with the array of param names. + LLSD _defaults; + // Indicate whether we have a default value for each param. + typedef std::vector<char> FilledVector; + FilledVector _has_dft; +}; + +LLSDArgsMapper::LLSDArgsMapper(const std::string& function, + const LLSD& names, const LLSD& defaults): + _function(function), + _names(names), + _has_dft(names.size()) +{ + if (! (_names.isUndefined() || _names.isArray())) + { + LL_ERRS("LLSDArgsMapper") << function << " names must be an array, not " << names << LL_ENDL; + } + LLSD::Integer nparams(_names.size()); + // From _names generate _indexes. + for (LLSD::Integer ni = 0, nend = _names.size(); ni < nend; ++ni) + { + _indexes[_names[ni]] = ni; + } + + // Presize _defaults() array so we don't have to resize it more than once. + // All entries are initialized to LLSD(); but since _has_dft is still all + // 0, they're all "holes" for now. + if (nparams) + { + _defaults[nparams - 1] = LLSD(); + } + + if (defaults.isUndefined() || defaults.isArray()) + { + LLSD::Integer ndefaults = defaults.size(); + // defaults is a (possibly empty) array. Right-align it with names. + if (ndefaults > nparams) + { + LL_ERRS("LLSDArgsMapper") << function << " names array " << names + << " shorter than defaults array " << defaults << LL_ENDL; + } + + // Offset by which we slide defaults array right to right-align with + // _names array + LLSD::Integer offset = nparams - ndefaults; + // Fill rightmost _defaults entries from defaults, and mark them as + // filled + for (LLSD::Integer i = 0, iend = ndefaults; i < iend; ++i) + { + _defaults[i + offset] = defaults[i]; + _has_dft[i + offset] = 1; + } + } + else if (defaults.isMap()) + { + // defaults is a map. Use it to populate the _defaults array. + LLSD bogus; + for (LLSD::map_const_iterator mi(defaults.beginMap()), mend(defaults.endMap()); + mi != mend; ++mi) + { + IndexMap::const_iterator ixit(_indexes.find(mi->first)); + if (ixit == _indexes.end()) + { + bogus.append(mi->first); + continue; + } + + LLSD::Integer pos = ixit->second; + // Store default value at that position in the _defaults array. + _defaults[pos] = mi->second; + // Don't forget to record the fact that we've filled this + // position. + _has_dft[pos] = 1; + } + if (bogus.size()) + { + LL_ERRS("LLSDArgsMapper") << function << " defaults specified for nonexistent params " + << formatlist(bogus) << LL_ENDL; + } + } + else + { + LL_ERRS("LLSDArgsMapper") << function << " defaults must be a map or an array, not " + << defaults << LL_ENDL; + } +} + +LLSD LLSDArgsMapper::map(const LLSD& argsmap) const +{ + if (! (argsmap.isUndefined() || argsmap.isMap() || argsmap.isArray())) + { + LL_ERRS("LLSDArgsMapper") << _function << " map() needs a map or array, not " + << argsmap << LL_ENDL; + } + // Initialize the args array. Indexing a non-const LLSD array grows it + // to appropriate size, but we don't want to resize this one on each + // new operation. Just make it as big as we need before we start + // stuffing values into it. + LLSD args(LLSD::emptyArray()); + if (_defaults.size() == 0) + { + // If this function requires no arguments, fast exit. (Don't try to + // assign to args[-1].) + return args; + } + args[_defaults.size() - 1] = LLSD(); + + // Get a vector of chars to indicate holes. It's tempting to just scan + // for LLSD::isUndefined() values after filling the args array from + // the map, but it's plausible for caller to explicitly pass + // isUndefined() as the value of some parameter name. That's legal + // since isUndefined() has well-defined conversions (default value) + // for LLSD data types. So use a whole separate array for detecting + // holes. (Avoid std::vector<bool> which is known to be odd -- can we + // iterate?) + FilledVector filled(args.size()); + + if (argsmap.isArray()) + { + // Fill args from array. If there are too many args in passed array, + // ignore the rest. + LLSD::Integer size(argsmap.size()); + if (size > args.size()) + { + // We don't just use std::min() because we want to sneak in this + // warning if caller passes too many args. + LL_WARNS("LLSDArgsMapper") << _function << " needs " << args.size() + << " params, ignoring last " << (size - args.size()) + << " of passed " << size << ": " << argsmap << LL_ENDL; + size = args.size(); + } + for (LLSD::Integer i(0); i < size; ++i) + { + // Copy the actual argument from argsmap + args[i] = argsmap[i]; + // Note that it's been filled + filled[i] = 1; + } + } + else + { + // argsmap is in fact a map. Walk the map. + for (LLSD::map_const_iterator mi(argsmap.beginMap()), mend(argsmap.endMap()); + mi != mend; ++mi) + { + // mi->first is a parameter-name string, with mi->second its + // value. Look up the name's position index in _indexes. + IndexMap::const_iterator ixit(_indexes.find(mi->first)); + if (ixit == _indexes.end()) + { + // Allow for a map containing more params than were passed in + // our names array. Caller typically receives a map containing + // the function name, cruft such as reqid, etc. Ignore keys + // not defined in _indexes. + LL_DEBUGS("LLSDArgsMapper") << _function << " ignoring " + << mi->first << "=" << mi->second << LL_ENDL; + continue; + } + LLSD::Integer pos = ixit->second; + // Store the value at that position in the args array. + args[pos] = mi->second; + // Don't forget to record the fact that we've filled this + // position. + filled[pos] = 1; + } + } + + // Fill any remaining holes from _defaults. + LLSD unfilled(LLSD::emptyArray()); + for (LLSD::Integer i = 0, iend = args.size(); i < iend; ++i) + { + if (! filled[i]) + { + // If there's no default value for this parameter, that's an + // error. + if (! _has_dft[i]) + { + unfilled.append(_names[i]); + } + else + { + args[i] = _defaults[i]; + } + } + } + // If any required args -- args without defaults -- were left unfilled + // by argsmap, that's a problem. + if (unfilled.size()) + { + LL_ERRS("LLSDArgsMapper") << _function << " missing required arguments " + << formatlist(unfilled) << " from " << argsmap << LL_ENDL; + } + + // done + return args; +} + +std::string LLSDArgsMapper::formatlist(const LLSD& list) +{ + std::ostringstream out; + const char* delim = ""; + for (LLSD::array_const_iterator li(list.beginArray()), lend(list.endArray()); + li != lend; ++li) + { + out << delim << li->asString(); + delim = ", "; + } + return out.str(); +} LLEventDispatcher::LLEventDispatcher(const std::string& desc, const std::string& key): mDesc(desc), @@ -52,12 +400,181 @@ LLEventDispatcher::~LLEventDispatcher() { } +/** + * DispatchEntry subclass used for callables accepting(const LLSD&) + */ +struct LLEventDispatcher::LLSDDispatchEntry: public LLEventDispatcher::DispatchEntry +{ + LLSDDispatchEntry(const std::string& desc, const Callable& func, const LLSD& required): + DispatchEntry(desc), + mFunc(func), + mRequired(required) + {} + + Callable mFunc; + LLSD mRequired; + + virtual void call(const std::string& desc, const LLSD& event) const + { + // Validate the syntax of the event itself. + std::string mismatch(llsd_matches(mRequired, event)); + if (! mismatch.empty()) + { + LL_ERRS("LLEventDispatcher") << desc << ": bad request: " << mismatch << LL_ENDL; + } + // Event syntax looks good, go for it! + mFunc(event); + } + + virtual LLSD addMetadata(LLSD meta) const + { + meta["required"] = mRequired; + return meta; + } +}; + +/** + * DispatchEntry subclass for passing LLSD to functions accepting + * arbitrary argument types (convertible via LLSDParam) + */ +struct LLEventDispatcher::ParamsDispatchEntry: public LLEventDispatcher::DispatchEntry +{ + ParamsDispatchEntry(const std::string& desc, const invoker_function& func): + DispatchEntry(desc), + mInvoker(func) + {} + + invoker_function mInvoker; + + virtual void call(const std::string& desc, const LLSD& event) const + { + LLSDArgsSource src(desc, event); + mInvoker(boost::bind(&LLSDArgsSource::next, boost::ref(src))); + } +}; + +/** + * DispatchEntry subclass for dispatching LLSD::Array to functions accepting + * arbitrary argument types (convertible via LLSDParam) + */ +struct LLEventDispatcher::ArrayParamsDispatchEntry: public LLEventDispatcher::ParamsDispatchEntry +{ + ArrayParamsDispatchEntry(const std::string& desc, const invoker_function& func, + LLSD::Integer arity): + ParamsDispatchEntry(desc, func), + mArity(arity) + {} + + LLSD::Integer mArity; + + virtual LLSD addMetadata(LLSD meta) const + { + LLSD array(LLSD::emptyArray()); + // Resize to number of arguments required + if (mArity) + array[mArity - 1] = LLSD(); + llassert_always(array.size() == mArity); + meta["required"] = array; + return meta; + } +}; + +/** + * DispatchEntry subclass for dispatching LLSD::Map to functions accepting + * arbitrary argument types (convertible via LLSDParam) + */ +struct LLEventDispatcher::MapParamsDispatchEntry: public LLEventDispatcher::ParamsDispatchEntry +{ + MapParamsDispatchEntry(const std::string& name, const std::string& desc, + const invoker_function& func, + const LLSD& params, const LLSD& defaults): + ParamsDispatchEntry(desc, func), + mMapper(name, params, defaults), + mRequired(LLSD::emptyMap()) + { + // Build the set of all param keys, then delete the ones that are + // optional. What's left are the ones that are required. + for (LLSD::array_const_iterator pi(params.beginArray()), pend(params.endArray()); + pi != pend; ++pi) + { + mRequired[pi->asString()] = LLSD(); + } + + if (defaults.isArray() || defaults.isUndefined()) + { + // Right-align the params and defaults arrays. + LLSD::Integer offset = params.size() - defaults.size(); + // Now the name of every defaults[i] is at params[i + offset]. + for (LLSD::Integer i(0), iend(defaults.size()); i < iend; ++i) + { + // Erase this optional param from mRequired. + mRequired.erase(params[i + offset].asString()); + // Instead, make an entry in mOptional with the default + // param's name and value. + mOptional[params[i + offset].asString()] = defaults[i]; + } + } + else if (defaults.isMap()) + { + // if defaults is already a map, then it's already in the form we + // intend to deliver in metadata + mOptional = defaults; + // Just delete from mRequired every key appearing in mOptional. + for (LLSD::map_const_iterator mi(mOptional.beginMap()), mend(mOptional.endMap()); + mi != mend; ++mi) + { + mRequired.erase(mi->first); + } + } + } + + LLSDArgsMapper mMapper; + LLSD mRequired; + LLSD mOptional; + + virtual void call(const std::string& desc, const LLSD& event) const + { + // Just convert from LLSD::Map to LLSD::Array using mMapper, then pass + // to base-class call() method. + ParamsDispatchEntry::call(desc, mMapper.map(event)); + } + + virtual LLSD addMetadata(LLSD meta) const + { + meta["required"] = mRequired; + meta["optional"] = mOptional; + return meta; + } +}; + +void LLEventDispatcher::addArrayParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + LLSD::Integer arity) +{ + mDispatch.insert( + DispatchMap::value_type(name, DispatchMap::mapped_type( + new ArrayParamsDispatchEntry(desc, invoker, arity)))); +} + +void LLEventDispatcher::addMapParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + const LLSD& params, + const LLSD& defaults) +{ + mDispatch.insert( + DispatchMap::value_type(name, DispatchMap::mapped_type( + new MapParamsDispatchEntry(name, desc, invoker, params, defaults)))); +} + /// Register a callable by name void LLEventDispatcher::add(const std::string& name, const std::string& desc, const Callable& callable, const LLSD& required) { - mDispatch.insert(DispatchMap::value_type(name, - DispatchMap::mapped_type(callable, desc, required))); + mDispatch.insert( + DispatchMap::value_type(name, DispatchMap::mapped_type( + new LLSDDispatchEntry(desc, callable, required)))); } void LLEventDispatcher::addFail(const std::string& name, const std::string& classname) const @@ -83,7 +600,7 @@ bool LLEventDispatcher::remove(const std::string& name) /// such callable exists, die with LL_ERRS. void LLEventDispatcher::operator()(const std::string& name, const LLSD& event) const { - if (! attemptCall(name, event)) + if (! try_call(name, event)) { LL_ERRS("LLEventDispatcher") << "LLEventDispatcher(" << mDesc << "): '" << name << "' not found" << LL_ENDL; @@ -98,44 +615,29 @@ void LLEventDispatcher::operator()(const LLSD& event) const // This could/should be implemented in terms of the two-arg overload. // However -- we can produce a more informative error message. std::string name(event[mKey]); - if (! attemptCall(name, event)) + if (! try_call(name, event)) { LL_ERRS("LLEventDispatcher") << "LLEventDispatcher(" << mDesc << "): bad " << mKey << " value '" << name << "'" << LL_ENDL; } } -bool LLEventDispatcher::attemptCall(const std::string& name, const LLSD& event) const +bool LLEventDispatcher::try_call(const LLSD& event) const { - DispatchMap::const_iterator found = mDispatch.find(name); - if (found == mDispatch.end()) - { - // The reason we only return false, leaving it up to our caller to die - // with LL_ERRS, is that different callers have different amounts of - // available information. - return false; - } - // Found the name, so it's plausible to even attempt the call. But first, - // validate the syntax of the event itself. - std::string mismatch(llsd_matches(found->second.mRequired, event)); - if (! mismatch.empty()) - { - LL_ERRS("LLEventDispatcher") << "LLEventDispatcher(" << mDesc << ") calling '" << name - << "': bad request: " << mismatch << LL_ENDL; - } - // Event syntax looks good, go for it! - (found->second.mFunc)(event); - return true; // tell caller we were able to call + return try_call(event[mKey], event); } -LLEventDispatcher::Callable LLEventDispatcher::get(const std::string& name) const +bool LLEventDispatcher::try_call(const std::string& name, const LLSD& event) const { DispatchMap::const_iterator found = mDispatch.find(name); if (found == mDispatch.end()) { - return Callable(); + return false; } - return found->second.mFunc; + // Found the name, so it's plausible to even attempt the call. + found->second->call(STRINGIZE("LLEventDispatcher(" << mDesc << ") calling '" << name << "'"), + event); + return true; // tell caller we were able to call } LLSD LLEventDispatcher::getMetadata(const std::string& name) const @@ -147,9 +649,8 @@ LLSD LLEventDispatcher::getMetadata(const std::string& name) const } LLSD meta; meta["name"] = name; - meta["desc"] = found->second.mDesc; - meta["required"] = found->second.mRequired; - return meta; + meta["desc"] = found->second->mDesc; + return found->second->addMetadata(meta); } LLDispatchListener::LLDispatchListener(const std::string& pumpname, const std::string& key): @@ -164,3 +665,8 @@ bool LLDispatchListener::process(const LLSD& event) (*this)(event); return false; } + +LLEventDispatcher::DispatchEntry::DispatchEntry(const std::string& desc): + mDesc(desc) +{} + diff --git a/indra/llcommon/lleventdispatcher.h b/indra/llcommon/lleventdispatcher.h index dfffd59eb6..7acc61de4e 100644 --- a/indra/llcommon/lleventdispatcher.h +++ b/indra/llcommon/lleventdispatcher.h @@ -27,18 +27,56 @@ * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ + * + * The invoker machinery that constructs a boost::fusion argument list for use + * with boost::fusion::invoke() is derived from + * http://www.boost.org/doc/libs/1_45_0/libs/function_types/example/interpreter.hpp + * whose license information is copied below: + * + * "(C) Copyright Tobias Schwinger + * + * Use modification and distribution are subject to the boost Software License, + * Version 1.0. (See http://www.boost.org/LICENSE_1_0.txt)." */ #if ! defined(LL_LLEVENTDISPATCHER_H) #define LL_LLEVENTDISPATCHER_H +// nil is too generic a term to be allowed to be a global macro. In +// particular, boost::fusion defines a 'class nil' (properly encapsulated in a +// namespace) that a global 'nil' macro breaks badly. +#if defined(nil) +// Capture the value of the macro 'nil', hoping int is an appropriate type. +static const int nil_(nil); +// Now forget the macro. +#undef nil +// Finally, reintroduce 'nil' as a properly-scoped alias for the previously- +// defined const 'nil_'. Make it static since otherwise it produces duplicate- +// symbol link errors later. +static const int& nil(nil_); +#endif + #include <string> -#include <map> +#include <boost/shared_ptr.hpp> #include <boost/function.hpp> #include <boost/bind.hpp> #include <boost/iterator/transform_iterator.hpp> +#include <boost/utility/enable_if.hpp> +#include <boost/function_types/is_nonmember_callable_builtin.hpp> +#include <boost/function_types/parameter_types.hpp> +#include <boost/function_types/function_arity.hpp> +#include <boost/type_traits/remove_cv.hpp> +#include <boost/type_traits/remove_reference.hpp> +#include <boost/fusion/include/push_back.hpp> +#include <boost/fusion/include/cons.hpp> +#include <boost/fusion/include/invoke.hpp> +#include <boost/mpl/begin.hpp> +#include <boost/mpl/end.hpp> +#include <boost/mpl/next.hpp> +#include <boost/mpl/deref.hpp> #include <typeinfo> #include "llevents.h" +#include "llsdutil.h" class LLSD; @@ -54,12 +92,18 @@ public: LLEventDispatcher(const std::string& desc, const std::string& key); virtual ~LLEventDispatcher(); - /// Accept any C++ callable, typically a boost::bind() expression + /// @name Register functions accepting(const LLSD&) + //@{ + + /// Accept any C++ callable with the right signature, typically a + /// boost::bind() expression typedef boost::function<void(const LLSD&)> Callable; /** - * Register a @a callable by @a name. The optional @a required parameter - * is used to validate the structure of each incoming event (see + * Register a @a callable by @a name. The passed @a callable accepts a + * single LLSD value and uses it in any way desired, e.g. extract + * parameters and call some other function. The optional @a required + * parameter is used to validate the structure of each incoming event (see * llsd_matches()). */ void add(const std::string& name, @@ -68,9 +112,23 @@ public: const LLSD& required=LLSD()); /** + * The case of a free function (or static method) accepting(const LLSD&) + * could also be intercepted by the arbitrary-args overload below. Ensure + * that it's directed to the Callable overload above instead. + */ + void add(const std::string& name, + const std::string& desc, + void (*f)(const LLSD&), + const LLSD& required=LLSD()) + { + add(name, desc, Callable(f), required); + } + + /** * Special case: a subclass of this class can pass an unbound member - * function pointer without explicitly specifying the - * <tt>boost::bind()</tt> expression. + * function pointer (of an LLEventDispatcher subclass) without explicitly + * specifying the <tt>boost::bind()</tt> expression. The passed @a method + * accepts a single LLSD value, presumably containing other parameters. */ template <class CLASS> void add(const std::string& name, @@ -81,7 +139,8 @@ public: addMethod<CLASS>(name, desc, method, required); } - /// Overload for both const and non-const methods + /// Overload for both const and non-const methods. The passed @a method + /// accepts a single LLSD value, presumably containing other parameters. template <class CLASS> void add(const std::string& name, const std::string& desc, @@ -91,15 +150,106 @@ public: addMethod<CLASS>(name, desc, method, required); } - /// Convenience: for LLEventDispatcher, not every callable needs a - /// documentation string. - template <typename CALLABLE> - void add(const std::string& name, - CALLABLE callable, - const LLSD& required=LLSD()) - { - add(name, "", callable, required); - } + //@} + + /// @name Register functions with arbitrary param lists + //@{ + + /** + * Register a free function with arbitrary parameters. (This also works + * for static class methods.) + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * When calling this name, pass an LLSD::Array. Each entry in turn will be + * converted to the corresponding parameter type using LLSDParam. + */ + template<typename Function> + typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> + >::type add(const std::string& name, + const std::string& desc, + Function f); + + /** + * Register a nonstatic class method with arbitrary parameters. + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * To cover cases such as a method on an LLSingleton we don't yet want to + * instantiate, instead of directly storing an instance pointer, accept a + * nullary callable returning a pointer/reference to the desired class + * instance. If you already have an instance in hand, + * boost::lambda::var(instance) or boost::lambda::constant(instance_ptr) + * produce suitable callables. + * + * When calling this name, pass an LLSD::Array. Each entry in turn will be + * converted to the corresponding parameter type using LLSDParam. + */ + template<typename Method, typename InstanceGetter> + typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> + >::type add(const std::string& name, + const std::string& desc, + Method f, + const InstanceGetter& getter); + + /** + * Register a free function with arbitrary parameters. (This also works + * for static class methods.) + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * Pass an LLSD::Array of parameter names, and optionally another + * LLSD::Array of default parameter values, a la LLSDArgsMapper. + * + * When calling this name, pass an LLSD::Map. We will internally generate + * an LLSD::Array using LLSDArgsMapper and then convert each entry in turn + * to the corresponding parameter type using LLSDParam. + */ + template<typename Function> + typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> + >::type add(const std::string& name, + const std::string& desc, + Function f, + const LLSD& params, + const LLSD& defaults=LLSD()); + + /** + * Register a nonstatic class method with arbitrary parameters. + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * To cover cases such as a method on an LLSingleton we don't yet want to + * instantiate, instead of directly storing an instance pointer, accept a + * nullary callable returning a pointer/reference to the desired class + * instance. If you already have an instance in hand, + * boost::lambda::var(instance) or boost::lambda::constant(instance_ptr) + * produce suitable callables. + * + * Pass an LLSD::Array of parameter names, and optionally another + * LLSD::Array of default parameter values, a la LLSDArgsMapper. + * + * When calling this name, pass an LLSD::Map. We will internally generate + * an LLSD::Array using LLSDArgsMapper and then convert each entry in turn + * to the corresponding parameter type using LLSDParam. + */ + template<typename Method, typename InstanceGetter> + typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> + >::type add(const std::string& name, + const std::string& desc, + Method f, + const InstanceGetter& getter, + const LLSD& params, + const LLSD& defaults=LLSD()); + + //@} /// Unregister a callable bool remove(const std::string& name); @@ -109,12 +259,25 @@ public: /// the @a required prototype specified at add() time, die with LL_ERRS. void operator()(const std::string& name, const LLSD& event) const; + /// Call a registered callable with an explicitly-specified name and + /// return <tt>true</tt>. If no such callable exists, return + /// <tt>false</tt>. If the @a event fails to match the @a required + /// prototype specified at add() time, die with LL_ERRS. + bool try_call(const std::string& name, const LLSD& event) const; + /// Extract the @a key value from the incoming @a event, and call the /// callable whose name is specified by that map @a key. If no such /// callable exists, die with LL_ERRS. If the @a event fails to match the /// @a required prototype specified at add() time, die with LL_ERRS. void operator()(const LLSD& event) const; + /// Extract the @a key value from the incoming @a event, call the callable + /// whose name is specified by that map @a key and return <tt>true</tt>. + /// If no such callable exists, return <tt>false</tt>. If the @a event + /// fails to match the @a required prototype specified at add() time, die + /// with LL_ERRS. + bool try_call(const LLSD& event) const; + /// @name Iterate over defined names //@{ typedef std::pair<std::string, std::string> NameDesc; @@ -122,16 +285,22 @@ public: private: struct DispatchEntry { - DispatchEntry(const Callable& func, const std::string& desc, const LLSD& required): - mFunc(func), - mDesc(desc), - mRequired(required) - {} - Callable mFunc; + DispatchEntry(const std::string& desc); + virtual ~DispatchEntry() {} // suppress MSVC warning, sigh + std::string mDesc; - LLSD mRequired; + + virtual void call(const std::string& desc, const LLSD& event) const = 0; + virtual LLSD addMetadata(LLSD) const = 0; }; - typedef std::map<std::string, DispatchEntry> DispatchMap; + // Tried using boost::ptr_map<std::string, DispatchEntry>, but ptr_map<> + // wants its value type to be "clonable," even just to dereference an + // iterator. I don't want to clone entries -- if I have to copy an entry + // around, I want it to continue pointing to the same DispatchEntry + // subclass object. However, I definitely want DispatchMap to destroy + // DispatchEntry if no references are outstanding at the time an entry is + // removed. This looks like a job for boost::shared_ptr. + typedef std::map<std::string, boost::shared_ptr<DispatchEntry> > DispatchMap; public: /// We want the flexibility to redefine what data we store per name, @@ -149,10 +318,6 @@ public: } //@} - /// Fetch the Callable for the specified name. If no such name was - /// registered, return an empty() Callable. - Callable get(const std::string& name) const; - /// Get information about a specific Callable LLSD getMetadata(const std::string& name) const; @@ -175,18 +340,184 @@ private: } } void addFail(const std::string& name, const std::string& classname) const; - /// try to dispatch, return @c true if success - bool attemptCall(const std::string& name, const LLSD& event) const; std::string mDesc, mKey; DispatchMap mDispatch; static NameDesc makeNameDesc(const DispatchMap::value_type& item) { - return NameDesc(item.first, item.second.mDesc); + return NameDesc(item.first, item.second->mDesc); + } + + struct LLSDDispatchEntry; + struct ParamsDispatchEntry; + struct ArrayParamsDispatchEntry; + struct MapParamsDispatchEntry; + + // Step 2 of parameter analysis. Instantiating invoker<some_function_type> + // implicitly sets its From and To parameters to the (compile time) begin + // and end iterators over that function's parameter types. + template< typename Function + , class From = typename boost::mpl::begin< boost::function_types::parameter_types<Function> >::type + , class To = typename boost::mpl::end< boost::function_types::parameter_types<Function> >::type + > + struct invoker; + + // deliver LLSD arguments one at a time + typedef boost::function<LLSD()> args_source; + // obtain args from an args_source to build param list and call target + // function + typedef boost::function<void(const args_source&)> invoker_function; + + template <typename Function> + invoker_function make_invoker(Function f); + template <typename Method, typename InstanceGetter> + invoker_function make_invoker(Method f, const InstanceGetter& getter); + void addArrayParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + LLSD::Integer arity); + void addMapParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + const LLSD& params, + const LLSD& defaults); +}; + +/***************************************************************************** +* LLEventDispatcher template implementation details +*****************************************************************************/ +// Step 3 of parameter analysis, the recursive case. +template<typename Function, class From, class To> +struct LLEventDispatcher::invoker +{ + template<typename T> + struct remove_cv_ref + : boost::remove_cv< typename boost::remove_reference<T>::type > + { }; + + // apply() accepts an arbitrary boost::fusion sequence as args. It + // examines the next parameter type in the parameter-types sequence + // bounded by From and To, obtains the next LLSD object from the passed + // args_source and constructs an LLSDParam of appropriate type to try + // to convert the value. It then recurs with the next parameter-types + // iterator, passing the args sequence thus far. + template<typename Args> + static inline + void apply(Function func, const args_source& argsrc, Args const & args) + { + typedef typename boost::mpl::deref<From>::type arg_type; + typedef typename boost::mpl::next<From>::type next_iter_type; + typedef typename remove_cv_ref<arg_type>::type plain_arg_type; + + invoker<Function, next_iter_type, To>::apply + ( func, argsrc, boost::fusion::push_back(args, LLSDParam<plain_arg_type>(argsrc()))); + } + + // Special treatment for instance (first) parameter of a non-static member + // function. Accept the instance-getter callable, calling that to produce + // the first args value. Since we know we're at the top of the recursion + // chain, we need not also require a partial args sequence from our caller. + template <typename InstanceGetter> + static inline + void method_apply(Function func, const args_source& argsrc, const InstanceGetter& getter) + { + typedef typename boost::mpl::next<From>::type next_iter_type; + + // Instead of grabbing the first item from argsrc and making an + // LLSDParam of it, call getter() and pass that as the instance param. + invoker<Function, next_iter_type, To>::apply + ( func, argsrc, boost::fusion::push_back(boost::fusion::nil(), boost::ref(getter()))); + } +}; + +// Step 4 of parameter analysis, the leaf case. When the general +// invoker<Function, From, To> logic has advanced From until it matches To, +// the compiler will pick this template specialization. +template<typename Function, class To> +struct LLEventDispatcher::invoker<Function,To,To> +{ + // the argument list is complete, now call the function + template<typename Args> + static inline + void apply(Function func, const args_source&, Args const & args) + { + boost::fusion::invoke(func, args); } }; +template<typename Function> +typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Function f) +{ + // Construct an invoker_function, a callable accepting const args_source&. + // Add to DispatchMap an ArrayParamsDispatchEntry that will handle the + // caller's LLSD::Array. + addArrayParamsDispatchEntry(name, desc, make_invoker(f), + boost::function_types::function_arity<Function>::value); +} + +template<typename Method, typename InstanceGetter> +typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Method f, + const InstanceGetter& getter) +{ + // Subtract 1 from the compile-time arity because the getter takes care of + // the first parameter. We only need (arity - 1) additional arguments. + addArrayParamsDispatchEntry(name, desc, make_invoker(f, getter), + boost::function_types::function_arity<Method>::value - 1); +} + +template<typename Function> +typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Function f, + const LLSD& params, const LLSD& defaults) +{ + // See comments for previous is_nonmember_callable_builtin add(). + addMapParamsDispatchEntry(name, desc, make_invoker(f), params, defaults); +} + +template<typename Method, typename InstanceGetter> +typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Method f, + const InstanceGetter& getter, + const LLSD& params, const LLSD& defaults) +{ + addMapParamsDispatchEntry(name, desc, make_invoker(f, getter), params, defaults); +} + +template <typename Function> +LLEventDispatcher::invoker_function +LLEventDispatcher::make_invoker(Function f) +{ + // Step 1 of parameter analysis, the top of the recursion. Passing a + // suitable f (see add()'s enable_if condition) to this method causes it + // to infer the function type; specifying that function type to invoker<> + // causes it to fill in the begin/end MPL iterators over the function's + // list of parameter types. + // While normally invoker::apply() could infer its template type from the + // boost::fusion::nil parameter value, here we must be explicit since + // we're boost::bind()ing it rather than calling it directly. + return boost::bind(&invoker<Function>::template apply<boost::fusion::nil>, + f, + _1, + boost::fusion::nil()); +} + +template <typename Method, typename InstanceGetter> +LLEventDispatcher::invoker_function +LLEventDispatcher::make_invoker(Method f, const InstanceGetter& getter) +{ + // Use invoker::method_apply() to treat the instance (first) arg specially. + return boost::bind(&invoker<Method>::template method_apply<InstanceGetter>, + f, + _1, + getter); +} + +/***************************************************************************** +* LLDispatchListener +*****************************************************************************/ /** * Bundle an LLEventPump and a listener with an LLEventDispatcher. A class * that contains (or derives from) LLDispatchListener need only specify the diff --git a/indra/llcommon/llevents.cpp b/indra/llcommon/llevents.cpp index 84a6620a77..0855180dcd 100644 --- a/indra/llcommon/llevents.cpp +++ b/indra/llcommon/llevents.cpp @@ -430,13 +430,13 @@ LLBoundListener LLEventPump::listen_impl(const std::string& name, const LLEventL { // The new node isn't last. Place it between the previous node and // the successor. - newNode = (myprev + mydmi->second)/2.0; + newNode = (myprev + mydmi->second)/2.f; } else { // The new node is last. Bump myprev up to the next integer, add // 1.0 and use that. - newNode = std::ceil(myprev) + 1.0; + newNode = std::ceil(myprev) + 1.f; } // Now that newNode has a value that places it appropriately in mSignal, // connect it. @@ -475,7 +475,7 @@ void LLEventPump::stopListening(const std::string& name) *****************************************************************************/ bool LLEventStream::post(const LLSD& event) { - if (! mEnabled) + if (! mEnabled || !mSignal) { return false; } @@ -515,6 +515,8 @@ bool LLEventQueue::post(const LLSD& event) void LLEventQueue::flush() { + if(!mSignal) return; + // Consider the case when a given listener on this LLEventQueue posts yet // another event on the same queue. If we loop over mEventQueue directly, // we'll end up processing all those events during the same flush() call @@ -586,3 +588,27 @@ void LLReqID::stamp(LLSD& response) const } response["reqid"] = mReqid; } + +bool sendReply(const LLSD& reply, const LLSD& request, const std::string& replyKey) +{ + // If the original request has no value for replyKey, it's pointless to + // construct or send a reply event: on which LLEventPump should we send + // it? Allow that to be optional: if the caller wants to require replyKey, + // it can so specify when registering the operation method. + if (! request.has(replyKey)) + { + return false; + } + + // Here the request definitely contains replyKey; reasonable to proceed. + + // Copy 'reply' to modify it. + LLSD newreply(reply); + // Get the ["reqid"] element from request + LLReqID reqID(request); + // and copy it to 'newreply'. + reqID.stamp(newreply); + // Send reply on LLEventPump named in request[replyKey]. Don't forget to + // send the modified 'newreply' instead of the original 'reply'. + return LLEventPumps::instance().obtain(request[replyKey]).post(newreply); +} diff --git a/indra/llcommon/llevents.h b/indra/llcommon/llevents.h index 2491cf1371..65b0fef354 100644 --- a/indra/llcommon/llevents.h +++ b/indra/llcommon/llevents.h @@ -692,6 +692,20 @@ private: }; /** + * Conventionally send a reply to a request event. + * + * @a reply is the LLSD reply event to send + * @a request is the corresponding LLSD request event + * @a replyKey is the key in the @a request event, conventionally ["reply"], + * whose value is the name of the LLEventPump on which to send the reply. + * + * Before sending the reply event, sendReply() copies the ["reqid"] item from + * the request to the reply. + */ +LL_COMMON_API bool sendReply(const LLSD& reply, const LLSD& request, + const std::string& replyKey="reply"); + +/** * Base class for LLListenerWrapper. See visit_and_connect() and llwrap(). We * provide virtual @c accept_xxx() methods, customization points allowing a * subclass access to certain data visible at LLEventPump::listen() time. diff --git a/indra/llcommon/lleventtimer.cpp b/indra/llcommon/lleventtimer.cpp index 7743826c60..0d96e03da4 100644 --- a/indra/llcommon/lleventtimer.cpp +++ b/indra/llcommon/lleventtimer.cpp @@ -58,19 +58,15 @@ LLEventTimer::~LLEventTimer() void LLEventTimer::updateClass() { std::list<LLEventTimer*> completed_timers; - + for (instance_iter iter = beginInstances(); iter != endInstances(); ) { - LLInstanceTrackerScopedGuard guard; - for (instance_iter iter = guard.beginInstances(); iter != guard.endInstances(); ) - { - LLEventTimer& timer = *iter++; - F32 et = timer.mEventTimer.getElapsedTimeF32(); - if (timer.mEventTimer.getStarted() && et > timer.mPeriod) { - timer.mEventTimer.reset(); - if ( timer.tick() ) - { - completed_timers.push_back( &timer ); - } + LLEventTimer& timer = *iter++; + F32 et = timer.mEventTimer.getElapsedTimeF32(); + if (timer.mEventTimer.getStarted() && et > timer.mPeriod) { + timer.mEventTimer.reset(); + if ( timer.tick() ) + { + completed_timers.push_back( &timer ); } } } diff --git a/indra/llcommon/llfasttimer.h b/indra/llcommon/llfasttimer.h index 4ff93a553c..2b25f2fabb 100644 --- a/indra/llcommon/llfasttimer.h +++ b/indra/llcommon/llfasttimer.h @@ -27,140 +27,9 @@ #ifndef LL_FASTTIMER_H #define LL_FASTTIMER_H +// Implementation of getCPUClockCount32() and getCPUClockCount64 are now in llfastertimer_class.cpp. + // pull in the actual class definition #include "llfasttimer_class.h" -// -// Important note: These implementations must be FAST! -// - -#if LL_WINDOWS -// -// Windows implementation of CPU clock -// - -// -// NOTE: put back in when we aren't using platform sdk anymore -// -// because MS has different signatures for these functions in winnt.h -// need to rename them to avoid conflicts -//#define _interlockedbittestandset _renamed_interlockedbittestandset -//#define _interlockedbittestandreset _renamed_interlockedbittestandreset -//#include <intrin.h> -//#undef _interlockedbittestandset -//#undef _interlockedbittestandreset - -//inline U32 LLFastTimer::getCPUClockCount32() -//{ -// U64 time_stamp = __rdtsc(); -// return (U32)(time_stamp >> 8); -//} -// -//// return full timer value, *not* shifted by 8 bits -//inline U64 LLFastTimer::getCPUClockCount64() -//{ -// return __rdtsc(); -//} - -// shift off lower 8 bits for lower resolution but longer term timing -// on 1Ghz machine, a 32-bit word will hold ~1000 seconds of timing -inline U32 LLFastTimer::getCPUClockCount32() -{ - U32 ret_val; - __asm - { - _emit 0x0f - _emit 0x31 - shr eax,8 - shl edx,24 - or eax, edx - mov dword ptr [ret_val], eax - } - return ret_val; -} - -// return full timer value, *not* shifted by 8 bits -inline U64 LLFastTimer::getCPUClockCount64() -{ - U64 ret_val; - __asm - { - _emit 0x0f - _emit 0x31 - mov eax,eax - mov edx,edx - mov dword ptr [ret_val+4], edx - mov dword ptr [ret_val], eax - } - return ret_val; -} -#endif - - -#if (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) -// -// Linux and Solaris implementation of CPU clock - non-x86. -// This is accurate but SLOW! Only use out of desperation. -// -// Try to use the MONOTONIC clock if available, this is a constant time counter -// with nanosecond resolution (but not necessarily accuracy) and attempts are -// made to synchronize this value between cores at kernel start. It should not -// be affected by CPU frequency. If not available use the REALTIME clock, but -// this may be affected by NTP adjustments or other user activity affecting -// the system time. -inline U64 LLFastTimer::getCPUClockCount64() -{ - struct timespec tp; - -#ifdef CLOCK_MONOTONIC // MONOTONIC supported at build-time? - if (-1 == clock_gettime(CLOCK_MONOTONIC,&tp)) // if MONOTONIC isn't supported at runtime then ouch, try REALTIME -#endif - clock_gettime(CLOCK_REALTIME,&tp); - - return (tp.tv_sec*LLFastTimer::sClockResolution)+tp.tv_nsec; -} - -inline U32 LLFastTimer::getCPUClockCount32() -{ - return (U32)(LLFastTimer::getCPUClockCount64() >> 8); -} -#endif // (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) - - -#if (LL_LINUX || LL_SOLARIS || LL_DARWIN) && (defined(__i386__) || defined(__amd64__)) -// -// Mac+Linux+Solaris FAST x86 implementation of CPU clock -inline U32 LLFastTimer::getCPUClockCount32() -{ - U64 x; - __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); - return (U32)(x >> 8); -} - -inline U64 LLFastTimer::getCPUClockCount64() -{ - U64 x; - __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); - return x; -} -#endif - - -#if ( LL_DARWIN && !(defined(__i386__) || defined(__amd64__))) -// -// Mac PPC (deprecated) implementation of CPU clock -// -// Just use gettimeofday implementation for now - -inline U32 LLFastTimer::getCPUClockCount32() -{ - return (U32)(get_clock_count()>>8); -} - -inline U64 LLFastTimer::getCPUClockCount64() -{ - return get_clock_count(); -} -#endif - #endif // LL_LLFASTTIMER_H diff --git a/indra/llcommon/llfasttimer_class.cpp b/indra/llcommon/llfasttimer_class.cpp index bce87ada96..463f558c2c 100644 --- a/indra/llcommon/llfasttimer_class.cpp +++ b/indra/llcommon/llfasttimer_class.cpp @@ -35,10 +35,13 @@ #include <boost/bind.hpp> + #if LL_WINDOWS +#include "lltimer.h" #elif LL_LINUX || LL_SOLARIS #include <sys/time.h> #include <sched.h> +#include "lltimer.h" #elif LL_DARWIN #include <sys/time.h> #include "lltimer.h" // get_clock_count() @@ -61,6 +64,8 @@ BOOL LLFastTimer::sMetricLog = FALSE; LLMutex* LLFastTimer::sLogLock = NULL; std::queue<LLSD> LLFastTimer::sLogQueue; +#define USE_RDTSC 0 + #if LL_LINUX || LL_SOLARIS U64 LLFastTimer::sClockResolution = 1000000000; // Nanosecond resolution #else @@ -214,15 +219,20 @@ LLFastTimer::DeclareTimer::DeclareTimer(const std::string& name) // static void LLFastTimer::DeclareTimer::updateCachedPointers() { - DeclareTimer::LLInstanceTrackerScopedGuard guard; // propagate frame state pointers to timer declarations - for (DeclareTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { // update cached pointer it->mFrameState = &it->mTimer.getFrameState(); } + + // also update frame states of timers on stack + LLFastTimer* cur_timerp = LLFastTimer::sCurTimerData.mCurTimer; + while(cur_timerp->mLastTimerData.mCurTimer != cur_timerp) + { + cur_timerp->mFrameState = &cur_timerp->mFrameState->mTimer->getFrameState(); + cur_timerp = cur_timerp->mLastTimerData.mCurTimer; + } } //static @@ -234,10 +244,23 @@ U64 LLFastTimer::countsPerSecond() // counts per second for the *32-bit* timer #else // windows or x86-mac or x86-linux or x86-solaris U64 LLFastTimer::countsPerSecond() // counts per second for the *32-bit* timer { +#if USE_RDTSC || !LL_WINDOWS //getCPUFrequency returns MHz and sCPUClockFrequency wants to be in Hz static U64 sCPUClockFrequency = U64(LLProcessorInfo().getCPUFrequency()*1000000.0); // we drop the low-order byte in our timers, so report a lower frequency +#else + // If we're not using RDTSC, each fasttimer tick is just a performance counter tick. + // Not redefining the clock frequency itself (in llprocessor.cpp/calculate_cpu_frequency()) + // since that would change displayed MHz stats for CPUs + static bool firstcall = true; + static U64 sCPUClockFrequency; + if (firstcall) + { + QueryPerformanceFrequency((LARGE_INTEGER*)&sCPUClockFrequency); + firstcall = false; + } +#endif return sCPUClockFrequency >> 8; } #endif @@ -280,14 +303,15 @@ LLFastTimer::NamedTimer::~NamedTimer() std::string LLFastTimer::NamedTimer::getToolTip(S32 history_idx) { + F64 ms_multiplier = 1000.0 / (F64)LLFastTimer::countsPerSecond(); if (history_idx < 0) { - // by default, show average number of calls - return llformat("%s (%d calls)", getName().c_str(), (S32)getCallAverage()); + // by default, show average number of call + return llformat("%s (%d ms, %d calls)", getName().c_str(), (S32)(getCountAverage() * ms_multiplier), (S32)getCallAverage()); } else { - return llformat("%s (%d calls)", getName().c_str(), (S32)getHistoricalCalls(history_idx)); + return llformat("%s (%d ms, %d calls)", getName().c_str(), (S32)(getHistoricalCount(history_idx) * ms_multiplier), (S32)getHistoricalCalls(history_idx)); } } @@ -370,10 +394,7 @@ void LLFastTimer::NamedTimer::buildHierarchy() // set up initial tree { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; if (&timer == NamedTimerFactory::instance().getRootTimer()) continue; @@ -482,16 +503,26 @@ void LLFastTimer::NamedTimer::resetFrame() { if (sLog) { //output current frame counts to performance log + + static S32 call_count = 0; + if (call_count % 100 == 0) + { + llinfos << "countsPerSecond (32 bit): " << countsPerSecond() << llendl; + llinfos << "get_clock_count (64 bit): " << get_clock_count() << llendl; + llinfos << "LLProcessorInfo().getCPUFrequency() " << LLProcessorInfo().getCPUFrequency() << llendl; + llinfos << "getCPUClockCount32() " << getCPUClockCount32() << llendl; + llinfos << "getCPUClockCount64() " << getCPUClockCount64() << llendl; + llinfos << "elapsed sec " << ((F64)getCPUClockCount64())/((F64)LLProcessorInfo().getCPUFrequency()*1000000.0) << llendl; + } + call_count++; + F64 iclock_freq = 1000.0 / countsPerSecond(); // good place to calculate clock frequency F64 total_time = 0; LLSD sd; { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (NamedTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; FrameState& info = timer.getFrameState(); @@ -528,7 +559,7 @@ void LLFastTimer::NamedTimer::resetFrame() llassert_always(timerp->mFrameStateIndex < (S32)getFrameStateList().size()); } - // sort timers by dfs traversal order to improve cache coherency + // sort timers by DFS traversal order to improve cache coherency std::sort(getFrameStateList().begin(), getFrameStateList().end(), SortTimersDFS()); // update pointers into framestatelist now that we've sorted it @@ -536,10 +567,7 @@ void LLFastTimer::NamedTimer::resetFrame() // reset for next frame { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (NamedTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; @@ -583,10 +611,7 @@ void LLFastTimer::NamedTimer::reset() // reset all history { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (NamedTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; if (&timer != NamedTimerFactory::instance().getRootTimer()) @@ -669,17 +694,7 @@ void LLFastTimer::nextFrame() llinfos << "Slow frame, fast timers inaccurate" << llendl; } - if (sPauseHistory) - { - sResetHistory = true; - } - else if (sResetHistory) - { - sLastFrameIndex = 0; - sCurFrameIndex = 0; - sResetHistory = false; - } - else // not paused + if (!sPauseHistory) { NamedTimer::processTimes(); sLastFrameIndex = sCurFrameIndex++; @@ -763,3 +778,144 @@ LLFastTimer::LLFastTimer(LLFastTimer::FrameState* state) ////////////////////////////////////////////////////////////////////////////// +// +// Important note: These implementations must be FAST! +// + + +#if LL_WINDOWS +// +// Windows implementation of CPU clock +// + +// +// NOTE: put back in when we aren't using platform sdk anymore +// +// because MS has different signatures for these functions in winnt.h +// need to rename them to avoid conflicts +//#define _interlockedbittestandset _renamed_interlockedbittestandset +//#define _interlockedbittestandreset _renamed_interlockedbittestandreset +//#include <intrin.h> +//#undef _interlockedbittestandset +//#undef _interlockedbittestandreset + +//inline U32 LLFastTimer::getCPUClockCount32() +//{ +// U64 time_stamp = __rdtsc(); +// return (U32)(time_stamp >> 8); +//} +// +//// return full timer value, *not* shifted by 8 bits +//inline U64 LLFastTimer::getCPUClockCount64() +//{ +// return __rdtsc(); +//} + +// shift off lower 8 bits for lower resolution but longer term timing +// on 1Ghz machine, a 32-bit word will hold ~1000 seconds of timing +#if USE_RDTSC +U32 LLFastTimer::getCPUClockCount32() +{ + U32 ret_val; + __asm + { + _emit 0x0f + _emit 0x31 + shr eax,8 + shl edx,24 + or eax, edx + mov dword ptr [ret_val], eax + } + return ret_val; +} + +// return full timer value, *not* shifted by 8 bits +U64 LLFastTimer::getCPUClockCount64() +{ + U64 ret_val; + __asm + { + _emit 0x0f + _emit 0x31 + mov eax,eax + mov edx,edx + mov dword ptr [ret_val+4], edx + mov dword ptr [ret_val], eax + } + return ret_val; +} + +std::string LLFastTimer::sClockType = "rdtsc"; + +#else +//LL_COMMON_API U64 get_clock_count(); // in lltimer.cpp +// These use QueryPerformanceCounter, which is arguably fine and also works on AMD architectures. +U32 LLFastTimer::getCPUClockCount32() +{ + return (U32)(get_clock_count()>>8); +} + +U64 LLFastTimer::getCPUClockCount64() +{ + return get_clock_count(); +} + +std::string LLFastTimer::sClockType = "QueryPerformanceCounter"; +#endif + +#endif + + +#if (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) +// +// Linux and Solaris implementation of CPU clock - non-x86. +// This is accurate but SLOW! Only use out of desperation. +// +// Try to use the MONOTONIC clock if available, this is a constant time counter +// with nanosecond resolution (but not necessarily accuracy) and attempts are +// made to synchronize this value between cores at kernel start. It should not +// be affected by CPU frequency. If not available use the REALTIME clock, but +// this may be affected by NTP adjustments or other user activity affecting +// the system time. +U64 LLFastTimer::getCPUClockCount64() +{ + struct timespec tp; + +#ifdef CLOCK_MONOTONIC // MONOTONIC supported at build-time? + if (-1 == clock_gettime(CLOCK_MONOTONIC,&tp)) // if MONOTONIC isn't supported at runtime then ouch, try REALTIME +#endif + clock_gettime(CLOCK_REALTIME,&tp); + + return (tp.tv_sec*LLFastTimer::sClockResolution)+tp.tv_nsec; +} + +U32 LLFastTimer::getCPUClockCount32() +{ + return (U32)(LLFastTimer::getCPUClockCount64() >> 8); +} + +std::string LLFastTimer::sClockType = "clock_gettime"; + +#endif // (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) + + +#if (LL_LINUX || LL_SOLARIS || LL_DARWIN) && (defined(__i386__) || defined(__amd64__)) +// +// Mac+Linux+Solaris FAST x86 implementation of CPU clock +U32 LLFastTimer::getCPUClockCount32() +{ + U64 x; + __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); + return (U32)(x >> 8); +} + +U64 LLFastTimer::getCPUClockCount64() +{ + U64 x; + __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); + return x; +} + +std::string LLFastTimer::sClockType = "rdtsc"; +#endif + diff --git a/indra/llcommon/llfasttimer_class.h b/indra/llcommon/llfasttimer_class.h index eb9789682b..f481e968a6 100644 --- a/indra/llcommon/llfasttimer_class.h +++ b/indra/llcommon/llfasttimer_class.h @@ -31,12 +31,15 @@ #define FAST_TIMER_ON 1 #define TIME_FAST_TIMERS 0 +#define DEBUG_FAST_TIMER_THREADS 1 class LLMutex; #include <queue> #include "llsd.h" +LL_COMMON_API void assert_main_thread(); + class LL_COMMON_API LLFastTimer { public: @@ -63,7 +66,7 @@ public: public: ~NamedTimer(); - enum { HISTORY_NUM = 60 }; + enum { HISTORY_NUM = 300 }; const std::string& getName() const { return mName; } NamedTimer* getParent() const { return mParent; } @@ -176,6 +179,11 @@ public: U64 timer_end = getCPUClockCount64(); sTimerCycles += timer_end - timer_start; #endif +#if DEBUG_FAST_TIMER_THREADS +#if !LL_RELEASE + assert_main_thread(); +#endif +#endif } LL_FORCE_INLINE ~LLFastTimer() @@ -245,6 +253,7 @@ public: U32 mChildTime; }; static CurTimerData sCurTimerData; + static std::string sClockType; private: static U32 getCPUClockCount32(); diff --git a/indra/llcommon/llfile.cpp b/indra/llcommon/llfile.cpp index 8f02391e75..c32a776c3f 100644 --- a/indra/llcommon/llfile.cpp +++ b/indra/llcommon/llfile.cpp @@ -92,6 +92,17 @@ LLFILE* LLFile::_fsopen(const std::string& filename, const char* mode, int shari #endif } +int LLFile::close(LLFILE * file) +{ + int ret_value = 0; + if (file) + { + ret_value = fclose(file); + } + return ret_value; +} + + int LLFile::remove(const std::string& filename) { #if LL_WINDOWS diff --git a/indra/llcommon/llfile.h b/indra/llcommon/llfile.h index 4913af7cb5..dd7d36513a 100644 --- a/indra/llcommon/llfile.h +++ b/indra/llcommon/llfile.h @@ -71,6 +71,8 @@ public: static LLFILE* fopen(const std::string& filename,const char* accessmode); /* Flawfinder: ignore */ static LLFILE* _fsopen(const std::string& filename,const char* accessmode,int sharingFlag); + static int close(LLFILE * file); + // perms is a permissions mask like 0777 or 0700. In most cases it will // be overridden by the user's umask. It is ignored on Windows. static int mkdir(const std::string& filename, int perms = 0700); diff --git a/indra/llcommon/llfoldertype.cpp b/indra/llcommon/llfoldertype.cpp index ebc79af412..f6d0f5bce8 100644 --- a/indra/llcommon/llfoldertype.cpp +++ b/indra/llcommon/llfoldertype.cpp @@ -89,7 +89,12 @@ LLFolderDictionary::LLFolderDictionary() addEntry(LLFolderType::FT_CURRENT_OUTFIT, new FolderEntry("current", TRUE)); addEntry(LLFolderType::FT_OUTFIT, new FolderEntry("outfit", FALSE)); addEntry(LLFolderType::FT_MY_OUTFITS, new FolderEntry("my_otfts", TRUE)); + + addEntry(LLFolderType::FT_MESH, new FolderEntry("mesh", TRUE)); + addEntry(LLFolderType::FT_INBOX, new FolderEntry("inbox", TRUE)); + addEntry(LLFolderType::FT_OUTBOX, new FolderEntry("outbox", TRUE)); + addEntry(LLFolderType::FT_BASIC_ROOT, new FolderEntry("basic_rt", TRUE)); addEntry(LLFolderType::FT_NONE, new FolderEntry("-1", FALSE)); }; diff --git a/indra/llcommon/llfoldertype.h b/indra/llcommon/llfoldertype.h index 936fbed17d..a0c847914f 100644 --- a/indra/llcommon/llfoldertype.h +++ b/indra/llcommon/llfoldertype.h @@ -80,9 +80,14 @@ public: FT_OUTFIT = 47, FT_MY_OUTFITS = 48, - FT_INBOX = 49, + FT_MESH = 49, - FT_COUNT = 50, + FT_INBOX = 50, + FT_OUTBOX = 51, + + FT_BASIC_ROOT = 52, + + FT_COUNT, FT_NONE = -1 }; diff --git a/indra/llcommon/llinstancetracker.cpp b/indra/llcommon/llinstancetracker.cpp index 89bc6cca39..5dc3ea5d7b 100644 --- a/indra/llcommon/llinstancetracker.cpp +++ b/indra/llcommon/llinstancetracker.cpp @@ -32,6 +32,18 @@ // external library headers // other Linden headers -// llinstancetracker.h is presently header-only. This file exists only because our CMake -// test macro ADD_BUILD_TEST requires it. -int dummy = 0; +//static +void * & LLInstanceTrackerBase::getInstances(std::type_info const & info) +{ + typedef std::map<std::string, void *> InstancesMap; + static InstancesMap instances; + + // std::map::insert() is just what we want here. You attempt to insert a + // (key, value) pair. If the specified key doesn't yet exist, it inserts + // the pair and returns a std::pair of (iterator, true). If the specified + // key DOES exist, insert() simply returns (iterator, false). One lookup + // handles both cases. + return instances.insert(InstancesMap::value_type(info.name(), + InstancesMap::mapped_type())) + .first->second; +} diff --git a/indra/llcommon/llinstancetracker.h b/indra/llcommon/llinstancetracker.h index 4945461d62..34d841a4e0 100644 --- a/indra/llcommon/llinstancetracker.h +++ b/indra/llcommon/llinstancetracker.h @@ -29,6 +29,7 @@ #define LL_LLINSTANCETRACKER_H #include <map> +#include <typeinfo> #include "string_table.h" #include <boost/utility.hpp> @@ -37,21 +38,132 @@ #include <boost/iterator/transform_iterator.hpp> #include <boost/iterator/indirect_iterator.hpp> +/** + * Base class manages "class-static" data that must actually have singleton + * semantics: one instance per process, rather than one instance per module as + * sometimes happens with data simply declared static. + */ +class LL_COMMON_API LLInstanceTrackerBase : public boost::noncopyable +{ +protected: + /// Get a process-unique void* pointer slot for the specified type_info + static void * & getInstances(std::type_info const & info); + + /// Find or create a STATICDATA instance for the specified TRACKED class. + /// STATICDATA must be default-constructible. + template<typename STATICDATA, class TRACKED> + static STATICDATA& getStatic() + { + void *& instances = getInstances(typeid(TRACKED)); + if (! instances) + { + instances = new STATICDATA; + } + return *static_cast<STATICDATA*>(instances); + } + + /// It's not essential to derive your STATICDATA (for use with + /// getStatic()) from StaticBase; it's just that both known + /// implementations do. + struct StaticBase + { + StaticBase(): + sIterationNestDepth(0) + {} + S32 sIterationNestDepth; + }; +}; + /// This mix-in class adds support for tracking all instances of the specified class parameter T /// The (optional) key associates a value of type KEY with a given instance of T, for quick lookup /// If KEY is not provided, then instances are stored in a simple set /// @NOTE: see explicit specialization below for default KEY==T* case template<typename T, typename KEY = T*> -class LLInstanceTracker : boost::noncopyable +class LLInstanceTracker : public LLInstanceTrackerBase { + typedef LLInstanceTracker<T, KEY> MyT; typedef typename std::map<KEY, T*> InstanceMap; - typedef boost::function<const KEY&(typename InstanceMap::value_type&)> KeyGetter; - typedef boost::function<T*(typename InstanceMap::value_type&)> InstancePtrGetter; + struct StaticData: public StaticBase + { + InstanceMap sMap; + }; + static StaticData& getStatic() { return LLInstanceTrackerBase::getStatic<StaticData, MyT>(); } + static InstanceMap& getMap_() { return getStatic().sMap; } + public: - /// Dereferencing key_iter gives you a const KEY& - typedef boost::transform_iterator<KeyGetter, typename InstanceMap::iterator> key_iter; - /// Dereferencing instance_iter gives you a T& - typedef boost::indirect_iterator< boost::transform_iterator<InstancePtrGetter, typename InstanceMap::iterator> > instance_iter; + class instance_iter : public boost::iterator_facade<instance_iter, T, boost::forward_traversal_tag> + { + public: + typedef boost::iterator_facade<instance_iter, T, boost::forward_traversal_tag> super_t; + + instance_iter(const typename InstanceMap::iterator& it) + : mIterator(it) + { + ++getStatic().sIterationNestDepth; + } + + ~instance_iter() + { + --getStatic().sIterationNestDepth; + } + + + private: + friend class boost::iterator_core_access; + + void increment() { mIterator++; } + bool equal(instance_iter const& other) const + { + return mIterator == other.mIterator; + } + + T& dereference() const + { + return *(mIterator->second); + } + + typename InstanceMap::iterator mIterator; + }; + + class key_iter : public boost::iterator_facade<key_iter, KEY, boost::forward_traversal_tag> + { + public: + typedef boost::iterator_facade<key_iter, KEY, boost::forward_traversal_tag> super_t; + + key_iter(typename InstanceMap::iterator it) + : mIterator(it) + { + ++getStatic().sIterationNestDepth; + } + + key_iter(const key_iter& other) + : mIterator(other.mIterator) + { + ++getStatic().sIterationNestDepth; + } + + ~key_iter() + { + --getStatic().sIterationNestDepth; + } + + + private: + friend class boost::iterator_core_access; + + void increment() { mIterator++; } + bool equal(key_iter const& other) const + { + return mIterator == other.mIterator; + } + + KEY& dereference() const + { + return const_cast<KEY&>(mIterator->first); + } + + typename InstanceMap::iterator mIterator; + }; static T* getInstance(const KEY& k) { @@ -59,132 +171,136 @@ public: return (found == getMap_().end()) ? NULL : found->second; } - static key_iter beginKeys() - { - return boost::make_transform_iterator(getMap_().begin(), - boost::bind(&InstanceMap::value_type::first, _1)); + static instance_iter beginInstances() + { + return instance_iter(getMap_().begin()); } - static key_iter endKeys() + + static instance_iter endInstances() { - return boost::make_transform_iterator(getMap_().end(), - boost::bind(&InstanceMap::value_type::first, _1)); + return instance_iter(getMap_().end()); } - static instance_iter beginInstances() + + static S32 instanceCount() { return getMap_().size(); } + + static key_iter beginKeys() { - return instance_iter(boost::make_transform_iterator(getMap_().begin(), - boost::bind(&InstanceMap::value_type::second, _1))); + return key_iter(getMap_().begin()); } - static instance_iter endInstances() + static key_iter endKeys() { - return instance_iter(boost::make_transform_iterator(getMap_().end(), - boost::bind(&InstanceMap::value_type::second, _1))); + return key_iter(getMap_().end()); } - static S32 instanceCount() { return getMap_().size(); } + protected: - LLInstanceTracker(KEY key) { add_(key); } - virtual ~LLInstanceTracker() { remove_(); } + LLInstanceTracker(KEY key) + { + // make sure static data outlives all instances + getStatic(); + add_(key); + } + virtual ~LLInstanceTracker() + { + // it's unsafe to delete instances of this type while all instances are being iterated over. + llassert_always(getStatic().sIterationNestDepth == 0); + remove_(); + } virtual void setKey(KEY key) { remove_(); add_(key); } - virtual const KEY& getKey() const { return mKey; } + virtual const KEY& getKey() const { return mInstanceKey; } private: void add_(KEY key) { - mKey = key; + mInstanceKey = key; getMap_()[key] = static_cast<T*>(this); } void remove_() { - getMap_().erase(mKey); + getMap_().erase(mInstanceKey); } - static InstanceMap& getMap_() - { - if (! sInstances) - { - sInstances = new InstanceMap; - } - return *sInstances; - } - private: - - KEY mKey; - static InstanceMap* sInstances; + KEY mInstanceKey; }; /// explicit specialization for default case where KEY is T* /// use a simple std::set<T*> template<typename T> -class LLInstanceTracker<T, T*> +class LLInstanceTracker<T, T*> : public LLInstanceTrackerBase { + typedef LLInstanceTracker<T, T*> MyT; typedef typename std::set<T*> InstanceSet; + struct StaticData: public StaticBase + { + InstanceSet sSet; + }; + static StaticData& getStatic() { return LLInstanceTrackerBase::getStatic<StaticData, MyT>(); } + static InstanceSet& getSet_() { return getStatic().sSet; } + public: - /// Dereferencing key_iter gives you a T* (since T* is the key) - typedef typename InstanceSet::iterator key_iter; - /// Dereferencing instance_iter gives you a T& - typedef boost::indirect_iterator<key_iter> instance_iter; /// for completeness of analogy with the generic implementation static T* getInstance(T* k) { return k; } static S32 instanceCount() { return getSet_().size(); } - // Instantiate this to get access to iterators for this type. It's a 'guard' in the sense - // that it treats deletes of this type as errors as long as there is an instance of - // this class alive in scope somewhere (i.e. deleting while iterating is bad). - class LLInstanceTrackerScopedGuard + class instance_iter : public boost::iterator_facade<instance_iter, T, boost::forward_traversal_tag> { public: - LLInstanceTrackerScopedGuard() + instance_iter(const typename InstanceSet::iterator& it) + : mIterator(it) + { + ++getStatic().sIterationNestDepth; + } + + instance_iter(const instance_iter& other) + : mIterator(other.mIterator) + { + ++getStatic().sIterationNestDepth; + } + + ~instance_iter() { - ++sIterationNestDepth; + --getStatic().sIterationNestDepth; } - ~LLInstanceTrackerScopedGuard() + private: + friend class boost::iterator_core_access; + + void increment() { mIterator++; } + bool equal(instance_iter const& other) const { - --sIterationNestDepth; + return mIterator == other.mIterator; } - static instance_iter beginInstances() { return instance_iter(getSet_().begin()); } - static instance_iter endInstances() { return instance_iter(getSet_().end()); } - static key_iter beginKeys() { return getSet_().begin(); } - static key_iter endKeys() { return getSet_().end(); } + T& dereference() const + { + return **mIterator; + } + + typename InstanceSet::iterator mIterator; }; + static instance_iter beginInstances() { return instance_iter(getSet_().begin()); } + static instance_iter endInstances() { return instance_iter(getSet_().end()); } + protected: LLInstanceTracker() { - // it's safe but unpredictable to create instances of this type while all instances are being iterated over. I hate unpredictable. This assert will probably be turned on early in the next development cycle. - //llassert(sIterationNestDepth == 0); + // make sure static data outlives all instances + getStatic(); getSet_().insert(static_cast<T*>(this)); } virtual ~LLInstanceTracker() { // it's unsafe to delete instances of this type while all instances are being iterated over. - llassert(sIterationNestDepth == 0); + llassert_always(getStatic().sIterationNestDepth == 0); getSet_().erase(static_cast<T*>(this)); } LLInstanceTracker(const LLInstanceTracker& other) { - //llassert(sIterationNestDepth == 0); getSet_().insert(static_cast<T*>(this)); } - - static InstanceSet& getSet_() - { - if (! sInstances) - { - sInstances = new InstanceSet; - } - return *sInstances; - } - - static InstanceSet* sInstances; - static S32 sIterationNestDepth; }; -template <typename T, typename KEY> typename LLInstanceTracker<T, KEY>::InstanceMap* LLInstanceTracker<T, KEY>::sInstances = NULL; -template <typename T> typename LLInstanceTracker<T, T*>::InstanceSet* LLInstanceTracker<T, T*>::sInstances = NULL; -template <typename T> S32 LLInstanceTracker<T, T*>::sIterationNestDepth = 0; - #endif diff --git a/indra/llcommon/lllslconstants.h b/indra/llcommon/lllslconstants.h index 539c807020..9f32598e61 100644 --- a/indra/llcommon/lllslconstants.h +++ b/indra/llcommon/lllslconstants.h @@ -181,7 +181,7 @@ const S32 OBJECT_GROUP = 7; const S32 OBJECT_CREATOR = 8; // llTextBox() magic token string - yes this is a hack. sue me. -const std::string TEXTBOX_MAGIC_TOKEN = "!!llTextBox!!"; +char const* const TEXTBOX_MAGIC_TOKEN = "!!llTextBox!!"; // changed() event flags const U32 CHANGED_NONE = 0x0; diff --git a/indra/llcommon/llmd5.cpp b/indra/llcommon/llmd5.cpp index 75fde8e5ba..1409c55d1c 100644 --- a/indra/llcommon/llmd5.cpp +++ b/indra/llcommon/llmd5.cpp @@ -175,7 +175,7 @@ void LLMD5::update(std::istream& stream){ while (stream.good()){ stream.read( (char*)buffer, BLOCK_LEN); /* Flawfinder: ignore */ // note that return value of read is unusable. - len=stream.gcount(); + len=(int)stream.gcount(); update(buffer, len); } diff --git a/indra/llcommon/llmd5.h b/indra/llcommon/llmd5.h index c8acbbe591..1526e6ac3c 100644 --- a/indra/llcommon/llmd5.h +++ b/indra/llcommon/llmd5.h @@ -103,7 +103,7 @@ public: void raw_digest(unsigned char *array) const; // provide 16-byte array for binary data void hex_digest(char *string) const; // provide 33-byte array for ascii-hex string - friend std::ostream& operator<< (std::ostream&, LLMD5 context); + friend LL_COMMON_API std::ostream& operator<< (std::ostream&, LLMD5 context); private: diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index a502d1a7eb..3b9758f996 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -26,8 +26,13 @@ #include "linden_common.h" + +//#if MEM_TRACK_MEM +#include "llthread.h" +//#endif + #if defined(LL_WINDOWS) -# include <windows.h> +//# include <windows.h> # include <psapi.h> #elif defined(LL_DARWIN) # include <sys/types.h> @@ -39,10 +44,22 @@ #include "llmemory.h" +#include "llsys.h" +#include "llframetimer.h" //---------------------------------------------------------------------------- //static char* LLMemory::reserveMem = 0; +U32 LLMemory::sAvailPhysicalMemInKB = U32_MAX ; +U32 LLMemory::sMaxPhysicalMemInKB = 0; +U32 LLMemory::sAllocatedMemInKB = 0; +U32 LLMemory::sAllocatedPageSizeInKB = 0 ; +U32 LLMemory::sMaxHeapSizeInKB = U32_MAX ; +BOOL LLMemory::sEnableMemoryFailurePrevention = FALSE; + +#if __DEBUG_PRIVATE_MEM__ +LLPrivateMemoryPoolManager::mem_allocation_info_t LLPrivateMemoryPoolManager::sMemAllocationTracker; +#endif //static void LLMemory::initClass() @@ -67,6 +84,162 @@ void LLMemory::freeReserve() reserveMem = NULL; } +//static +void LLMemory::initMaxHeapSizeGB(F32 max_heap_size_gb, BOOL prevent_heap_failure) +{ + sMaxHeapSizeInKB = (U32)(max_heap_size_gb * 1024 * 1024) ; + sEnableMemoryFailurePrevention = prevent_heap_failure ; +} + +//static +void LLMemory::updateMemoryInfo() +{ +#if LL_WINDOWS + HANDLE self = GetCurrentProcess(); + PROCESS_MEMORY_COUNTERS counters; + + if (!GetProcessMemoryInfo(self, &counters, sizeof(counters))) + { + llwarns << "GetProcessMemoryInfo failed" << llendl; + return ; + } + + sAllocatedMemInKB = (U32)(counters.WorkingSetSize / 1024) ; + sAllocatedPageSizeInKB = (U32)(counters.PagefileUsage / 1024) ; + + U32 avail_phys, avail_virtual; + LLMemoryInfo::getAvailableMemoryKB(avail_phys, avail_virtual) ; + sMaxPhysicalMemInKB = llmin(avail_phys + sAllocatedMemInKB, sMaxHeapSizeInKB); + + if(sMaxPhysicalMemInKB > sAllocatedMemInKB) + { + sAvailPhysicalMemInKB = sMaxPhysicalMemInKB - sAllocatedMemInKB ; + } + else + { + sAvailPhysicalMemInKB = 0 ; + } +#else + //not valid for other systems for now. + sAllocatedMemInKB = (U32)(LLMemory::getCurrentRSS() / 1024) ; + sMaxPhysicalMemInKB = U32_MAX ; + sAvailPhysicalMemInKB = U32_MAX ; +#endif + + return ; +} + +// +//this function is to test if there is enough space with the size in the virtual address space. +//it does not do any real allocation +//if success, it returns the address where the memory chunk can fit in; +//otherwise it returns NULL. +// +//static +void* LLMemory::tryToAlloc(void* address, U32 size) +{ +#if LL_WINDOWS + address = VirtualAlloc(address, size, MEM_RESERVE | MEM_TOP_DOWN, PAGE_NOACCESS) ; + if(address) + { + if(!VirtualFree(address, 0, MEM_RELEASE)) + { + llerrs << "error happens when free some memory reservation." << llendl ; + } + } + return address ; +#else + return (void*)0x01 ; //skip checking +#endif +} + +//static +void LLMemory::logMemoryInfo(BOOL update) +{ + if(update) + { + updateMemoryInfo() ; + LLPrivateMemoryPoolManager::getInstance()->updateStatistics() ; + } + + llinfos << "Current allocated physical memory(KB): " << sAllocatedMemInKB << llendl ; + llinfos << "Current allocated page size (KB): " << sAllocatedPageSizeInKB << llendl ; + llinfos << "Current availabe physical memory(KB): " << sAvailPhysicalMemInKB << llendl ; + llinfos << "Current max usable memory(KB): " << sMaxPhysicalMemInKB << llendl ; + + llinfos << "--- private pool information -- " << llendl ; + llinfos << "Total reserved (KB): " << LLPrivateMemoryPoolManager::getInstance()->mTotalReservedSize / 1024 << llendl ; + llinfos << "Total allocated (KB): " << LLPrivateMemoryPoolManager::getInstance()->mTotalAllocatedSize / 1024 << llendl ; +} + +//return 0: everything is normal; +//return 1: the memory pool is low, but not in danger; +//return -1: the memory pool is in danger, is about to crash. +//static +bool LLMemory::isMemoryPoolLow() +{ + static const U32 LOW_MEMEOY_POOL_THRESHOLD_KB = 64 * 1024 ; //64 MB for emergency use + const static U32 MAX_SIZE_CHECKED_MEMORY_BLOCK = 64 * 1024 * 1024 ; //64 MB + static void* last_reserved_address = NULL ; + + if(!sEnableMemoryFailurePrevention) + { + return false ; //no memory failure prevention. + } + + if(sAvailPhysicalMemInKB < (LOW_MEMEOY_POOL_THRESHOLD_KB >> 2)) //out of physical memory + { + return true ; + } + + if(sAllocatedPageSizeInKB + (LOW_MEMEOY_POOL_THRESHOLD_KB >> 2) > sMaxHeapSizeInKB) //out of virtual address space. + { + return true ; + } + + bool is_low = (S32)(sAvailPhysicalMemInKB < LOW_MEMEOY_POOL_THRESHOLD_KB || + sAllocatedPageSizeInKB + LOW_MEMEOY_POOL_THRESHOLD_KB > sMaxHeapSizeInKB) ; + + //check the virtual address space fragmentation + if(!is_low) + { + if(!last_reserved_address) + { + last_reserved_address = LLMemory::tryToAlloc(last_reserved_address, MAX_SIZE_CHECKED_MEMORY_BLOCK) ; + } + else + { + last_reserved_address = LLMemory::tryToAlloc(last_reserved_address, MAX_SIZE_CHECKED_MEMORY_BLOCK) ; + if(!last_reserved_address) //failed, try once more + { + last_reserved_address = LLMemory::tryToAlloc(last_reserved_address, MAX_SIZE_CHECKED_MEMORY_BLOCK) ; + } + } + + is_low = !last_reserved_address ; //allocation failed + } + + return is_low ; +} + +//static +U32 LLMemory::getAvailableMemKB() +{ + return sAvailPhysicalMemInKB ; +} + +//static +U32 LLMemory::getMaxMemKB() +{ + return sMaxPhysicalMemInKB ; +} + +//static +U32 LLMemory::getAllocatedMemKB() +{ + return sAllocatedMemInKB ; +} + void* ll_allocate (size_t size) { if (size == 0) @@ -82,11 +255,6 @@ void* ll_allocate (size_t size) return p; } -void ll_release (void *p) -{ - free(p); -} - //---------------------------------------------------------------------------- #if defined(LL_WINDOWS) @@ -105,6 +273,20 @@ U64 LLMemory::getCurrentRSS() return counters.WorkingSetSize; } +//static +U32 LLMemory::getWorkingSetSize() +{ + PROCESS_MEMORY_COUNTERS pmc ; + U32 ret = 0 ; + + if (GetProcessMemoryInfo( GetCurrentProcess(), &pmc, sizeof(pmc)) ) + { + ret = pmc.WorkingSetSize ; + } + + return ret ; +} + #elif defined(LL_DARWIN) /* @@ -151,6 +333,11 @@ U64 LLMemory::getCurrentRSS() return residentSize; } +U32 LLMemory::getWorkingSetSize() +{ + return 0 ; +} + #elif defined(LL_LINUX) U64 LLMemory::getCurrentRSS() @@ -185,6 +372,11 @@ bail: return rss; } +U32 LLMemory::getWorkingSetSize() +{ + return 0 ; +} + #elif LL_SOLARIS #include <sys/types.h> #include <sys/stat.h> @@ -213,6 +405,12 @@ U64 LLMemory::getCurrentRSS() return((U64)proc_psinfo.pr_rssize * 1024); } + +U32 LLMemory::getWorkingSetSize() +{ + return 0 ; +} + #else U64 LLMemory::getCurrentRSS() @@ -220,4 +418,1869 @@ U64 LLMemory::getCurrentRSS() return 0; } +U32 LLMemory::getWorkingSetSize() +{ + return 0; +} + +#endif + +//-------------------------------------------------------------------------------------------------- +#if MEM_TRACK_MEM +#include "llframetimer.h" + +//static +LLMemTracker* LLMemTracker::sInstance = NULL ; + +LLMemTracker::LLMemTracker() +{ + mLastAllocatedMem = LLMemory::getWorkingSetSize() ; + mCapacity = 128 ; + mCurIndex = 0 ; + mCounter = 0 ; + mDrawnIndex = 0 ; + mPaused = FALSE ; + + mMutexp = new LLMutex() ; + mStringBuffer = new char*[128] ; + mStringBuffer[0] = new char[mCapacity * 128] ; + for(S32 i = 1 ; i < mCapacity ; i++) + { + mStringBuffer[i] = mStringBuffer[i-1] + 128 ; + } +} + +LLMemTracker::~LLMemTracker() +{ + delete[] mStringBuffer[0] ; + delete[] mStringBuffer; + delete mMutexp ; +} + +//static +LLMemTracker* LLMemTracker::getInstance() +{ + if(!sInstance) + { + sInstance = new LLMemTracker() ; + } + return sInstance ; +} + +//static +void LLMemTracker::release() +{ + if(sInstance) + { + delete sInstance ; + sInstance = NULL ; + } +} + +//static +void LLMemTracker::track(const char* function, const int line) +{ + static const S32 MIN_ALLOCATION = 0 ; //1KB + + if(mPaused) + { + return ; + } + + U32 allocated_mem = LLMemory::getWorkingSetSize() ; + + LLMutexLock lock(mMutexp) ; + + S32 delta_mem = allocated_mem - mLastAllocatedMem ; + mLastAllocatedMem = allocated_mem ; + + if(delta_mem <= 0) + { + return ; //occupied memory does not grow + } + + if(delta_mem < MIN_ALLOCATION) + { + return ; + } + + char* buffer = mStringBuffer[mCurIndex++] ; + F32 time = (F32)LLFrameTimer::getElapsedSeconds() ; + S32 hours = (S32)(time / (60*60)); + S32 mins = (S32)((time - hours*(60*60)) / 60); + S32 secs = (S32)((time - hours*(60*60) - mins*60)); + strcpy(buffer, function) ; + sprintf(buffer + strlen(function), " line: %d DeltaMem: %d (bytes) Time: %d:%02d:%02d", line, delta_mem, hours,mins,secs) ; + + if(mCounter < mCapacity) + { + mCounter++ ; + } + if(mCurIndex >= mCapacity) + { + mCurIndex = 0 ; + } +} + + +//static +void LLMemTracker::preDraw(BOOL pause) +{ + mMutexp->lock() ; + + mPaused = pause ; + mDrawnIndex = mCurIndex - 1; + mNumOfDrawn = 0 ; +} + +//static +void LLMemTracker::postDraw() +{ + mMutexp->unlock() ; +} + +//static +const char* LLMemTracker::getNextLine() +{ + if(mNumOfDrawn >= mCounter) + { + return NULL ; + } + mNumOfDrawn++; + + if(mDrawnIndex < 0) + { + mDrawnIndex = mCapacity - 1 ; + } + + return mStringBuffer[mDrawnIndex--] ; +} + +#endif //MEM_TRACK_MEM +//-------------------------------------------------------------------------------------------------- + + +//-------------------------------------------------------------------------------------------------- +//-------------------------------------------------------------------------------------------------- +//minimum slot size and minimal slot size interval +const U32 ATOMIC_MEM_SLOT = 16 ; //bytes + +//minimum block sizes (page size) for small allocation, medium allocation, large allocation +const U32 MIN_BLOCK_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {2 << 10, 4 << 10, 16 << 10} ; // + +//maximum block sizes for small allocation, medium allocation, large allocation +const U32 MAX_BLOCK_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {64 << 10, 1 << 20, 4 << 20} ; + +//minimum slot sizes for small allocation, medium allocation, large allocation +const U32 MIN_SLOT_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {ATOMIC_MEM_SLOT, 2 << 10, 512 << 10}; + +//maximum slot sizes for small allocation, medium allocation, large allocation +const U32 MAX_SLOT_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {(2 << 10) - ATOMIC_MEM_SLOT, (512 - 2) << 10, 4 << 20}; + +//size of a block with multiple slots can not exceed CUT_OFF_SIZE +const U32 CUT_OFF_SIZE = (64 << 10) ; //64 KB + +//max number of slots in a block +const U32 MAX_NUM_SLOTS_IN_A_BLOCK = llmin(MIN_BLOCK_SIZES[0] / ATOMIC_MEM_SLOT, ATOMIC_MEM_SLOT * 8) ; + +//------------------------------------------------------------- +//align val to be integer times of ATOMIC_MEM_SLOT +U32 align(U32 val) +{ + U32 aligned = (val / ATOMIC_MEM_SLOT) * ATOMIC_MEM_SLOT ; + if(aligned < val) + { + aligned += ATOMIC_MEM_SLOT ; + } + + return aligned ; +} + +//------------------------------------------------------------- +//class LLPrivateMemoryPool::LLMemoryBlock +//------------------------------------------------------------- +// +//each memory block could fit for two page sizes: 0.75 * mSlotSize, which starts from the beginning of the memory chunk and grow towards the end of the +//the block; another is mSlotSize, which starts from the end of the block and grows towards the beginning of the block. +// +LLPrivateMemoryPool::LLMemoryBlock::LLMemoryBlock() +{ + //empty +} + +LLPrivateMemoryPool::LLMemoryBlock::~LLMemoryBlock() +{ + //empty +} + +//create and initialize a memory block +void LLPrivateMemoryPool::LLMemoryBlock::init(char* buffer, U32 buffer_size, U32 slot_size) +{ + mBuffer = buffer ; + mBufferSize = buffer_size ; + mSlotSize = slot_size ; + mTotalSlots = buffer_size / mSlotSize ; + + llassert_always(buffer_size / mSlotSize <= MAX_NUM_SLOTS_IN_A_BLOCK) ; //max number is 128 + + mAllocatedSlots = 0 ; + mDummySize = 0 ; + + //init the bit map. + //mark free bits + if(mTotalSlots > 32) //reserve extra space from mBuffer to store bitmap if needed. + { + mDummySize = ATOMIC_MEM_SLOT ; + mTotalSlots -= (mDummySize + mSlotSize - 1) / mSlotSize ; + mUsageBits = 0 ; + + S32 usage_bit_len = (mTotalSlots + 31) / 32 ; + + for(S32 i = 0 ; i < usage_bit_len - 1 ; i++) + { + *((U32*)mBuffer + i) = 0 ; + } + for(S32 i = usage_bit_len - 1 ; i < mDummySize / sizeof(U32) ; i++) + { + *((U32*)mBuffer + i) = 0xffffffff ; + } + + if(mTotalSlots & 31) + { + *((U32*)mBuffer + usage_bit_len - 2) = (0xffffffff << (mTotalSlots & 31)) ; + } + } + else//no extra bitmap space reserved + { + mUsageBits = 0 ; + if(mTotalSlots & 31) + { + mUsageBits = (0xffffffff << (mTotalSlots & 31)) ; + } + } + + mSelf = this ; + mNext = NULL ; + mPrev = NULL ; + + llassert_always(mTotalSlots > 0) ; +} + +//mark this block to be free with the memory [mBuffer, mBuffer + mBufferSize). +void LLPrivateMemoryPool::LLMemoryBlock::setBuffer(char* buffer, U32 buffer_size) +{ + mBuffer = buffer ; + mBufferSize = buffer_size ; + mSelf = NULL ; + mTotalSlots = 0 ; //set the block is free. +} + +//reserve a slot +char* LLPrivateMemoryPool::LLMemoryBlock::allocate() +{ + llassert_always(mAllocatedSlots < mTotalSlots) ; + + //find a free slot + U32* bits = NULL ; + U32 k = 0 ; + if(mUsageBits != 0xffffffff) + { + bits = &mUsageBits ; + } + else if(mDummySize > 0)//go to extra space + { + for(S32 i = 0 ; i < mDummySize / sizeof(U32); i++) + { + if(*((U32*)mBuffer + i) != 0xffffffff) + { + bits = (U32*)mBuffer + i ; + k = i + 1 ; + break ; + } + } + } + S32 idx = 0 ; + U32 tmp = *bits ; + for(; tmp & 1 ; tmp >>= 1, idx++) ; + + //set the slot reserved + if(!idx) + { + *bits |= 1 ; + } + else + { + *bits |= (1 << idx) ; + } + + mAllocatedSlots++ ; + + return mBuffer + mDummySize + (k * 32 + idx) * mSlotSize ; +} + +//free a slot +void LLPrivateMemoryPool::LLMemoryBlock::freeMem(void* addr) +{ + //bit index + U32 idx = ((U32)addr - (U32)mBuffer - mDummySize) / mSlotSize ; + + U32* bits = &mUsageBits ; + if(idx >= 32) + { + bits = (U32*)mBuffer + (idx - 32) / 32 ; + } + + //reset the bit + if(idx & 31) + { + *bits &= ~(1 << (idx & 31)) ; + } + else + { + *bits &= ~1 ; + } + + mAllocatedSlots-- ; +} + +//for debug use: reset the entire bitmap. +void LLPrivateMemoryPool::LLMemoryBlock::resetBitMap() +{ + for(S32 i = 0 ; i < mDummySize / sizeof(U32) ; i++) + { + *((U32*)mBuffer + i) = 0 ; + } + mUsageBits = 0 ; +} +//------------------------------------------------------------------- +//class LLMemoryChunk +//-------------------------------------------------------------------- +LLPrivateMemoryPool::LLMemoryChunk::LLMemoryChunk() +{ + //empty +} + +LLPrivateMemoryPool::LLMemoryChunk::~LLMemoryChunk() +{ + //empty +} + +//create and init a memory chunk +void LLPrivateMemoryPool::LLMemoryChunk::init(char* buffer, U32 buffer_size, U32 min_slot_size, U32 max_slot_size, U32 min_block_size, U32 max_block_size) +{ + mBuffer = buffer ; + mBufferSize = buffer_size ; + mAlloatedSize = 0 ; + + mMetaBuffer = mBuffer + sizeof(LLMemoryChunk) ; + + mMinBlockSize = min_block_size; //page size + mMinSlotSize = min_slot_size; + mMaxSlotSize = max_slot_size ; + mBlockLevels = mMaxSlotSize / mMinSlotSize ; + mPartitionLevels = max_block_size / mMinBlockSize + 1 ; + + S32 max_num_blocks = (buffer_size - sizeof(LLMemoryChunk) - mBlockLevels * sizeof(LLMemoryBlock*) - mPartitionLevels * sizeof(LLMemoryBlock*)) / + (mMinBlockSize + sizeof(LLMemoryBlock)) ; + //meta data space + mBlocks = (LLMemoryBlock*)mMetaBuffer ; //space reserved for all memory blocks. + mAvailBlockList = (LLMemoryBlock**)((char*)mBlocks + sizeof(LLMemoryBlock) * max_num_blocks) ; + mFreeSpaceList = (LLMemoryBlock**)((char*)mAvailBlockList + sizeof(LLMemoryBlock*) * mBlockLevels) ; + + //data buffer, which can be used for allocation + mDataBuffer = (char*)mFreeSpaceList + sizeof(LLMemoryBlock*) * mPartitionLevels ; + + //alignmnet + mDataBuffer = mBuffer + align(mDataBuffer - mBuffer) ; + + //init + for(U32 i = 0 ; i < mBlockLevels; i++) + { + mAvailBlockList[i] = NULL ; + } + for(U32 i = 0 ; i < mPartitionLevels ; i++) + { + mFreeSpaceList[i] = NULL ; + } + + //assign the entire chunk to the first block + mBlocks[0].mPrev = NULL ; + mBlocks[0].mNext = NULL ; + mBlocks[0].setBuffer(mDataBuffer, buffer_size - (mDataBuffer - mBuffer)) ; + addToFreeSpace(&mBlocks[0]) ; + + mNext = NULL ; + mPrev = NULL ; +} + +//static +U32 LLPrivateMemoryPool::LLMemoryChunk::getMaxOverhead(U32 data_buffer_size, U32 min_slot_size, + U32 max_slot_size, U32 min_block_size, U32 max_block_size) +{ + //for large allocations, reserve some extra memory for meta data to avoid wasting much + if(data_buffer_size / min_slot_size < 64) //large allocations + { + U32 overhead = sizeof(LLMemoryChunk) + (data_buffer_size / min_block_size) * sizeof(LLMemoryBlock) + + sizeof(LLMemoryBlock*) * (max_slot_size / min_slot_size) + sizeof(LLMemoryBlock*) * (max_block_size / min_block_size + 1) ; + + //round to integer times of min_block_size + overhead = ((overhead + min_block_size - 1) / min_block_size) * min_block_size ; + return overhead ; + } + else + { + return 0 ; //do not reserve extra overhead if for small allocations + } +} + +char* LLPrivateMemoryPool::LLMemoryChunk::allocate(U32 size) +{ + if(mMinSlotSize > size) + { + size = mMinSlotSize ; + } + if(mAlloatedSize + size > mBufferSize - (mDataBuffer - mBuffer)) + { + return NULL ; //no enough space in this chunk. + } + + char* p = NULL ; + U32 blk_idx = getBlockLevel(size); + + LLMemoryBlock* blk = NULL ; + + //check if there is free block available + if(mAvailBlockList[blk_idx]) + { + blk = mAvailBlockList[blk_idx] ; + p = blk->allocate() ; + + if(blk->isFull()) + { + popAvailBlockList(blk_idx) ; + } + } + + //ask for a new block + if(!p) + { + blk = addBlock(blk_idx) ; + if(blk) + { + p = blk->allocate() ; + + if(blk->isFull()) + { + popAvailBlockList(blk_idx) ; + } + } + } + + //ask for space from larger blocks + if(!p) + { + for(S32 i = blk_idx + 1 ; i < mBlockLevels; i++) + { + if(mAvailBlockList[i]) + { + blk = mAvailBlockList[i] ; + p = blk->allocate() ; + + if(blk->isFull()) + { + popAvailBlockList(i) ; + } + break ; + } + } + } + + if(p && blk) + { + mAlloatedSize += blk->getSlotSize() ; + } + return p ; +} + +void LLPrivateMemoryPool::LLMemoryChunk::freeMem(void* addr) +{ + U32 blk_idx = getPageIndex((U32)addr) ; + LLMemoryBlock* blk = (LLMemoryBlock*)(mMetaBuffer + blk_idx * sizeof(LLMemoryBlock)) ; + blk = blk->mSelf ; + + bool was_full = blk->isFull() ; + blk->freeMem(addr) ; + mAlloatedSize -= blk->getSlotSize() ; + + if(blk->empty()) + { + removeBlock(blk) ; + } + else if(was_full) + { + addToAvailBlockList(blk) ; + } +} + +bool LLPrivateMemoryPool::LLMemoryChunk::empty() +{ + return !mAlloatedSize ; +} + +bool LLPrivateMemoryPool::LLMemoryChunk::containsAddress(const char* addr) const +{ + return (U32)mBuffer <= (U32)addr && (U32)mBuffer + mBufferSize > (U32)addr ; +} + +//debug use +void LLPrivateMemoryPool::LLMemoryChunk::dump() +{ +#if 0 + //sanity check + //for(S32 i = 0 ; i < mBlockLevels ; i++) + //{ + // LLMemoryBlock* blk = mAvailBlockList[i] ; + // while(blk) + // { + // blk_list.push_back(blk) ; + // blk = blk->mNext ; + // } + //} + for(S32 i = 0 ; i < mPartitionLevels ; i++) + { + LLMemoryBlock* blk = mFreeSpaceList[i] ; + while(blk) + { + blk_list.push_back(blk) ; + blk = blk->mNext ; + } + } + + std::sort(blk_list.begin(), blk_list.end(), LLMemoryBlock::CompareAddress()); + + U32 total_size = blk_list[0]->getBufferSize() ; + for(U32 i = 1 ; i < blk_list.size(); i++) + { + total_size += blk_list[i]->getBufferSize() ; + if((U32)blk_list[i]->getBuffer() < (U32)blk_list[i-1]->getBuffer() + blk_list[i-1]->getBufferSize()) + { + llerrs << "buffer corrupted." << llendl ; + } + } + + llassert_always(total_size + mMinBlockSize >= mBufferSize - ((U32)mDataBuffer - (U32)mBuffer)) ; + + U32 blk_num = (mBufferSize - (mDataBuffer - mBuffer)) / mMinBlockSize ; + for(U32 i = 0 ; i < blk_num ; ) + { + LLMemoryBlock* blk = &mBlocks[i] ; + if(blk->mSelf) + { + U32 end = blk->getBufferSize() / mMinBlockSize ; + for(U32 j = 0 ; j < end ; j++) + { + llassert_always(blk->mSelf == blk || !blk->mSelf) ; + } + i += end ; + } + else + { + llerrs << "gap happens" << llendl ; + } + } +#endif +#if 0 + llinfos << "---------------------------" << llendl ; + llinfos << "Chunk buffer: " << (U32)getBuffer() << " size: " << getBufferSize() << llendl ; + + llinfos << "available blocks ... " << llendl ; + for(S32 i = 0 ; i < mBlockLevels ; i++) + { + LLMemoryBlock* blk = mAvailBlockList[i] ; + while(blk) + { + llinfos << "blk buffer " << (U32)blk->getBuffer() << " size: " << blk->getBufferSize() << llendl ; + blk = blk->mNext ; + } + } + + llinfos << "free blocks ... " << llendl ; + for(S32 i = 0 ; i < mPartitionLevels ; i++) + { + LLMemoryBlock* blk = mFreeSpaceList[i] ; + while(blk) + { + llinfos << "blk buffer " << (U32)blk->getBuffer() << " size: " << blk->getBufferSize() << llendl ; + blk = blk->mNext ; + } + } +#endif +} + +//compute the size for a block, the size is round to integer times of mMinBlockSize. +U32 LLPrivateMemoryPool::LLMemoryChunk::calcBlockSize(U32 slot_size) +{ + // + //Note: we try to make a block to have 32 slots if the size is not over 32 pages + //32 is the number of bits of an integer in a 32-bit system + // + + U32 block_size; + U32 cut_off_size = llmin(CUT_OFF_SIZE, (U32)(mMinBlockSize << 5)) ; + + if((slot_size << 5) <= mMinBlockSize)//for small allocations, return one page + { + block_size = mMinBlockSize ; + } + else if(slot_size >= cut_off_size)//for large allocations, return one-slot block + { + block_size = (slot_size / mMinBlockSize) * mMinBlockSize ; + if(block_size < slot_size) + { + block_size += mMinBlockSize ; + } + } + else //medium allocations + { + if((slot_size << 5) >= cut_off_size) + { + block_size = cut_off_size ; + } + else + { + block_size = ((slot_size << 5) / mMinBlockSize) * mMinBlockSize ; + } + } + + llassert_always(block_size >= slot_size) ; + + return block_size ; +} + +//create a new block in the chunk +LLPrivateMemoryPool::LLMemoryBlock* LLPrivateMemoryPool::LLMemoryChunk::addBlock(U32 blk_idx) +{ + U32 slot_size = mMinSlotSize * (blk_idx + 1) ; + U32 preferred_block_size = calcBlockSize(slot_size) ; + U16 idx = getPageLevel(preferred_block_size); + LLMemoryBlock* blk = NULL ; + + if(mFreeSpaceList[idx])//if there is free slot for blk_idx + { + blk = createNewBlock(mFreeSpaceList[idx], preferred_block_size, slot_size, blk_idx) ; + } + else if(mFreeSpaceList[mPartitionLevels - 1]) //search free pool + { + blk = createNewBlock(mFreeSpaceList[mPartitionLevels - 1], preferred_block_size, slot_size, blk_idx) ; + } + else //search for other non-preferred but enough space slot. + { + S32 min_idx = 0 ; + if(slot_size > mMinBlockSize) + { + min_idx = getPageLevel(slot_size) ; + } + for(S32 i = (S32)idx - 1 ; i >= min_idx ; i--) //search the small slots first + { + if(mFreeSpaceList[i]) + { + U32 new_preferred_block_size = mFreeSpaceList[i]->getBufferSize(); + new_preferred_block_size = (new_preferred_block_size / mMinBlockSize) * mMinBlockSize ; //round to integer times of mMinBlockSize. + + //create a NEW BLOCK THERE. + if(new_preferred_block_size >= slot_size) //at least there is space for one slot. + { + + blk = createNewBlock(mFreeSpaceList[i], new_preferred_block_size, slot_size, blk_idx) ; + } + break ; + } + } + + if(!blk) + { + for(U16 i = idx + 1 ; i < mPartitionLevels - 1; i++) //search the large slots + { + if(mFreeSpaceList[i]) + { + //create a NEW BLOCK THERE. + blk = createNewBlock(mFreeSpaceList[i], preferred_block_size, slot_size, blk_idx) ; + break ; + } + } + } + } + + return blk ; +} + +//create a new block at the designed location +LLPrivateMemoryPool::LLMemoryBlock* LLPrivateMemoryPool::LLMemoryChunk::createNewBlock(LLMemoryBlock* blk, U32 buffer_size, U32 slot_size, U32 blk_idx) +{ + //unlink from the free space + removeFromFreeSpace(blk) ; + + //check the rest space + U32 new_free_blk_size = blk->getBufferSize() - buffer_size ; + if(new_free_blk_size < mMinBlockSize) //can not partition the memory into size smaller than mMinBlockSize + { + new_free_blk_size = 0 ; //discard the last small extra space. + } + + //add the rest space back to the free list + if(new_free_blk_size > 0) //blk still has free space + { + LLMemoryBlock* next_blk = blk + (buffer_size / mMinBlockSize) ; + next_blk->mPrev = NULL ; + next_blk->mNext = NULL ; + next_blk->setBuffer(blk->getBuffer() + buffer_size, new_free_blk_size) ; + addToFreeSpace(next_blk) ; + } + + blk->init(blk->getBuffer(), buffer_size, slot_size) ; + //insert to the available block list... + mAvailBlockList[blk_idx] = blk ; + + //mark the address map: all blocks covered by this block space pointing back to this block. + U32 end = (buffer_size / mMinBlockSize) ; + for(U32 i = 1 ; i < end ; i++) + { + (blk + i)->mSelf = blk ; + } + + return blk ; +} + +//delete a block, release the block to the free pool. +void LLPrivateMemoryPool::LLMemoryChunk::removeBlock(LLMemoryBlock* blk) +{ + //remove from the available block list + if(blk->mPrev) + { + blk->mPrev->mNext = blk->mNext ; + } + if(blk->mNext) + { + blk->mNext->mPrev = blk->mPrev ; + } + U32 blk_idx = getBlockLevel(blk->getSlotSize()); + if(mAvailBlockList[blk_idx] == blk) + { + mAvailBlockList[blk_idx] = blk->mNext ; + } + + blk->mNext = NULL ; + blk->mPrev = NULL ; + + //mark it free + blk->setBuffer(blk->getBuffer(), blk->getBufferSize()) ; + +#if 1 + //merge blk with neighbors if possible + if(blk->getBuffer() > mDataBuffer) //has the left neighbor + { + if((blk - 1)->mSelf->isFree()) + { + LLMemoryBlock* left_blk = (blk - 1)->mSelf ; + removeFromFreeSpace((blk - 1)->mSelf); + left_blk->setBuffer(left_blk->getBuffer(), left_blk->getBufferSize() + blk->getBufferSize()) ; + blk = left_blk ; + } + } + if(blk->getBuffer() + blk->getBufferSize() <= mBuffer + mBufferSize - mMinBlockSize) //has the right neighbor + { + U32 d = blk->getBufferSize() / mMinBlockSize ; + if((blk + d)->isFree()) + { + LLMemoryBlock* right_blk = blk + d ; + removeFromFreeSpace(blk + d) ; + blk->setBuffer(blk->getBuffer(), blk->getBufferSize() + right_blk->getBufferSize()) ; + } + } +#endif + + addToFreeSpace(blk) ; + + return ; +} + +//the top block in the list is full, pop it out of the list +void LLPrivateMemoryPool::LLMemoryChunk::popAvailBlockList(U32 blk_idx) +{ + if(mAvailBlockList[blk_idx]) + { + LLMemoryBlock* next = mAvailBlockList[blk_idx]->mNext ; + if(next) + { + next->mPrev = NULL ; + } + mAvailBlockList[blk_idx]->mPrev = NULL ; + mAvailBlockList[blk_idx]->mNext = NULL ; + mAvailBlockList[blk_idx] = next ; + } +} + +//add the block back to the free pool +void LLPrivateMemoryPool::LLMemoryChunk::addToFreeSpace(LLMemoryBlock* blk) +{ + llassert_always(!blk->mPrev) ; + llassert_always(!blk->mNext) ; + + U16 free_idx = blk->getBufferSize() / mMinBlockSize - 1; + + (blk + free_idx)->mSelf = blk ; //mark the end pointing back to the head. + free_idx = llmin(free_idx, (U16)(mPartitionLevels - 1)) ; + + blk->mNext = mFreeSpaceList[free_idx] ; + if(mFreeSpaceList[free_idx]) + { + mFreeSpaceList[free_idx]->mPrev = blk ; + } + mFreeSpaceList[free_idx] = blk ; + blk->mPrev = NULL ; + blk->mSelf = blk ; + + return ; +} + +//remove the space from the free pool +void LLPrivateMemoryPool::LLMemoryChunk::removeFromFreeSpace(LLMemoryBlock* blk) +{ + U16 free_idx = blk->getBufferSize() / mMinBlockSize - 1; + free_idx = llmin(free_idx, (U16)(mPartitionLevels - 1)) ; + + if(mFreeSpaceList[free_idx] == blk) + { + mFreeSpaceList[free_idx] = blk->mNext ; + } + if(blk->mPrev) + { + blk->mPrev->mNext = blk->mNext ; + } + if(blk->mNext) + { + blk->mNext->mPrev = blk->mPrev ; + } + blk->mNext = NULL ; + blk->mPrev = NULL ; + blk->mSelf = NULL ; + + return ; +} + +void LLPrivateMemoryPool::LLMemoryChunk::addToAvailBlockList(LLMemoryBlock* blk) +{ + llassert_always(!blk->mPrev) ; + llassert_always(!blk->mNext) ; + + U32 blk_idx = getBlockLevel(blk->getSlotSize()); + + blk->mNext = mAvailBlockList[blk_idx] ; + if(blk->mNext) + { + blk->mNext->mPrev = blk ; + } + blk->mPrev = NULL ; + mAvailBlockList[blk_idx] = blk ; + + return ; +} + +U32 LLPrivateMemoryPool::LLMemoryChunk::getPageIndex(U32 addr) +{ + return (addr - (U32)mDataBuffer) / mMinBlockSize ; +} + +//for mAvailBlockList +U32 LLPrivateMemoryPool::LLMemoryChunk::getBlockLevel(U32 size) +{ + llassert(size >= mMinSlotSize && size <= mMaxSlotSize) ; + + //start from 0 + return (size + mMinSlotSize - 1) / mMinSlotSize - 1 ; +} + +//for mFreeSpaceList +U16 LLPrivateMemoryPool::LLMemoryChunk::getPageLevel(U32 size) +{ + //start from 0 + U16 level = size / mMinBlockSize - 1 ; + if(level >= mPartitionLevels) + { + level = mPartitionLevels - 1 ; + } + return level ; +} + +//------------------------------------------------------------------- +//class LLPrivateMemoryPool +//-------------------------------------------------------------------- +const U32 CHUNK_SIZE = 4 << 20 ; //4 MB +const U32 LARGE_CHUNK_SIZE = 4 * CHUNK_SIZE ; //16 MB +LLPrivateMemoryPool::LLPrivateMemoryPool(S32 type, U32 max_pool_size) : + mMutexp(NULL), + mReservedPoolSize(0), + mHashFactor(1), + mType(type), + mMaxPoolSize(max_pool_size) +{ + if(type == STATIC_THREADED || type == VOLATILE_THREADED) + { + mMutexp = new LLMutex(NULL) ; + } + + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + mChunkList[i] = NULL ; + } + + mNumOfChunks = 0 ; +} + +LLPrivateMemoryPool::~LLPrivateMemoryPool() +{ + destroyPool(); + delete mMutexp ; +} + +char* LLPrivateMemoryPool::allocate(U32 size) +{ + if(!size) + { + return NULL ; + } + + //if the asked size larger than MAX_BLOCK_SIZE, fetch from heap directly, the pool does not manage it + if(size >= CHUNK_SIZE) + { + return (char*)malloc(size) ; + } + + char* p = NULL ; + + //find the appropriate chunk + S32 chunk_idx = getChunkIndex(size) ; + + lock() ; + + LLMemoryChunk* chunk = mChunkList[chunk_idx]; + while(chunk) + { + if((p = chunk->allocate(size))) + { + break ; + } + chunk = chunk->mNext ; + } + + //fetch new memory chunk + if(!p) + { + if(mReservedPoolSize + CHUNK_SIZE > mMaxPoolSize) + { + chunk = mChunkList[chunk_idx]; + while(chunk) + { + if((p = chunk->allocate(size))) + { + break ; + } + chunk = chunk->mNext ; + } + } + else + { + chunk = addChunk(chunk_idx) ; + if(chunk) + { + p = chunk->allocate(size) ; + } + } + } + + unlock() ; + + if(!p) //to get memory from the private pool failed, try the heap directly + { + static bool to_log = true ; + + if(to_log) + { + llwarns << "The memory pool overflows, now using heap directly!" << llendl ; + to_log = false ; + } + + return (char*)malloc(size) ; + } + + return p ; +} + +void LLPrivateMemoryPool::freeMem(void* addr) +{ + if(!addr) + { + return ; + } + + lock() ; + + LLMemoryChunk* chunk = findChunk((char*)addr) ; + + if(!chunk) + { + free(addr) ; //release from heap + } + else + { + chunk->freeMem(addr) ; + + if(chunk->empty()) + { + removeChunk(chunk) ; + } + } + + unlock() ; +} + +void LLPrivateMemoryPool::dump() +{ +} + +U32 LLPrivateMemoryPool::getTotalAllocatedSize() +{ + U32 total_allocated = 0 ; + + LLMemoryChunk* chunk ; + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + chunk = mChunkList[i]; + while(chunk) + { + total_allocated += chunk->getAllocatedSize() ; + chunk = chunk->mNext ; + } + } + + return total_allocated ; +} + +void LLPrivateMemoryPool::lock() +{ + if(mMutexp) + { + mMutexp->lock() ; + } +} + +void LLPrivateMemoryPool::unlock() +{ + if(mMutexp) + { + mMutexp->unlock() ; + } +} + +S32 LLPrivateMemoryPool::getChunkIndex(U32 size) +{ + S32 i ; + for(i = 0 ; size > MAX_SLOT_SIZES[i]; i++); + + llassert_always(i < SUPER_ALLOCATION); + + return i ; +} + +//destroy the entire pool +void LLPrivateMemoryPool::destroyPool() +{ + lock() ; + + if(mNumOfChunks > 0) + { + llwarns << "There is some memory not freed when destroy the memory pool!" << llendl ; + } + + mNumOfChunks = 0 ; + mChunkHashList.clear() ; + mHashFactor = 1 ; + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + mChunkList[i] = NULL ; + } + + unlock() ; +} + +bool LLPrivateMemoryPool::checkSize(U32 asked_size) +{ + if(mReservedPoolSize + asked_size > mMaxPoolSize) + { + llinfos << "Max pool size: " << mMaxPoolSize << llendl ; + llinfos << "Total reserved size: " << mReservedPoolSize + asked_size << llendl ; + llinfos << "Total_allocated Size: " << getTotalAllocatedSize() << llendl ; + + //llerrs << "The pool is overflowing..." << llendl ; + + return false ; + } + + return true ; +} + +LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::addChunk(S32 chunk_index) +{ + U32 preferred_size ; + U32 overhead ; + if(chunk_index < LARGE_ALLOCATION) + { + preferred_size = CHUNK_SIZE ; //4MB + overhead = LLMemoryChunk::getMaxOverhead(preferred_size, MIN_SLOT_SIZES[chunk_index], + MAX_SLOT_SIZES[chunk_index], MIN_BLOCK_SIZES[chunk_index], MAX_BLOCK_SIZES[chunk_index]) ; + } + else + { + preferred_size = LARGE_CHUNK_SIZE ; //16MB + overhead = LLMemoryChunk::getMaxOverhead(preferred_size, MIN_SLOT_SIZES[chunk_index], + MAX_SLOT_SIZES[chunk_index], MIN_BLOCK_SIZES[chunk_index], MAX_BLOCK_SIZES[chunk_index]) ; + } + + if(!checkSize(preferred_size + overhead)) + { + return NULL ; + } + + mReservedPoolSize += preferred_size + overhead ; + + char* buffer = (char*)malloc(preferred_size + overhead) ; + if(!buffer) + { + return NULL ; + } + + LLMemoryChunk* chunk = new (buffer) LLMemoryChunk() ; + chunk->init(buffer, preferred_size + overhead, MIN_SLOT_SIZES[chunk_index], + MAX_SLOT_SIZES[chunk_index], MIN_BLOCK_SIZES[chunk_index], MAX_BLOCK_SIZES[chunk_index]) ; + + //add to the tail of the linked list + { + if(!mChunkList[chunk_index]) + { + mChunkList[chunk_index] = chunk ; + } + else + { + LLMemoryChunk* cur = mChunkList[chunk_index] ; + while(cur->mNext) + { + cur = cur->mNext ; + } + cur->mNext = chunk ; + chunk->mPrev = cur ; + } + } + + //insert into the hash table + addToHashTable(chunk) ; + + mNumOfChunks++; + + return chunk ; +} + +void LLPrivateMemoryPool::removeChunk(LLMemoryChunk* chunk) +{ + if(!chunk) + { + return ; + } + + //remove from the linked list + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + if(mChunkList[i] == chunk) + { + mChunkList[i] = chunk->mNext ; + } + } + + if(chunk->mPrev) + { + chunk->mPrev->mNext = chunk->mNext ; + } + if(chunk->mNext) + { + chunk->mNext->mPrev = chunk->mPrev ; + } + + //remove from the hash table + removeFromHashTable(chunk) ; + + mNumOfChunks--; + mReservedPoolSize -= chunk->getBufferSize() ; + + //release memory + free(chunk->getBuffer()) ; +} + +U16 LLPrivateMemoryPool::findHashKey(const char* addr) +{ + return (((U32)addr) / CHUNK_SIZE) % mHashFactor ; +} + +LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::findChunk(const char* addr) +{ + U16 key = findHashKey(addr) ; + if(mChunkHashList.size() <= key) + { + return NULL ; + } + + return mChunkHashList[key].findChunk(addr) ; +} + +void LLPrivateMemoryPool::addToHashTable(LLMemoryChunk* chunk) +{ + static const U16 HASH_FACTORS[] = {41, 83, 193, 317, 419, 523, 719, 997, 1523, 0xFFFF}; + + U16 i ; + if(mChunkHashList.empty()) + { + mHashFactor = HASH_FACTORS[0] ; + rehash() ; + } + + U16 start_key = findHashKey(chunk->getBuffer()) ; + U16 end_key = findHashKey(chunk->getBuffer() + chunk->getBufferSize() - 1) ; + bool need_rehash = false ; + + if(mChunkHashList[start_key].hasElement(chunk)) + { + return; //already inserted. + } + need_rehash = mChunkHashList[start_key].add(chunk) ; + + if(start_key == end_key && !need_rehash) + { + return ; //done + } + + if(!need_rehash) + { + need_rehash = mChunkHashList[end_key].add(chunk) ; + } + + if(!need_rehash) + { + if(end_key < start_key) + { + need_rehash = fillHashTable(start_key + 1, mHashFactor, chunk) ; + if(!need_rehash) + { + need_rehash = fillHashTable(0, end_key, chunk) ; + } + } + else + { + need_rehash = fillHashTable(start_key + 1, end_key, chunk) ; + } + } + + if(need_rehash) + { + i = 0 ; + while(HASH_FACTORS[i] <= mHashFactor) i++; + + mHashFactor = HASH_FACTORS[i] ; + llassert_always(mHashFactor != 0xFFFF) ;//stop point to prevent endlessly recursive calls + + rehash() ; + } +} + +void LLPrivateMemoryPool::removeFromHashTable(LLMemoryChunk* chunk) +{ + U16 start_key = findHashKey(chunk->getBuffer()) ; + U16 end_key = findHashKey(chunk->getBuffer() + chunk->getBufferSize() - 1) ; + + mChunkHashList[start_key].remove(chunk) ; + if(start_key == end_key) + { + return ; //done + } + + mChunkHashList[end_key].remove(chunk) ; + + if(end_key < start_key) + { + for(U16 i = start_key + 1 ; i < mHashFactor; i++) + { + mChunkHashList[i].remove(chunk) ; + } + for(U16 i = 0 ; i < end_key; i++) + { + mChunkHashList[i].remove(chunk) ; + } + } + else + { + for(U16 i = start_key + 1 ; i < end_key; i++) + { + mChunkHashList[i].remove(chunk) ; + } + } +} + +void LLPrivateMemoryPool::rehash() +{ + llinfos << "new hash factor: " << mHashFactor << llendl ; + + mChunkHashList.clear() ; + mChunkHashList.resize(mHashFactor) ; + + LLMemoryChunk* chunk ; + for(U16 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + chunk = mChunkList[i] ; + while(chunk) + { + addToHashTable(chunk) ; + chunk = chunk->mNext ; + } + } +} + +bool LLPrivateMemoryPool::fillHashTable(U16 start, U16 end, LLMemoryChunk* chunk) +{ + for(U16 i = start; i < end; i++) + { + if(mChunkHashList[i].add(chunk)) + { + return true ; + } + } + + return false ; +} + +//-------------------------------------------------------------------- +// class LLChunkHashElement +//-------------------------------------------------------------------- +LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::LLChunkHashElement::findChunk(const char* addr) +{ + if(mFirst && mFirst->containsAddress(addr)) + { + return mFirst ; + } + else if(mSecond && mSecond->containsAddress(addr)) + { + return mSecond ; + } + + return NULL ; +} + +//return false if successfully inserted to the hash slot. +bool LLPrivateMemoryPool::LLChunkHashElement::add(LLPrivateMemoryPool::LLMemoryChunk* chunk) +{ + llassert_always(!hasElement(chunk)) ; + + if(!mFirst) + { + mFirst = chunk ; + } + else if(!mSecond) + { + mSecond = chunk ; + } + else + { + return true ; //failed + } + + return false ; +} + +void LLPrivateMemoryPool::LLChunkHashElement::remove(LLPrivateMemoryPool::LLMemoryChunk* chunk) +{ + if(mFirst == chunk) + { + mFirst = NULL ; + } + else if(mSecond ==chunk) + { + mSecond = NULL ; + } + else + { + llerrs << "This slot does not contain this chunk!" << llendl ; + } +} + +//-------------------------------------------------------------------- +//class LLPrivateMemoryPoolManager +//-------------------------------------------------------------------- +LLPrivateMemoryPoolManager* LLPrivateMemoryPoolManager::sInstance = NULL ; +BOOL LLPrivateMemoryPoolManager::sPrivatePoolEnabled = FALSE ; +std::vector<LLPrivateMemoryPool*> LLPrivateMemoryPoolManager::sDanglingPoolList ; + +LLPrivateMemoryPoolManager::LLPrivateMemoryPoolManager(BOOL enabled, U32 max_pool_size) +{ + mPoolList.resize(LLPrivateMemoryPool::MAX_TYPES) ; + + for(S32 i = 0 ; i < LLPrivateMemoryPool::MAX_TYPES; i++) + { + mPoolList[i] = NULL ; + } + + sPrivatePoolEnabled = enabled ; + + const U32 MAX_POOL_SIZE = 256 * 1024 * 1024 ; //256 MB + mMaxPrivatePoolSize = llmax(max_pool_size, MAX_POOL_SIZE) ; +} + +LLPrivateMemoryPoolManager::~LLPrivateMemoryPoolManager() +{ + +#if __DEBUG_PRIVATE_MEM__ + if(!sMemAllocationTracker.empty()) + { + llwarns << "there is potential memory leaking here. The list of not freed memory blocks are from: " <<llendl ; + + S32 k = 0 ; + for(mem_allocation_info_t::iterator iter = sMemAllocationTracker.begin() ; iter != sMemAllocationTracker.end() ; ++iter) + { + llinfos << k++ << ", " << (U32)iter->first << " : " << iter->second << llendl ; + } + sMemAllocationTracker.clear() ; + } +#endif + +#if 0 + //all private pools should be released by their owners before reaching here. + for(S32 i = 0 ; i < LLPrivateMemoryPool::MAX_TYPES; i++) + { + llassert_always(!mPoolList[i]) ; + } + mPoolList.clear() ; + +#else + //forcefully release all memory + for(S32 i = 0 ; i < LLPrivateMemoryPool::MAX_TYPES; i++) + { + if(mPoolList[i]) + { + if(mPoolList[i]->isEmpty()) + { + delete mPoolList[i] ; + } + else + { + //can not delete this pool because it has alloacted memory to be freed. + //move it to the dangling list. + sDanglingPoolList.push_back(mPoolList[i]) ; + } + + mPoolList[i] = NULL ; + } + } + mPoolList.clear() ; +#endif +} + +//static +void LLPrivateMemoryPoolManager::initClass(BOOL enabled, U32 max_pool_size) +{ + llassert_always(!sInstance) ; + + sInstance = new LLPrivateMemoryPoolManager(enabled, max_pool_size) ; +} + +//static +LLPrivateMemoryPoolManager* LLPrivateMemoryPoolManager::getInstance() +{ + //if(!sInstance) + //{ + // sInstance = new LLPrivateMemoryPoolManager(FALSE) ; + //} + return sInstance ; +} + +//static +void LLPrivateMemoryPoolManager::destroyClass() +{ + if(sInstance) + { + delete sInstance ; + sInstance = NULL ; + } +} + +LLPrivateMemoryPool* LLPrivateMemoryPoolManager::newPool(S32 type) +{ + if(!sPrivatePoolEnabled) + { + return NULL ; + } + + if(!mPoolList[type]) + { + mPoolList[type] = new LLPrivateMemoryPool(type, mMaxPrivatePoolSize) ; + } + + return mPoolList[type] ; +} + +void LLPrivateMemoryPoolManager::deletePool(LLPrivateMemoryPool* pool) +{ + if(pool && pool->isEmpty()) + { + mPoolList[pool->getType()] = NULL ; + delete pool; + } +} + +//debug +void LLPrivateMemoryPoolManager::updateStatistics() +{ + mTotalReservedSize = 0 ; + mTotalAllocatedSize = 0 ; + + for(U32 i = 0; i < mPoolList.size(); i++) + { + if(mPoolList[i]) + { + mTotalReservedSize += mPoolList[i]->getTotalReservedSize() ; + mTotalAllocatedSize += mPoolList[i]->getTotalAllocatedSize() ; + } + } +} + +#if __DEBUG_PRIVATE_MEM__ +//static +char* LLPrivateMemoryPoolManager::allocate(LLPrivateMemoryPool* poolp, U32 size, const char* function, const int line) +{ + char* p ; + + if(!poolp) + { + p = (char*)malloc(size) ; + } + else + { + p = poolp->allocate(size) ; + } + + if(p) + { + char num[16] ; + sprintf(num, " line: %d ", line) ; + std::string str(function) ; + str += num; + + sMemAllocationTracker[p] = str ; + } + + return p ; +} +#else +//static +char* LLPrivateMemoryPoolManager::allocate(LLPrivateMemoryPool* poolp, U32 size) +{ + if(poolp) + { + return poolp->allocate(size) ; + } + else + { + return (char*)malloc(size) ; + } +} +#endif + +//static +void LLPrivateMemoryPoolManager::freeMem(LLPrivateMemoryPool* poolp, void* addr) +{ + if(!addr) + { + return ; + } + +#if __DEBUG_PRIVATE_MEM__ + sMemAllocationTracker.erase((char*)addr) ; +#endif + + if(poolp) + { + poolp->freeMem(addr) ; + } + else + { + if(!sPrivatePoolEnabled) + { + free(addr) ; //private pool is disabled. + } + else if(!sInstance) //the private memory manager is destroyed, try the dangling list + { + for(S32 i = 0 ; i < sDanglingPoolList.size(); i++) + { + if(sDanglingPoolList[i]->findChunk((char*)addr)) + { + sDanglingPoolList[i]->freeMem(addr) ; + if(sDanglingPoolList[i]->isEmpty()) + { + delete sDanglingPoolList[i] ; + + if(i < sDanglingPoolList.size() - 1) + { + sDanglingPoolList[i] = sDanglingPoolList[sDanglingPoolList.size() - 1] ; + } + sDanglingPoolList.pop_back() ; + } + + addr = NULL ; + break ; + } + } + llassert_always(!addr) ; //addr should be release before hitting here! + } + else + { + llerrs << "private pool is used before initialized.!" << llendl ; + } + } +} + +//-------------------------------------------------------------------- +//class LLPrivateMemoryPoolTester +//-------------------------------------------------------------------- +#if 0 +LLPrivateMemoryPoolTester* LLPrivateMemoryPoolTester::sInstance = NULL ; +LLPrivateMemoryPool* LLPrivateMemoryPoolTester::sPool = NULL ; +LLPrivateMemoryPoolTester::LLPrivateMemoryPoolTester() +{ +} + +LLPrivateMemoryPoolTester::~LLPrivateMemoryPoolTester() +{ +} + +//static +LLPrivateMemoryPoolTester* LLPrivateMemoryPoolTester::getInstance() +{ + if(!sInstance) + { + sInstance = ::new LLPrivateMemoryPoolTester() ; + } + return sInstance ; +} + +//static +void LLPrivateMemoryPoolTester::destroy() +{ + if(sInstance) + { + ::delete sInstance ; + sInstance = NULL ; + } + + if(sPool) + { + LLPrivateMemoryPoolManager::getInstance()->deletePool(sPool) ; + sPool = NULL ; + } +} + +void LLPrivateMemoryPoolTester::run(S32 type) +{ + if(sPool) + { + LLPrivateMemoryPoolManager::getInstance()->deletePool(sPool) ; + } + sPool = LLPrivateMemoryPoolManager::getInstance()->newPool(type) ; + + //run the test + correctnessTest() ; + performanceTest() ; + //fragmentationtest() ; + + //release pool. + LLPrivateMemoryPoolManager::getInstance()->deletePool(sPool) ; + sPool = NULL ; +} + +void LLPrivateMemoryPoolTester::test(U32 min_size, U32 max_size, U32 stride, U32 times, + bool random_deletion, bool output_statistics) +{ + U32 levels = (max_size - min_size) / stride + 1 ; + char*** p ; + U32 i, j ; + U32 total_allocated_size = 0 ; + + //allocate space for p ; + if(!(p = ::new char**[times]) || !(*p = ::new char*[times * levels])) + { + llerrs << "memory initialization for p failed" << llendl ; + } + + //init + for(i = 0 ; i < times; i++) + { + p[i] = *p + i * levels ; + for(j = 0 ; j < levels; j++) + { + p[i][j] = NULL ; + } + } + + //allocation + U32 size ; + for(i = 0 ; i < times ; i++) + { + for(j = 0 ; j < levels; j++) + { + size = min_size + j * stride ; + p[i][j] = ALLOCATE_MEM(sPool, size) ; + + total_allocated_size+= size ; + + *(U32*)p[i][j] = i ; + *((U32*)p[i][j] + 1) = j ; + //p[i][j][size - 1] = '\0' ; //access the last element to verify the success of the allocation. + + //randomly release memory + if(random_deletion) + { + S32 k = rand() % levels ; + + if(p[i][k]) + { + llassert_always(*(U32*)p[i][k] == i && *((U32*)p[i][k] + 1) == k) ; + FREE_MEM(sPool, p[i][k]) ; + total_allocated_size -= min_size + k * stride ; + p[i][k] = NULL ; + } + } + } + } + + //output pool allocation statistics + if(output_statistics) + { + } + + //release all memory allocations + for(i = 0 ; i < times; i++) + { + for(j = 0 ; j < levels; j++) + { + if(p[i][j]) + { + llassert_always(*(U32*)p[i][j] == i && *((U32*)p[i][j] + 1) == j) ; + FREE_MEM(sPool, p[i][j]) ; + total_allocated_size -= min_size + j * stride ; + p[i][j] = NULL ; + } + } + } + + ::delete[] *p ; + ::delete[] p ; +} + +void LLPrivateMemoryPoolTester::testAndTime(U32 size, U32 times) +{ + LLTimer timer ; + + llinfos << " -**********************- " << llendl ; + llinfos << "test size: " << size << " test times: " << times << llendl ; + + timer.reset() ; + char** p = new char*[times] ; + + //using the customized memory pool + //allocation + for(U32 i = 0 ; i < times; i++) + { + p[i] = ALLOCATE_MEM(sPool, size) ; + if(!p[i]) + { + llerrs << "allocation failed" << llendl ; + } + } + //de-allocation + for(U32 i = 0 ; i < times; i++) + { + FREE_MEM(sPool, p[i]) ; + p[i] = NULL ; + } + llinfos << "time spent using customized memory pool: " << timer.getElapsedTimeF32() << llendl ; + + timer.reset() ; + + //using the standard allocator/de-allocator: + //allocation + for(U32 i = 0 ; i < times; i++) + { + p[i] = ::new char[size] ; + if(!p[i]) + { + llerrs << "allocation failed" << llendl ; + } + } + //de-allocation + for(U32 i = 0 ; i < times; i++) + { + ::delete[] p[i] ; + p[i] = NULL ; + } + llinfos << "time spent using standard allocator/de-allocator: " << timer.getElapsedTimeF32() << llendl ; + + delete[] p; +} + +void LLPrivateMemoryPoolTester::correctnessTest() +{ + //try many different sized allocation, and all kinds of edge cases, access the allocated memory + //to see if allocation is right. + + //edge case + char* p = ALLOCATE_MEM(sPool, 0) ; + FREE_MEM(sPool, p) ; + + //small sized + // [8 bytes, 2KB), each asks for 256 allocations and deallocations + test(8, 2040, 8, 256, true, true) ; + + //medium sized + //[2KB, 512KB), each asks for 16 allocations and deallocations + test(2048, 512 * 1024 - 2048, 2048, 16, true, true) ; + + //large sized + //[512KB, 4MB], each asks for 8 allocations and deallocations + test(512 * 1024, 4 * 1024 * 1024, 64 * 1024, 6, true, true) ; +} + +void LLPrivateMemoryPoolTester::performanceTest() +{ + U32 test_size[3] = {768, 3* 1024, 3* 1024 * 1024}; + + //small sized + testAndTime(test_size[0], 8) ; + + //medium sized + testAndTime(test_size[1], 8) ; + + //large sized + testAndTime(test_size[2], 8) ; +} + +void LLPrivateMemoryPoolTester::fragmentationtest() +{ + //for internal fragmentation statistics: + //every time when asking for a new chunk during correctness test, and performance test, + //print out the chunk usage statistices. +} #endif +//-------------------------------------------------------------------- diff --git a/indra/llcommon/llmemory.h b/indra/llcommon/llmemory.h index 9bf4248bb7..bbbdaa6497 100644 --- a/indra/llcommon/llmemory.h +++ b/indra/llcommon/llmemory.h @@ -1,39 +1,112 @@ -/** +/** * @file llmemory.h * @brief Memory allocation/deallocation header-stuff goes here. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ #ifndef LLMEMORY_H #define LLMEMORY_H +#include "llmemtype.h" +#if LL_DEBUG +inline void* ll_aligned_malloc( size_t size, int align ) +{ + void* mem = malloc( size + (align - 1) + sizeof(void*) ); + char* aligned = ((char*)mem) + sizeof(void*); + aligned += align - ((uintptr_t)aligned & (align - 1)); + + ((void**)aligned)[-1] = mem; + return aligned; +} + +inline void ll_aligned_free( void* ptr ) +{ + free( ((void**)ptr)[-1] ); +} + +inline void* ll_aligned_malloc_16(size_t size) // returned hunk MUST be freed with ll_aligned_free_16(). +{ +#if defined(LL_WINDOWS) + return _mm_malloc(size, 16); +#elif defined(LL_DARWIN) + return malloc(size); // default osx malloc is 16 byte aligned. +#else + void *rtn; + if (LL_LIKELY(0 == posix_memalign(&rtn, 16, size))) + return rtn; + else // bad alignment requested, or out of memory + return NULL; +#endif +} + +inline void ll_aligned_free_16(void *p) +{ +#if defined(LL_WINDOWS) + _mm_free(p); +#elif defined(LL_DARWIN) + return free(p); +#else + free(p); // posix_memalign() is compatible with heap deallocator +#endif +} +inline void* ll_aligned_malloc_32(size_t size) // returned hunk MUST be freed with ll_aligned_free_32(). +{ +#if defined(LL_WINDOWS) + return _mm_malloc(size, 32); +#elif defined(LL_DARWIN) + return ll_aligned_malloc( size, 32 ); +#else + void *rtn; + if (LL_LIKELY(0 == posix_memalign(&rtn, 32, size))) + return rtn; + else // bad alignment requested, or out of memory + return NULL; +#endif +} -extern S32 gTotalDAlloc; -extern S32 gTotalDAUse; -extern S32 gDACount; +inline void ll_aligned_free_32(void *p) +{ +#if defined(LL_WINDOWS) + _mm_free(p); +#elif defined(LL_DARWIN) + ll_aligned_free( p ); +#else + free(p); // posix_memalign() is compatible with heap deallocator +#endif +} +#else // LL_DEBUG +// ll_aligned_foo are noops now that we use tcmalloc everywhere (tcmalloc aligns automatically at appropriate intervals) +#define ll_aligned_malloc( size, align ) malloc(size) +#define ll_aligned_free( ptr ) free(ptr) +#define ll_aligned_malloc_16 malloc +#define ll_aligned_free_16 free +#define ll_aligned_malloc_32 malloc +#define ll_aligned_free_32 free +#endif // LL_DEBUG -extern void* ll_allocate (size_t size); -extern void ll_release (void *p); +#ifndef __DEBUG_PRIVATE_MEM__ +#define __DEBUG_PRIVATE_MEM__ 0 +#endif class LL_COMMON_API LLMemory { @@ -44,10 +117,393 @@ public: // Return the resident set size of the current process, in bytes. // Return value is zero if not known. static U64 getCurrentRSS(); + static U32 getWorkingSetSize(); + static void* tryToAlloc(void* address, U32 size); + static void initMaxHeapSizeGB(F32 max_heap_size_gb, BOOL prevent_heap_failure); + static void updateMemoryInfo() ; + static void logMemoryInfo(BOOL update = FALSE); + static bool isMemoryPoolLow(); + + static U32 getAvailableMemKB() ; + static U32 getMaxMemKB() ; + static U32 getAllocatedMemKB() ; private: static char* reserveMem; + static U32 sAvailPhysicalMemInKB ; + static U32 sMaxPhysicalMemInKB ; + static U32 sAllocatedMemInKB; + static U32 sAllocatedPageSizeInKB ; + + static U32 sMaxHeapSizeInKB; + static BOOL sEnableMemoryFailurePrevention; +}; + +//---------------------------------------------------------------------------- +#if MEM_TRACK_MEM +class LLMutex ; +class LL_COMMON_API LLMemTracker +{ +private: + LLMemTracker() ; + ~LLMemTracker() ; + +public: + static void release() ; + static LLMemTracker* getInstance() ; + + void track(const char* function, const int line) ; + void preDraw(BOOL pause) ; + void postDraw() ; + const char* getNextLine() ; + +private: + static LLMemTracker* sInstance ; + + char** mStringBuffer ; + S32 mCapacity ; + U32 mLastAllocatedMem ; + S32 mCurIndex ; + S32 mCounter; + S32 mDrawnIndex; + S32 mNumOfDrawn; + BOOL mPaused; + LLMutex* mMutexp ; +}; + +#define MEM_TRACK_RELEASE LLMemTracker::release() ; +#define MEM_TRACK LLMemTracker::getInstance()->track(__FUNCTION__, __LINE__) ; + +#else // MEM_TRACK_MEM + +#define MEM_TRACK_RELEASE +#define MEM_TRACK + +#endif // MEM_TRACK_MEM + +//---------------------------------------------------------------------------- + + +// +//class LLPrivateMemoryPool defines a private memory pool for an application to use, so the application does not +//need to access the heap directly fro each memory allocation. Throught this, the allocation speed is faster, +//and reduces virtaul address space gragmentation problem. +//Note: this class is thread-safe by passing true to the constructor function. However, you do not need to do this unless +//you are sure the memory allocation and de-allocation will happen in different threads. To make the pool thread safe +//increases allocation and deallocation cost. +// +class LL_COMMON_API LLPrivateMemoryPool +{ + friend class LLPrivateMemoryPoolManager ; + +public: + class LL_COMMON_API LLMemoryBlock //each block is devided into slots uniformly + { + public: + LLMemoryBlock() ; + ~LLMemoryBlock() ; + + void init(char* buffer, U32 buffer_size, U32 slot_size) ; + void setBuffer(char* buffer, U32 buffer_size) ; + + char* allocate() ; + void freeMem(void* addr) ; + + bool empty() {return !mAllocatedSlots;} + bool isFull() {return mAllocatedSlots == mTotalSlots;} + bool isFree() {return !mTotalSlots;} + + U32 getSlotSize()const {return mSlotSize;} + U32 getTotalSlots()const {return mTotalSlots;} + U32 getBufferSize()const {return mBufferSize;} + char* getBuffer() const {return mBuffer;} + + //debug use + void resetBitMap() ; + private: + char* mBuffer; + U32 mSlotSize ; //when the block is not initialized, it is the buffer size. + U32 mBufferSize ; + U32 mUsageBits ; + U8 mTotalSlots ; + U8 mAllocatedSlots ; + U8 mDummySize ; //size of extra bytes reserved for mUsageBits. + + public: + LLMemoryBlock* mPrev ; + LLMemoryBlock* mNext ; + LLMemoryBlock* mSelf ; + + struct CompareAddress + { + bool operator()(const LLMemoryBlock* const& lhs, const LLMemoryBlock* const& rhs) + { + return (U32)lhs->getBuffer() < (U32)rhs->getBuffer(); + } + }; + }; + + class LL_COMMON_API LLMemoryChunk //is divided into memory blocks. + { + public: + LLMemoryChunk() ; + ~LLMemoryChunk() ; + + void init(char* buffer, U32 buffer_size, U32 min_slot_size, U32 max_slot_size, U32 min_block_size, U32 max_block_size) ; + void setBuffer(char* buffer, U32 buffer_size) ; + + bool empty() ; + + char* allocate(U32 size) ; + void freeMem(void* addr) ; + + char* getBuffer() const {return mBuffer;} + U32 getBufferSize() const {return mBufferSize;} + U32 getAllocatedSize() const {return mAlloatedSize;} + + bool containsAddress(const char* addr) const; + + static U32 getMaxOverhead(U32 data_buffer_size, U32 min_slot_size, + U32 max_slot_size, U32 min_block_size, U32 max_block_size) ; + + void dump() ; + + private: + U32 getPageIndex(U32 addr) ; + U32 getBlockLevel(U32 size) ; + U16 getPageLevel(U32 size) ; + LLMemoryBlock* addBlock(U32 blk_idx) ; + void popAvailBlockList(U32 blk_idx) ; + void addToFreeSpace(LLMemoryBlock* blk) ; + void removeFromFreeSpace(LLMemoryBlock* blk) ; + void removeBlock(LLMemoryBlock* blk) ; + void addToAvailBlockList(LLMemoryBlock* blk) ; + U32 calcBlockSize(U32 slot_size); + LLMemoryBlock* createNewBlock(LLMemoryBlock* blk, U32 buffer_size, U32 slot_size, U32 blk_idx) ; + + private: + LLMemoryBlock** mAvailBlockList ;//256 by mMinSlotSize + LLMemoryBlock** mFreeSpaceList; + LLMemoryBlock* mBlocks ; //index of blocks by address. + + char* mBuffer ; + U32 mBufferSize ; + char* mDataBuffer ; + char* mMetaBuffer ; + U32 mMinBlockSize ; + U32 mMinSlotSize ; + U32 mMaxSlotSize ; + U32 mAlloatedSize ; + U16 mBlockLevels; + U16 mPartitionLevels; + + public: + //form a linked list + LLMemoryChunk* mNext ; + LLMemoryChunk* mPrev ; + } ; + +private: + LLPrivateMemoryPool(S32 type, U32 max_pool_size) ; + ~LLPrivateMemoryPool() ; + + char *allocate(U32 size) ; + void freeMem(void* addr) ; + + void dump() ; + U32 getTotalAllocatedSize() ; + U32 getTotalReservedSize() {return mReservedPoolSize;} + S32 getType() const {return mType; } + bool isEmpty() const {return !mNumOfChunks; } + +private: + void lock() ; + void unlock() ; + S32 getChunkIndex(U32 size) ; + LLMemoryChunk* addChunk(S32 chunk_index) ; + bool checkSize(U32 asked_size) ; + void removeChunk(LLMemoryChunk* chunk) ; + U16 findHashKey(const char* addr); + void addToHashTable(LLMemoryChunk* chunk) ; + void removeFromHashTable(LLMemoryChunk* chunk) ; + void rehash() ; + bool fillHashTable(U16 start, U16 end, LLMemoryChunk* chunk) ; + LLMemoryChunk* findChunk(const char* addr) ; + + void destroyPool() ; + +public: + enum + { + SMALL_ALLOCATION = 0, //from 8 bytes to 2KB(exclusive), page size 2KB, max chunk size is 4MB. + MEDIUM_ALLOCATION, //from 2KB to 512KB(exclusive), page size 32KB, max chunk size 4MB + LARGE_ALLOCATION, //from 512KB to 4MB(inclusive), page size 64KB, max chunk size 16MB + SUPER_ALLOCATION //allocation larger than 4MB. + }; + + enum + { + STATIC = 0 , //static pool(each alllocation stays for a long time) without threading support + VOLATILE, //Volatile pool(each allocation stays for a very short time) without threading support + STATIC_THREADED, //static pool with threading support + VOLATILE_THREADED, //volatile pool with threading support + MAX_TYPES + }; //pool types + +private: + LLMutex* mMutexp ; + U32 mMaxPoolSize; + U32 mReservedPoolSize ; + + LLMemoryChunk* mChunkList[SUPER_ALLOCATION] ; //all memory chunks reserved by this pool, sorted by address + U16 mNumOfChunks ; + U16 mHashFactor ; + + S32 mType ; + + class LLChunkHashElement + { + public: + LLChunkHashElement() {mFirst = NULL ; mSecond = NULL ;} + + bool add(LLMemoryChunk* chunk) ; + void remove(LLMemoryChunk* chunk) ; + LLMemoryChunk* findChunk(const char* addr) ; + + bool empty() {return !mFirst && !mSecond; } + bool full() {return mFirst && mSecond; } + bool hasElement(LLMemoryChunk* chunk) {return mFirst == chunk || mSecond == chunk;} + + private: + LLMemoryChunk* mFirst ; + LLMemoryChunk* mSecond ; + }; + std::vector<LLChunkHashElement> mChunkHashList ; +}; + +class LL_COMMON_API LLPrivateMemoryPoolManager +{ +private: + LLPrivateMemoryPoolManager(BOOL enabled, U32 max_pool_size) ; + ~LLPrivateMemoryPoolManager() ; + +public: + static LLPrivateMemoryPoolManager* getInstance() ; + static void initClass(BOOL enabled, U32 pool_size) ; + static void destroyClass() ; + + LLPrivateMemoryPool* newPool(S32 type) ; + void deletePool(LLPrivateMemoryPool* pool) ; + +private: + std::vector<LLPrivateMemoryPool*> mPoolList ; + U32 mMaxPrivatePoolSize; + + static LLPrivateMemoryPoolManager* sInstance ; + static BOOL sPrivatePoolEnabled; + static std::vector<LLPrivateMemoryPool*> sDanglingPoolList ; +public: + //debug and statistics info. + void updateStatistics() ; + + U32 mTotalReservedSize ; + U32 mTotalAllocatedSize ; + +public: +#if __DEBUG_PRIVATE_MEM__ + static char* allocate(LLPrivateMemoryPool* poolp, U32 size, const char* function, const int line) ; + + typedef std::map<char*, std::string> mem_allocation_info_t ; + static mem_allocation_info_t sMemAllocationTracker; +#else + static char* allocate(LLPrivateMemoryPool* poolp, U32 size) ; +#endif + static void freeMem(LLPrivateMemoryPool* poolp, void* addr) ; +}; + +//------------------------------------------------------------------------------------- +#if __DEBUG_PRIVATE_MEM__ +#define ALLOCATE_MEM(poolp, size) LLPrivateMemoryPoolManager::allocate((poolp), (size), __FUNCTION__, __LINE__) +#else +#define ALLOCATE_MEM(poolp, size) LLPrivateMemoryPoolManager::allocate((poolp), (size)) +#endif +#define FREE_MEM(poolp, addr) LLPrivateMemoryPoolManager::freeMem((poolp), (addr)) +//------------------------------------------------------------------------------------- + +// +//the below singleton is used to test the private memory pool. +// +#if 0 +class LL_COMMON_API LLPrivateMemoryPoolTester +{ +private: + LLPrivateMemoryPoolTester() ; + ~LLPrivateMemoryPoolTester() ; + +public: + static LLPrivateMemoryPoolTester* getInstance() ; + static void destroy() ; + + void run(S32 type) ; + +private: + void correctnessTest() ; + void performanceTest() ; + void fragmentationtest() ; + + void test(U32 min_size, U32 max_size, U32 stride, U32 times, bool random_deletion, bool output_statistics) ; + void testAndTime(U32 size, U32 times) ; + +#if 0 +public: + void* operator new(size_t size) + { + return (void*)sPool->allocate(size) ; + } + void operator delete(void* addr) + { + sPool->freeMem(addr) ; + } + void* operator new[](size_t size) + { + return (void*)sPool->allocate(size) ; + } + void operator delete[](void* addr) + { + sPool->freeMem(addr) ; + } +#endif + +private: + static LLPrivateMemoryPoolTester* sInstance; + static LLPrivateMemoryPool* sPool ; + static LLPrivateMemoryPool* sThreadedPool ; }; +#if 0 +//static +void* LLPrivateMemoryPoolTester::operator new(size_t size) +{ + return (void*)sPool->allocate(size) ; +} +//static +void LLPrivateMemoryPoolTester::operator delete(void* addr) +{ + sPool->free(addr) ; +} + +//static +void* LLPrivateMemoryPoolTester::operator new[](size_t size) +{ + return (void*)sPool->allocate(size) ; +} + +//static +void LLPrivateMemoryPoolTester::operator delete[](void* addr) +{ + sPool->free(addr) ; +} +#endif +#endif // LLRefCount moved to llrefcount.h // LLPointer moved to llpointer.h diff --git a/indra/llcommon/llmemtype.cpp b/indra/llcommon/llmemtype.cpp index fe83f87d4b..6290a7158f 100644 --- a/indra/llcommon/llmemtype.cpp +++ b/indra/llcommon/llmemtype.cpp @@ -229,3 +229,4 @@ char const * LLMemType::getNameFromID(S32 id) return DeclareMemType::mNameList[id]; } +//-------------------------------------------------------------------------------------------------- diff --git a/indra/llcommon/llmetricperformancetester.cpp b/indra/llcommon/llmetricperformancetester.cpp index 5fa3a5ea07..41d3eb0bf3 100644 --- a/indra/llcommon/llmetricperformancetester.cpp +++ b/indra/llcommon/llmetricperformancetester.cpp @@ -63,7 +63,18 @@ BOOL LLMetricPerformanceTesterBasic::addTester(LLMetricPerformanceTesterBasic* t sTesterMap.insert(std::make_pair(name, tester)); return TRUE; } - + +/*static*/ +void LLMetricPerformanceTesterBasic::deleteTester(std::string name) +{ + name_tester_map_t::iterator tester = sTesterMap.find(name); + if (tester != sTesterMap.end()) + { + delete tester->second; + sTesterMap.erase(tester); + } +} + /*static*/ LLMetricPerformanceTesterBasic* LLMetricPerformanceTesterBasic::getTester(std::string name) { @@ -83,7 +94,78 @@ BOOL LLMetricPerformanceTesterBasic::isMetricLogRequested(std::string name) return (LLFastTimer::sMetricLog && ((LLFastTimer::sLogName == name) || (LLFastTimer::sLogName == DEFAULT_METRIC_NAME))); } +/*static*/ +LLSD LLMetricPerformanceTesterBasic::analyzeMetricPerformanceLog(std::istream& is) +{ + LLSD ret; + LLSD cur; + + while (!is.eof() && LLSDSerialize::fromXML(cur, is)) + { + for (LLSD::map_iterator iter = cur.beginMap(); iter != cur.endMap(); ++iter) + { + std::string label = iter->first; + + LLMetricPerformanceTesterBasic* tester = LLMetricPerformanceTesterBasic::getTester(iter->second["Name"].asString()) ; + if(tester) + { + ret[label]["Name"] = iter->second["Name"] ; + + S32 num_of_metrics = tester->getNumberOfMetrics() ; + for(S32 index = 0 ; index < num_of_metrics ; index++) + { + ret[label][ tester->getMetricName(index) ] = iter->second[ tester->getMetricName(index) ] ; + } + } + } + } + + return ret; +} + +/*static*/ +void LLMetricPerformanceTesterBasic::doAnalysisMetrics(std::string baseline, std::string target, std::string output) +{ + if(!LLMetricPerformanceTesterBasic::hasMetricPerformanceTesters()) + { + return ; + } + + // Open baseline and current target, exit if one is inexistent + std::ifstream base_is(baseline.c_str()); + std::ifstream target_is(target.c_str()); + if (!base_is.is_open() || !target_is.is_open()) + { + llwarns << "'-analyzeperformance' error : baseline or current target file inexistent" << llendl; + base_is.close(); + target_is.close(); + return; + } + //analyze baseline + LLSD base = analyzeMetricPerformanceLog(base_is); + base_is.close(); + + //analyze current + LLSD current = analyzeMetricPerformanceLog(target_is); + target_is.close(); + + //output comparision + std::ofstream os(output.c_str()); + + os << "Label, Metric, Base(B), Target(T), Diff(T-B), Percentage(100*T/B)\n"; + for(LLMetricPerformanceTesterBasic::name_tester_map_t::iterator iter = LLMetricPerformanceTesterBasic::sTesterMap.begin() ; + iter != LLMetricPerformanceTesterBasic::sTesterMap.end() ; ++iter) + { + LLMetricPerformanceTesterBasic* tester = ((LLMetricPerformanceTesterBasic*)iter->second) ; + tester->analyzePerformance(&os, &base, ¤t) ; + } + + os.flush(); + os.close(); +} + + //---------------------------------------------------------------------------------------------- // LLMetricPerformanceTesterBasic : Tester instance methods //---------------------------------------------------------------------------------------------- diff --git a/indra/llcommon/llmetricperformancetester.h b/indra/llcommon/llmetricperformancetester.h index 925010ac96..1a18cdf36f 100644 --- a/indra/llcommon/llmetricperformancetester.h +++ b/indra/llcommon/llmetricperformancetester.h @@ -27,7 +27,7 @@ #ifndef LL_METRICPERFORMANCETESTER_H #define LL_METRICPERFORMANCETESTER_H -const std::string DEFAULT_METRIC_NAME("metric"); +char const* const DEFAULT_METRIC_NAME = "metric"; /** * @class LLMetricPerformanceTesterBasic @@ -62,6 +62,8 @@ public: */ virtual void analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) ; + static void doAnalysisMetrics(std::string baseline, std::string target, std::string output) ; + /** * @return Returns the number of the test metrics in this tester instance. */ @@ -116,6 +118,7 @@ protected: private: void preOutputTestResults(LLSD* sd) ; void postOutputTestResults(LLSD* sd) ; + static LLSD analyzeMetricPerformanceLog(std::istream& is) ; std::string mName ; // Name of this tester instance S32 mCount ; // Current record count @@ -135,6 +138,12 @@ public: static LLMetricPerformanceTesterBasic* getTester(std::string name) ; /** + * @return Delete the named tester from the list + * @param[in] name - Name of the tester instance to delete. + */ + static void deleteTester(std::string name); + + /** * @return Returns TRUE if that metric *or* the default catch all metric has been requested to be logged * @param[in] name - Name of the tester queried. */ diff --git a/indra/llcommon/llpreprocessor.h b/indra/llcommon/llpreprocessor.h index 17a4287538..7fdb537ab5 100644 --- a/indra/llcommon/llpreprocessor.h +++ b/indra/llcommon/llpreprocessor.h @@ -132,7 +132,7 @@ #pragma warning( 3 : 4265 ) // "class has virtual functions, but destructor is not virtual" #pragma warning( 3 : 4266 ) // 'function' : no override available for virtual member function from base 'type'; function is hidden #pragma warning (disable : 4180) // qualifier applied to function type has no meaning; ignored -#pragma warning( disable : 4284 ) // silly MS warning deep inside their <map> include file +//#pragma warning( disable : 4284 ) // silly MS warning deep inside their <map> include file #pragma warning( disable : 4503 ) // 'decorated name length exceeded, name was truncated'. Does not seem to affect compilation. #pragma warning( disable : 4800 ) // 'BOOL' : forcing value to bool 'true' or 'false' (performance warning) #pragma warning( disable : 4996 ) // warning: deprecated @@ -151,6 +151,8 @@ #pragma warning (disable : 4251) // member needs to have dll-interface to be used by clients of class #pragma warning (disable : 4275) // non dll-interface class used as base for dll-interface class +#pragma warning (disable : 4018) // '<' : signed/unsigned mismatch + #endif // LL_MSVC #if LL_WINDOWS diff --git a/indra/llcommon/llprocesslauncher.cpp b/indra/llcommon/llprocesslauncher.cpp index 81e5f8820d..10950181fd 100644 --- a/indra/llcommon/llprocesslauncher.cpp +++ b/indra/llcommon/llprocesslauncher.cpp @@ -103,10 +103,30 @@ int LLProcessLauncher::launch(void) char *args2 = new char[args.size() + 1]; strcpy(args2, args.c_str()); - if( ! CreateProcessA( NULL, args2, NULL, NULL, FALSE, 0, NULL, NULL, &sinfo, &pinfo ) ) + const char * working_directory = 0; + if(!mWorkingDir.empty()) working_directory = mWorkingDir.c_str(); + if( ! CreateProcessA( NULL, args2, NULL, NULL, FALSE, 0, NULL, working_directory, &sinfo, &pinfo ) ) { - // TODO: do better than returning the OS-specific error code on failure... result = GetLastError(); + + LPTSTR error_str = 0; + if( + FormatMessage( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + result, + 0, + (LPTSTR)&error_str, + 0, + NULL) + != 0) + { + char message[256]; + wcstombs(message, error_str, 256); + message[255] = 0; + llwarns << "CreateProcessA failed: " << message << llendl; + LocalFree(error_str); + } + if(result == 0) { // Make absolutely certain we return a non-zero value on failure. @@ -265,14 +285,7 @@ int LLProcessLauncher::launch(void) delete[] fake_argv; mProcessID = id; - - // At this point, the child process will have been created (since that's how vfork works -- the child borrowed our execution context until it forked) - // If the process doesn't exist at this point, the exec failed. - if(!isRunning()) - { - result = -1; - } - + return result; } diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index efd9c4b68f..1738c16dea 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -32,7 +32,7 @@ //============================================================================ // MAIN THREAD -LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded) : +LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded, bool should_pause) : LLThread(name), mThreaded(threaded), mIdleThread(TRUE), @@ -41,6 +41,11 @@ LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded) : { if (mThreaded) { + if(should_pause) + { + pause() ; //call this before start the thread. + } + start(); } } @@ -104,7 +109,7 @@ void LLQueuedThread::shutdown() // MAIN THREAD // virtual -S32 LLQueuedThread::update(U32 max_time_ms) +S32 LLQueuedThread::update(F32 max_time_ms) { if (!mStarted) { @@ -117,7 +122,7 @@ S32 LLQueuedThread::update(U32 max_time_ms) return updateQueue(max_time_ms); } -S32 LLQueuedThread::updateQueue(U32 max_time_ms) +S32 LLQueuedThread::updateQueue(F32 max_time_ms) { F64 max_time = (F64)max_time_ms * .001; LLTimer timer; diff --git a/indra/llcommon/llqueuedthread.h b/indra/llcommon/llqueuedthread.h index c75e0e2bbf..d3704b0fe2 100644 --- a/indra/llcommon/llqueuedthread.h +++ b/indra/llcommon/llqueuedthread.h @@ -149,7 +149,7 @@ public: static handle_t nullHandle() { return handle_t(0); } public: - LLQueuedThread(const std::string& name, bool threaded = true); + LLQueuedThread(const std::string& name, bool threaded = true, bool should_pause = false); virtual ~LLQueuedThread(); virtual void shutdown(); @@ -173,13 +173,13 @@ protected: public: bool waitForResult(handle_t handle, bool auto_complete = true); - virtual S32 update(U32 max_time_ms); - S32 updateQueue(U32 max_time_ms); + virtual S32 update(F32 max_time_ms); + S32 updateQueue(F32 max_time_ms); void waitOnPending(); void printQueueStats(); - S32 getPending(); + virtual S32 getPending(); bool getThreaded() { return mThreaded ? true : false; } // Request accessors diff --git a/indra/llcommon/llrefcount.cpp b/indra/llcommon/llrefcount.cpp index 55d0c85cbd..e1876599fc 100644 --- a/indra/llcommon/llrefcount.cpp +++ b/indra/llcommon/llrefcount.cpp @@ -29,9 +29,25 @@ #include "llerror.h" +#if LL_REF_COUNT_DEBUG +#include "llthread.h" +#include "llapr.h" +#endif + LLRefCount::LLRefCount(const LLRefCount& other) : mRef(0) { +#if LL_REF_COUNT_DEBUG + if(gAPRPoolp) + { + mMutexp = new LLMutex(gAPRPoolp) ; + } + else + { + mMutexp = NULL ; + } + mCrashAtUnlock = FALSE ; +#endif } LLRefCount& LLRefCount::operator=(const LLRefCount&) @@ -43,6 +59,17 @@ LLRefCount& LLRefCount::operator=(const LLRefCount&) LLRefCount::LLRefCount() : mRef(0) { +#if LL_REF_COUNT_DEBUG + if(gAPRPoolp) + { + mMutexp = new LLMutex(gAPRPoolp) ; + } + else + { + mMutexp = NULL ; + } + mCrashAtUnlock = FALSE ; +#endif } LLRefCount::~LLRefCount() @@ -51,4 +78,87 @@ LLRefCount::~LLRefCount() { llerrs << "deleting non-zero reference" << llendl; } + +#if LL_REF_COUNT_DEBUG + if(gAPRPoolp) + { + delete mMutexp ; + } +#endif } + +#if LL_REF_COUNT_DEBUG +void LLRefCount::ref() const +{ + if(mMutexp) + { + if(mMutexp->isLocked()) + { + mCrashAtUnlock = TRUE ; + llerrs << "the mutex is locked by the thread: " << mLockedThreadID + << " Current thread: " << LLThread::currentID() << llendl ; + } + + mMutexp->lock() ; + mLockedThreadID = LLThread::currentID() ; + + mRef++; + + if(mCrashAtUnlock) + { + while(1); //crash here. + } + mMutexp->unlock() ; + } + else + { + mRef++; + } +} + +S32 LLRefCount::unref() const +{ + if(mMutexp) + { + if(mMutexp->isLocked()) + { + mCrashAtUnlock = TRUE ; + llerrs << "the mutex is locked by the thread: " << mLockedThreadID + << " Current thread: " << LLThread::currentID() << llendl ; + } + + mMutexp->lock() ; + mLockedThreadID = LLThread::currentID() ; + + llassert(mRef >= 1); + if (0 == --mRef) + { + if(mCrashAtUnlock) + { + while(1); //crash here. + } + mMutexp->unlock() ; + + delete this; + return 0; + } + + if(mCrashAtUnlock) + { + while(1); //crash here. + } + mMutexp->unlock() ; + return mRef; + } + else + { + llassert(mRef >= 1); + if (0 == --mRef) + { + delete this; + return 0; + } + return mRef; + } +} +#endif diff --git a/indra/llcommon/llrefcount.h b/indra/llcommon/llrefcount.h index 19f008b15c..8eb5d53f3f 100644 --- a/indra/llcommon/llrefcount.h +++ b/indra/llcommon/llrefcount.h @@ -28,6 +28,11 @@ #include <boost/noncopyable.hpp> +#define LL_REF_COUNT_DEBUG 0 +#if LL_REF_COUNT_DEBUG +class LLMutex ; +#endif + //---------------------------------------------------------------------------- // RefCount objects should generally only be accessed by way of LLPointer<>'s // see llthread.h for LLThreadSafeRefCount @@ -43,12 +48,16 @@ protected: public: LLRefCount(); - void ref() const +#if LL_REF_COUNT_DEBUG + void ref() const ; + S32 unref() const ; +#else + inline void ref() const { mRef++; } - S32 unref() const + inline S32 unref() const { llassert(mRef >= 1); if (0 == --mRef) @@ -58,6 +67,7 @@ public: } return mRef; } +#endif //NOTE: when passing around a const LLRefCount object, this can return different results // at different types, since mRef is mutable @@ -68,6 +78,12 @@ public: private: mutable S32 mRef; + +#if LL_REF_COUNT_DEBUG + LLMutex* mMutexp ; + mutable U32 mLockedThreadID ; + mutable BOOL mCrashAtUnlock ; +#endif }; #endif diff --git a/indra/llcommon/llsd.cpp b/indra/llcommon/llsd.cpp index 6ca0737445..e295e3c621 100644 --- a/indra/llcommon/llsd.cpp +++ b/indra/llcommon/llsd.cpp @@ -24,6 +24,9 @@ * $/LicenseInfo$ */ +// Must turn on conditional declarations in header file so definitions end up +// with proper linkage. +#define LLSD_DEBUG_INFO #include "linden_common.h" #include "llsd.h" @@ -31,6 +34,7 @@ #include "../llmath/llmath.h" #include "llformat.h" #include "llsdserialize.h" +#include "stringize.h" #ifndef LL_RELEASE_FOR_DOWNLOAD #define NAME_UNNAMED_NAMESPACE @@ -50,6 +54,18 @@ namespace using namespace LLSDUnnamedNamespace; #endif +namespace llsd +{ + +// statics +S32 sLLSDAllocationCount = 0; +S32 sLLSDNetObjects = 0; + +} // namespace llsd + +#define ALLOC_LLSD_OBJECT { llsd::sLLSDNetObjects++; llsd::sLLSDAllocationCount++; } +#define FREE_LLSD_OBJECT { llsd::sLLSDNetObjects--; } + class LLSD::Impl /**< This class is the abstract base class of the implementation of LLSD It provides the reference counting implementation, and the default @@ -58,13 +74,10 @@ class LLSD::Impl */ { -private: - U32 mUseCount; - protected: Impl(); - enum StaticAllocationMarker { STATIC }; + enum StaticAllocationMarker { STATIC_USAGE_COUNT = 0xFFFFFFFF }; Impl(StaticAllocationMarker); ///< This constructor is used for static objects and causes the // suppresses adjusting the debugging counters when they are @@ -72,8 +85,10 @@ protected: virtual ~Impl(); - bool shared() const { return mUseCount > 1; } + bool shared() const { return (mUseCount > 1) && (mUseCount != STATIC_USAGE_COUNT); } + U32 mUseCount; + public: static void reset(Impl*& var, Impl* impl); ///< safely set var to refer to the new impl (possibly shared) @@ -128,6 +143,18 @@ public: virtual LLSD::array_const_iterator beginArray() const { return endArray(); } virtual LLSD::array_const_iterator endArray() const { static const std::vector<LLSD> empty; return empty.end(); } + virtual void dumpStats() const; + virtual void calcStats(S32 type_counts[], S32 share_counts[]) const; + // Container subclasses contain LLSD objects, rather than directly + // containing Impl objects. This helper forwards through LLSD. + void calcStats(const LLSD& llsd, S32 type_counts[], S32 share_counts[]) const + { + safe(llsd.impl).calcStats(type_counts, share_counts); + } + + static const Impl& getImpl(const LLSD& llsd) { return safe(llsd.impl); } + static Impl& getImpl(LLSD& llsd) { return safe(llsd.impl); } + static const LLSD& undef(); static U32 sAllocationCount; @@ -360,6 +387,9 @@ namespace LLSD::map_iterator endMap() { return mData.end(); } virtual LLSD::map_const_iterator beginMap() const { return mData.begin(); } virtual LLSD::map_const_iterator endMap() const { return mData.end(); } + + virtual void dumpStats() const; + virtual void calcStats(S32 type_counts[], S32 share_counts[]) const; }; ImplMap& ImplMap::makeMap(LLSD::Impl*& var) @@ -414,6 +444,34 @@ namespace return i->second; } + void ImplMap::dumpStats() const + { + std::cout << "Map size: " << mData.size() << std::endl; + + std::cout << "LLSD Net Objects: " << llsd::sLLSDNetObjects << std::endl; + std::cout << "LLSD allocations: " << llsd::sLLSDAllocationCount << std::endl; + + std::cout << "LLSD::Impl Net Objects: " << sOutstandingCount << std::endl; + std::cout << "LLSD::Impl allocations: " << sAllocationCount << std::endl; + + Impl::dumpStats(); + } + + void ImplMap::calcStats(S32 type_counts[], S32 share_counts[]) const + { + LLSD::map_const_iterator iter = beginMap(); + while (iter != endMap()) + { + //std::cout << " " << (*iter).first << ": " << (*iter).second << std::endl; + Impl::calcStats((*iter).second, type_counts, share_counts); + iter++; + } + + // Add in the values for this map + Impl::calcStats(type_counts, share_counts); + } + + class ImplArray : public LLSD::Impl { private: @@ -449,6 +507,8 @@ namespace LLSD::array_iterator endArray() { return mData.end(); } virtual LLSD::array_const_iterator beginArray() const { return mData.begin(); } virtual LLSD::array_const_iterator endArray() const { return mData.end(); } + + virtual void calcStats(S32 type_counts[], S32 share_counts[]) const; }; ImplArray& ImplArray::makeArray(Impl*& var) @@ -490,12 +550,13 @@ namespace void ImplArray::insert(LLSD::Integer i, const LLSD& v) { - if (i < 0) { + if (i < 0) + { return; } DataVector::size_type index = i; - if (index >= mData.size()) + if (index >= mData.size()) // tbd - sanity check limit for index ? { mData.resize(index + 1); } @@ -543,6 +604,19 @@ namespace return mData[index]; } + + void ImplArray::calcStats(S32 type_counts[], S32 share_counts[]) const + { + LLSD::array_const_iterator iter = beginArray(); + while (iter != endArray()) + { // Add values for all items held in the array + Impl::calcStats((*iter), type_counts, share_counts); + iter++; + } + + // Add in the values for this array + Impl::calcStats(type_counts, share_counts); + } } LLSD::Impl::Impl() @@ -564,8 +638,11 @@ LLSD::Impl::~Impl() void LLSD::Impl::reset(Impl*& var, Impl* impl) { - if (impl) ++impl->mUseCount; - if (var && --var->mUseCount == 0) + if (impl && impl->mUseCount != STATIC_USAGE_COUNT) + { + ++impl->mUseCount; + } + if (var && var->mUseCount != STATIC_USAGE_COUNT && --var->mUseCount == 0) { delete var; } @@ -574,13 +651,13 @@ void LLSD::Impl::reset(Impl*& var, Impl* impl) LLSD::Impl& LLSD::Impl::safe(Impl* impl) { - static Impl theUndefined(STATIC); + static Impl theUndefined(STATIC_USAGE_COUNT); return impl ? *impl : theUndefined; } const LLSD::Impl& LLSD::Impl::safe(const Impl* impl) { - static Impl theUndefined(STATIC); + static Impl theUndefined(STATIC_USAGE_COUNT); return impl ? *impl : theUndefined; } @@ -656,6 +733,43 @@ const LLSD& LLSD::Impl::undef() return immutableUndefined; } +void LLSD::Impl::dumpStats() const +{ + S32 type_counts[LLSD::TypeLLSDNumTypes + 1]; + memset(&type_counts, 0, sizeof(type_counts)); + + S32 share_counts[LLSD::TypeLLSDNumTypes + 1]; + memset(&share_counts, 0, sizeof(share_counts)); + + // Add info from all the values this object has + calcStats(type_counts, share_counts); + + S32 type_index = LLSD::TypeLLSDTypeBegin; + while (type_index != LLSD::TypeLLSDTypeEnd) + { + std::cout << LLSD::typeString((LLSD::Type)type_index) << " type " + << type_counts[type_index] << " objects, " + << share_counts[type_index] << " shared" + << std::endl; + type_index++; + } +} + + +void LLSD::Impl::calcStats(S32 type_counts[], S32 share_counts[]) const +{ + S32 tp = S32(type()); + if (0 <= tp && tp < LLSD::TypeLLSDNumTypes) + { + type_counts[tp]++; + if (shared()) + { + share_counts[tp]++; + } + } +} + + U32 LLSD::Impl::sAllocationCount = 0; U32 LLSD::Impl::sOutstandingCount = 0; @@ -681,10 +795,10 @@ namespace } -LLSD::LLSD() : impl(0) { } -LLSD::~LLSD() { Impl::reset(impl, 0); } +LLSD::LLSD() : impl(0) { ALLOC_LLSD_OBJECT; } +LLSD::~LLSD() { FREE_LLSD_OBJECT; Impl::reset(impl, 0); } -LLSD::LLSD(const LLSD& other) : impl(0) { assign(other); } +LLSD::LLSD(const LLSD& other) : impl(0) { ALLOC_LLSD_OBJECT; assign(other); } void LLSD::assign(const LLSD& other) { Impl::assign(impl, other.impl); } @@ -692,18 +806,18 @@ void LLSD::clear() { Impl::assignUndefined(impl); } LLSD::Type LLSD::type() const { return safe(impl).type(); } -// Scaler Constructors -LLSD::LLSD(Boolean v) : impl(0) { assign(v); } -LLSD::LLSD(Integer v) : impl(0) { assign(v); } -LLSD::LLSD(Real v) : impl(0) { assign(v); } -LLSD::LLSD(const UUID& v) : impl(0) { assign(v); } -LLSD::LLSD(const String& v) : impl(0) { assign(v); } -LLSD::LLSD(const Date& v) : impl(0) { assign(v); } -LLSD::LLSD(const URI& v) : impl(0) { assign(v); } -LLSD::LLSD(const Binary& v) : impl(0) { assign(v); } +// Scalar Constructors +LLSD::LLSD(Boolean v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(Integer v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(Real v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(const UUID& v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(const String& v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(const Date& v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(const URI& v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } +LLSD::LLSD(const Binary& v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } // Convenience Constructors -LLSD::LLSD(F32 v) : impl(0) { assign((Real)v); } +LLSD::LLSD(F32 v) : impl(0) { ALLOC_LLSD_OBJECT; assign((Real)v); } // Scalar Assignment void LLSD::assign(Boolean v) { safe(impl).assign(impl, v); } @@ -726,7 +840,7 @@ LLSD::URI LLSD::asURI() const { return safe(impl).asURI(); } LLSD::Binary LLSD::asBinary() const { return safe(impl).asBinary(); } // const char * helpers -LLSD::LLSD(const char* v) : impl(0) { assign(v); } +LLSD::LLSD(const char* v) : impl(0) { ALLOC_LLSD_OBJECT; assign(v); } void LLSD::assign(const char* v) { if(v) assign(std::string(v)); @@ -784,9 +898,6 @@ LLSD& LLSD::operator[](Integer i) const LLSD& LLSD::operator[](Integer i) const { return safe(impl).ref(i); } -U32 LLSD::allocationCount() { return Impl::sAllocationCount; } -U32 LLSD::outstandingCount() { return Impl::sOutstandingCount; } - static const char *llsd_dump(const LLSD &llsd, bool useXMLFormat) { // sStorage is used to hold the string representation of the llsd last @@ -801,15 +912,9 @@ static const char *llsd_dump(const LLSD &llsd, bool useXMLFormat) { std::ostringstream out; if (useXMLFormat) - { - LLSDXMLStreamer xml_streamer(llsd); - out << xml_streamer; - } + out << LLSDXMLStreamer(llsd); else - { - LLSDNotationStreamer notation_streamer(llsd); - out << notation_streamer; - } + out << LLSDNotationStreamer(llsd); out_string = out.str(); } int len = out_string.length(); @@ -840,3 +945,38 @@ LLSD::array_iterator LLSD::beginArray() { return makeArray(impl).beginArray(); LLSD::array_iterator LLSD::endArray() { return makeArray(impl).endArray(); } LLSD::array_const_iterator LLSD::beginArray() const{ return safe(impl).beginArray(); } LLSD::array_const_iterator LLSD::endArray() const { return safe(impl).endArray(); } + +namespace llsd +{ + +U32 allocationCount() { return LLSD::Impl::sAllocationCount; } +U32 outstandingCount() { return LLSD::Impl::sOutstandingCount; } + +// Diagnostic dump of contents in an LLSD object +void dumpStats(const LLSD& llsd) { LLSD::Impl::getImpl(llsd).dumpStats(); } + +} // namespace llsd + +// static +std::string LLSD::typeString(Type type) +{ + static const char * sTypeNameArray[] = { + "Undefined", + "Boolean", + "Integer", + "Real", + "String", + "UUID", + "Date", + "URI", + "Binary", + "Map", + "Array" + }; + + if (0 <= type && type < LL_ARRAY_SIZE(sTypeNameArray)) + { + return sTypeNameArray[type]; + } + return STRINGIZE("** invalid type value " << type); +} diff --git a/indra/llcommon/llsd.h b/indra/llcommon/llsd.h index 90d0f97873..5eb69059ac 100644 --- a/indra/llcommon/llsd.h +++ b/indra/llcommon/llsd.h @@ -40,10 +40,10 @@ /** LLSD provides a flexible data system similar to the data facilities of dynamic languages like Perl and Python. It is created to support exchange - of structured data between loosly coupled systems. (Here, "loosly coupled" + of structured data between loosely coupled systems. (Here, "loosely coupled" means not compiled together into the same module.) - Data in such exchanges must be highly tollerant of changes on either side + Data in such exchanges must be highly tolerant of changes on either side such as: - recompilation - implementation in a different langauge @@ -51,19 +51,19 @@ - execution of older versions (with fewer parameters) To this aim, the C++ API of LLSD strives to be very easy to use, and to - default to "the right thing" whereever possible. It is extremely tollerant + default to "the right thing" wherever possible. It is extremely tolerant of errors and unexpected situations. - The fundimental class is LLSD. LLSD is a value holding object. It holds + The fundamental class is LLSD. LLSD is a value holding object. It holds one value that is either undefined, one of the scalar types, or a map or an array. LLSD objects have value semantics (copying them copies the value, - though it can be considered efficient, due to shareing.), and mutable. + though it can be considered efficient, due to sharing), and mutable. Undefined is the singular value given to LLSD objects that are not initialized with any data. It is also used as the return value for - operations that return an LLSD, + operations that return an LLSD. - The sclar data types are: + The scalar data types are: - Boolean - true or false - Integer - a 32 bit signed integer - Real - a 64 IEEE 754 floating point value @@ -80,9 +80,73 @@ An array is a sequence of zero or more LLSD values. + Thread Safety + + In general, these LLSD classes offer *less* safety than STL container + classes. Implementations prior to this one were unsafe even when + completely unrelated LLSD trees were in two threads due to reference + sharing of special 'undefined' values that participated in the reference + counting mechanism. + + The dereference-before-refcount and aggressive tree sharing also make + it impractical to share an LLSD across threads. A strategy of passing + ownership or a copy to another thread is still difficult due to a lack + of a cloning interface but it can be done with some care. + + One way of transferring ownership is as follows: + + void method(const LLSD input) { + ... + LLSD * xfer_tree = new LLSD(); + { + // Top-level values + (* xfer_tree)['label'] = "Some text"; + (* xfer_tree)['mode'] = APP_MODE_CONSTANT; + + // There will be a second-level + LLSD subtree(LLSD::emptyMap()); + (* xfer_tree)['subtree'] = subtree; + + // Do *not* copy from LLSD objects via LLSD + // intermediaries. Only use plain-old-data + // types as intermediaries to prevent reference + // sharing. + subtree['value1'] = input['value1'].asInteger(); + subtree['value2'] = input['value2'].asString(); + + // Close scope and drop 'subtree's reference. + // Only xfer_tree has a reference to the second + // level data. + } + ... + // Transfer the LLSD pointer to another thread. Ownership + // transfers, this thread no longer has a reference to any + // part of the xfer_tree and there's nothing to free or + // release here. Receiving thread does need to delete the + // pointer when it is done with the LLSD. Transfer + // mechanism must perform correct data ordering operations + // as dictated by architecture. + other_thread.sendMessageAndPointer("Take This", xfer_tree); + xfer_tree = NULL; + + + Avoid this pattern which provides half of a race condition: + + void method(const LLSD input) { + ... + LLSD xfer_tree(LLSD::emptyMap()); + xfer_tree['label'] = "Some text"; + xfer_tree['mode'] = APP_MODE_CONSTANT; + ... + other_thread.sendMessageAndPointer("Take This", xfer_tree); + + @nosubgrouping */ +// Normally undefined, used for diagnostics +//#define LLSD_DEBUG_INFO 1 + class LL_COMMON_API LLSD { public: @@ -202,7 +266,7 @@ public: //@} /** @name Character Pointer Helpers - These are helper routines to make working with char* the same as easy as + These are helper routines to make working with char* as easy as working with strings. */ //@{ @@ -266,7 +330,7 @@ public: /** @name Type Testing */ //@{ enum Type { - TypeUndefined, + TypeUndefined = 0, TypeBoolean, TypeInteger, TypeReal, @@ -276,7 +340,10 @@ public: TypeURI, TypeBinary, TypeMap, - TypeArray + TypeArray, + TypeLLSDTypeEnd, + TypeLLSDTypeBegin = TypeUndefined, + TypeLLSDNumTypes = (TypeLLSDTypeEnd - TypeLLSDTypeBegin) }; Type type() const; @@ -302,7 +369,7 @@ public: If you get a linker error about these being missing, you have made mistake in your code. DO NOT IMPLEMENT THESE FUNCTIONS as a fix. - All of thse problems stem from trying to support char* in LLSD or in + All of these problems stem from trying to support char* in LLSD or in std::string. There are too many automatic casts that will lead to using an arbitrary pointer or scalar type to std::string. */ @@ -311,7 +378,7 @@ public: void assign(const void*); ///< assign from arbitrary pointers LLSD& operator=(const void*); ///< assign from arbitrary pointers - bool has(Integer) const; ///< has only works for Maps + bool has(Integer) const; ///< has() only works for Maps //@} /** @name Implementation */ @@ -320,13 +387,7 @@ public: class Impl; private: Impl* impl; - //@} - - /** @name Unit Testing Interface */ - //@{ -public: - static U32 allocationCount(); ///< how many Impls have been made - static U32 outstandingCount(); ///< how many Impls are still alive + friend class LLSD::Impl; //@} private: @@ -338,6 +399,10 @@ private: /// Returns Notation version of llsd -- only to be called from debugger static const char *dump(const LLSD &llsd); //@} + +public: + + static std::string typeString(Type type); // Return human-readable type as a string }; struct llsd_select_bool : public std::unary_function<LLSD, LLSD::Boolean> @@ -385,9 +450,32 @@ struct llsd_select_string : public std::unary_function<LLSD, LLSD::String> LL_COMMON_API std::ostream& operator<<(std::ostream& s, const LLSD& llsd); +namespace llsd +{ + +#ifdef LLSD_DEBUG_INFO +/** @name Unit Testing Interface */ +//@{ + LL_COMMON_API void dumpStats(const LLSD&); ///< Output information on object and usage + + /// @warn THE FOLLOWING COUNTS WILL NOT BE ACCURATE IN A MULTI-THREADED + /// ENVIRONMENT. + /// + /// These counts track LLSD::Impl (hidden) objects. + LL_COMMON_API U32 allocationCount(); ///< how many Impls have been made + LL_COMMON_API U32 outstandingCount(); ///< how many Impls are still alive + + /// These counts track LLSD (public) objects. + LL_COMMON_API extern S32 sLLSDAllocationCount; ///< Number of LLSD objects ever created + LL_COMMON_API extern S32 sLLSDNetObjects; ///< Number of LLSD objects that exist +#endif +//@} + +} // namespace llsd + /** QUESTIONS & TO DOS - - Would Binary be more convenient as usigned char* buffer semantics? - - Should Binary be convertable to/from String, and if so how? + - Would Binary be more convenient as unsigned char* buffer semantics? + - Should Binary be convertible to/from String, and if so how? - as UTF8 encoded strings (making not like UUID<->String) - as Base64 or Base96 encoded (making like UUID<->String) - Conversions to std::string and LLUUID do not result in easy assignment diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 10f460e8a6..b419101b7e 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -34,6 +34,12 @@ #include <iostream> #include "apr_base64.h" +#ifdef LL_STANDALONE +# include <zlib.h> +#else +# include "zlib/zlib.h" // for davep's dirty little zip functions +#endif + #if !LL_WINDOWS #include <netinet/in.h> // htonl & ntohl #endif @@ -104,7 +110,7 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, S32 max_bytes) if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ { legacy_no_header = true; - inbuf = str.gcount(); + inbuf = (int)str.gcount(); } else { @@ -337,7 +343,7 @@ std::istream& LLSDParser::get( char delim) const { istr.get(s, n, delim); - if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); return istr; } @@ -347,7 +353,7 @@ std::istream& LLSDParser::get( char delim) const { istr.get(sb, delim); - if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); return istr; } @@ -371,7 +377,7 @@ std::istream& LLSDParser::read( std::streamsize n) const { istr.read(s, n); - if(mCheckLimits) mMaxBytesLeft -= istr.gcount(); + if(mCheckLimits) mMaxBytesLeft -= (int)istr.gcount(); return istr; } @@ -783,7 +789,7 @@ bool LLSDNotationParser::parseBinary(std::istream& istr, LLSD& data) const if(len) { value.resize(len); - account(fullread(istr, (char *)&value[0], len)); + account((int)fullread(istr, (char *)&value[0], len)); } c = get(istr); // strip off the trailing double-quote data = value; @@ -1063,7 +1069,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const if(size > 0) { value.resize(size); - account(fullread(istr, (char*)&value[0], size)); + account((int)fullread(istr, (char*)&value[0], size)); } data = value; } @@ -1194,7 +1200,7 @@ bool LLSDBinaryParser::parseString( if(size) { buf.resize(size); - account(fullread(istr, &buf[0], size)); + account((int)fullread(istr, &buf[0], size)); value.assign(buf.begin(), buf.end()); } return true; @@ -1636,7 +1642,7 @@ int deserialize_string_raw( const S32 BUF_LEN = 20; char buf[BUF_LEN]; /* Flawfinder: ignore */ istr.get(buf, BUF_LEN - 1, ')'); - count += istr.gcount(); + count += (int)istr.gcount(); int c = istr.get(); c = istr.get(); count += 2; @@ -1651,7 +1657,7 @@ int deserialize_string_raw( if(len) { buf.resize(len); - count += fullread(istr, (char *)&buf[0], len); + count += (int)fullread(istr, (char *)&buf[0], len); value.assign(buf.begin(), buf.end()); } c = istr.get(); @@ -1983,3 +1989,182 @@ std::ostream& operator<<(std::ostream& s, const LLSD& llsd) return s; } + +//dirty little zippers -- yell at davep if these are horrid + +//return a string containing gzipped bytes of binary serialized LLSD +// VERY inefficient -- creates several copies of LLSD block in memory +std::string zip_llsd(LLSD& data) +{ + std::stringstream llsd_strm; + + LLSDSerialize::toBinary(data, llsd_strm); + + const U32 CHUNK = 65536; + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + S32 ret = deflateInit(&strm, Z_BEST_COMPRESSION); + if (ret != Z_OK) + { + llwarns << "Failed to compress LLSD block." << llendl; + return std::string(); + } + + std::string source = llsd_strm.str(); + + U8 out[CHUNK]; + + strm.avail_in = source.size(); + strm.next_in = (U8*) source.data(); + U8* output = NULL; + + U32 cur_size = 0; + + U32 have = 0; + + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + + ret = deflate(&strm, Z_FINISH); + if (ret == Z_OK || ret == Z_STREAM_END) + { //copy result into output + if (strm.avail_out >= CHUNK) + { + free(output); + llwarns << "Failed to compress LLSD block." << llendl; + return std::string(); + } + + have = CHUNK-strm.avail_out; + output = (U8*) realloc(output, cur_size+have); + memcpy(output+cur_size, out, have); + cur_size += have; + } + else + { + free(output); + llwarns << "Failed to compress LLSD block." << llendl; + return std::string(); + } + } + while (ret == Z_OK); + + std::string::size_type size = cur_size; + + std::string result((char*) output, size); + deflateEnd(&strm); + free(output); + +#if 0 //verify results work with unzip_llsd + std::istringstream test(result); + LLSD test_sd; + if (!unzip_llsd(test_sd, test, result.size())) + { + llerrs << "Invalid compression result!" << llendl; + } +#endif + + return result; +} + +//decompress a block of LLSD from provided istream +// not very efficient -- creats a copy of decompressed LLSD block in memory +// and deserializes from that copy using LLSDSerialize +bool unzip_llsd(LLSD& data, std::istream& is, S32 size) +{ + U8* result = NULL; + U32 cur_size = 0; + z_stream strm; + + const U32 CHUNK = 65536; + + U8 *in = new U8[size]; + is.read((char*) in, size); + + U8 out[CHUNK]; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = size; + strm.next_in = in; + + S32 ret = inflateInit(&strm); + + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR) + { + inflateEnd(&strm); + free(result); + delete [] in; + return false; + } + + switch (ret) + { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + inflateEnd(&strm); + free(result); + delete [] in; + return false; + break; + } + + U32 have = CHUNK-strm.avail_out; + + result = (U8*) realloc(result, cur_size + have); + memcpy(result+cur_size, out, have); + cur_size += have; + + } while (ret == Z_OK); + + inflateEnd(&strm); + delete [] in; + + if (ret != Z_STREAM_END) + { + free(result); + return false; + } + + //result now points to the decompressed LLSD block + { + std::string res_str((char*) result, cur_size); + + std::string deprecated_header("<? LLSD/Binary ?>"); + + if (res_str.substr(0, deprecated_header.size()) == deprecated_header) + { + res_str = res_str.substr(deprecated_header.size()+1, cur_size); + } + cur_size = res_str.size(); + + std::istringstream istr(res_str); + + if (!LLSDSerialize::fromBinary(data, istr, cur_size)) + { + llwarns << "Failed to unzip LLSD block" << llendl; + free(result); + return false; + } + } + + free(result); + return true; +} + + + diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 1f096b5254..99a3ea3cd4 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -790,4 +790,8 @@ public: } }; +//dirty little zip functions -- yell at davep +LL_COMMON_API std::string zip_llsd(LLSD& data); +LL_COMMON_API bool unzip_llsd(LLSD& data, std::istream& is, S32 size); + #endif // LL_LLSDSERIALIZE_H diff --git a/indra/llcommon/llsdserialize_xml.cpp b/indra/llcommon/llsdserialize_xml.cpp index c5a7c6fc15..34b3dbb99a 100644 --- a/indra/llcommon/llsdserialize_xml.cpp +++ b/indra/llcommon/llsdserialize_xml.cpp @@ -373,10 +373,13 @@ S32 LLSDXMLParser::Impl::parse(std::istream& input, LLSD& data) { break; } - count = get_till_eol(input, (char *)buffer, BUFFER_SIZE); - if (!count) { - break; + + count = get_till_eol(input, (char *)buffer, BUFFER_SIZE); + if (!count) + { + break; + } } status = XML_ParseBuffer(mParser, count, false); @@ -461,7 +464,7 @@ S32 LLSDXMLParser::Impl::parseLines(std::istream& input, LLSD& data) } } - status = XML_ParseBuffer(mParser, num_read, false); + status = XML_ParseBuffer(mParser, (int)num_read, false); if (status == XML_STATUS_ERROR) { break; @@ -716,6 +719,7 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name) case ELEMENT_INTEGER: { S32 i; + // sscanf okay here with different locales - ints don't change for different locale settings like floats do. if ( sscanf(mCurrentContent.c_str(), "%d", &i ) == 1 ) { // See if sscanf works - it's faster value = i; @@ -729,15 +733,19 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name) case ELEMENT_REAL: { - F64 r; - if ( sscanf(mCurrentContent.c_str(), "%lf", &r ) == 1 ) - { // See if sscanf works - it's faster - value = r; - } - else - { - value = LLSD(mCurrentContent).asReal(); - } + value = LLSD(mCurrentContent).asReal(); + // removed since this breaks when locale has decimal separator that isn't '.' + // investigated changing local to something compatible each time but deemed higher + // risk that just using LLSD.asReal() each time. + //F64 r; + //if ( sscanf(mCurrentContent.c_str(), "%lf", &r ) == 1 ) + //{ // See if sscanf works - it's faster + // value = r; + //} + //else + //{ + // value = LLSD(mCurrentContent).asReal(); + //} } break; diff --git a/indra/llcommon/llsdutil.cpp b/indra/llcommon/llsdutil.cpp index f8f9ece058..803417d368 100644 --- a/indra/llcommon/llsdutil.cpp +++ b/indra/llcommon/llsdutil.cpp @@ -41,6 +41,7 @@ #include "llsdserialize.h" #include "stringize.h" +#include "is_approx_equal_fraction.h" #include <map> #include <set> @@ -571,7 +572,7 @@ std::string llsd_matches(const LLSD& prototype, const LLSD& data, const std::str return match_types(prototype.type(), TypeVector(), data.type(), pfx); } -bool llsd_equals(const LLSD& lhs, const LLSD& rhs) +bool llsd_equals(const LLSD& lhs, const LLSD& rhs, unsigned bits) { // We're comparing strict equality of LLSD representation rather than // performing any conversions. So if the types aren't equal, the LLSD @@ -588,6 +589,20 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) // Both are TypeUndefined. There's nothing more to know. return true; + case LLSD::TypeReal: + // This is where the 'bits' argument comes in handy. If passed + // explicitly, it means to use is_approx_equal_fraction() to compare. + if (bits >= 0) + { + return is_approx_equal_fraction(lhs.asReal(), rhs.asReal(), bits); + } + // Otherwise we compare bit representations, and the usual caveats + // about comparing floating-point numbers apply. Omitting 'bits' when + // comparing Real values is only useful when we expect identical bit + // representation for a given Real value, e.g. for integer-valued + // Reals. + return (lhs.asReal() == rhs.asReal()); + #define COMPARE_SCALAR(type) \ case LLSD::Type##type: \ /* LLSD::URI has operator!=() but not operator==() */ \ @@ -596,10 +611,6 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) COMPARE_SCALAR(Boolean); COMPARE_SCALAR(Integer); - // The usual caveats about comparing floating-point numbers apply. This is - // only useful when we expect identical bit representation for a given - // Real value, e.g. for integer-valued Reals. - COMPARE_SCALAR(Real); COMPARE_SCALAR(String); COMPARE_SCALAR(UUID); COMPARE_SCALAR(Date); @@ -617,7 +628,7 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) for ( ; lai != laend && rai != raend; ++lai, ++rai) { // If any one array element is unequal, the arrays are unequal. - if (! llsd_equals(*lai, *rai)) + if (! llsd_equals(*lai, *rai, bits)) return false; } // Here we've reached the end of one or the other array. They're equal @@ -644,7 +655,7 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) if (rhskeys.erase(lmi->first) != 1) return false; // Both maps have the current key. Compare values. - if (! llsd_equals(lmi->second, rhs[lmi->first])) + if (! llsd_equals(lmi->second, rhs[lmi->first], bits)) return false; } // We've now established that all the lhs keys have equal values in @@ -657,7 +668,7 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) // We expect that every possible type() value is specifically handled // above. Failing to extend this switch to support a new LLSD type is // an error that must be brought to the coder's attention. - LL_ERRS("llsd_equals") << "llsd_equals(" << lhs << ", " << rhs << "): " + LL_ERRS("llsd_equals") << "llsd_equals(" << lhs << ", " << rhs << ", " << bits << "): " "unknown type " << lhs.type() << LL_ENDL; return false; // pacify the compiler } diff --git a/indra/llcommon/llsdutil.h b/indra/llcommon/llsdutil.h index bb8c0690b1..532d3f9341 100644 --- a/indra/llcommon/llsdutil.h +++ b/indra/llcommon/llsdutil.h @@ -123,8 +123,10 @@ LL_COMMON_API BOOL compare_llsd_with_template( */ LL_COMMON_API std::string llsd_matches(const LLSD& prototype, const LLSD& data, const std::string& pfx=""); -/// Deep equality -LL_COMMON_API bool llsd_equals(const LLSD& lhs, const LLSD& rhs); +/// Deep equality. If you want to compare LLSD::Real values for approximate +/// equality rather than bitwise equality, pass @a bits as for +/// is_approx_equal_fraction(). +LL_COMMON_API bool llsd_equals(const LLSD& lhs, const LLSD& rhs, unsigned bits=-1); // Simple function to copy data out of input & output iterators if // there is no need for casting. @@ -138,4 +140,283 @@ template<typename Input> LLSD llsd_copy_array(Input iter, Input end) return dest; } +/***************************************************************************** +* LLSDArray +*****************************************************************************/ +/** + * Construct an LLSD::Array inline, with implicit conversion to LLSD. Usage: + * + * @code + * void somefunc(const LLSD&); + * ... + * somefunc(LLSDArray("text")(17)(3.14)); + * @endcode + * + * For completeness, LLSDArray() with no args constructs an empty array, so + * <tt>LLSDArray()("text")(17)(3.14)</tt> produces an array equivalent to the + * above. But for most purposes, LLSD() is already equivalent to an empty + * array, and if you explicitly want an empty isArray(), there's + * LLSD::emptyArray(). However, supporting a no-args LLSDArray() constructor + * follows the principle of least astonishment. + */ +class LLSDArray +{ +public: + LLSDArray(): + _data(LLSD::emptyArray()) + {} + + /** + * Need an explicit copy constructor. Consider the following: + * + * @code + * LLSD array_of_arrays(LLSDArray(LLSDArray(17)(34)) + * (LLSDArray("x")("y"))); + * @endcode + * + * The coder intends to construct [[17, 34], ["x", "y"]]. + * + * With the compiler's implicit copy constructor, s/he gets instead + * [17, 34, ["x", "y"]]. + * + * The expression LLSDArray(17)(34) constructs an LLSDArray with those two + * values. The reader assumes it should be converted to LLSD, as we always + * want with LLSDArray, before passing it to the @em outer LLSDArray + * constructor! This copy constructor makes that happen. + */ + LLSDArray(const LLSDArray& inner): + _data(LLSD::emptyArray()) + { + _data.append(inner); + } + + LLSDArray(const LLSD& value): + _data(LLSD::emptyArray()) + { + _data.append(value); + } + + LLSDArray& operator()(const LLSD& value) + { + _data.append(value); + return *this; + } + + operator LLSD() const { return _data; } + LLSD get() const { return _data; } + +private: + LLSD _data; +}; + +/***************************************************************************** +* LLSDMap +*****************************************************************************/ +/** + * Construct an LLSD::Map inline, with implicit conversion to LLSD. Usage: + * + * @code + * void somefunc(const LLSD&); + * ... + * somefunc(LLSDMap("alpha", "abc")("number", 17)("pi", 3.14)); + * @endcode + * + * For completeness, LLSDMap() with no args constructs an empty map, so + * <tt>LLSDMap()("alpha", "abc")("number", 17)("pi", 3.14)</tt> produces a map + * equivalent to the above. But for most purposes, LLSD() is already + * equivalent to an empty map, and if you explicitly want an empty isMap(), + * there's LLSD::emptyMap(). However, supporting a no-args LLSDMap() + * constructor follows the principle of least astonishment. + */ +class LLSDMap +{ +public: + LLSDMap(): + _data(LLSD::emptyMap()) + {} + LLSDMap(const LLSD::String& key, const LLSD& value): + _data(LLSD::emptyMap()) + { + _data[key] = value; + } + + LLSDMap& operator()(const LLSD::String& key, const LLSD& value) + { + _data[key] = value; + return *this; + } + + operator LLSD() const { return _data; } + LLSD get() const { return _data; } + +private: + LLSD _data; +}; + +/***************************************************************************** +* LLSDParam +*****************************************************************************/ +/** + * LLSDParam is a customization point for passing LLSD values to function + * parameters of more or less arbitrary type. LLSD provides a small set of + * native conversions; but if a generic algorithm explicitly constructs an + * LLSDParam object in the function's argument list, a consumer can provide + * LLSDParam specializations to support more different parameter types than + * LLSD's native conversions. + * + * Usage: + * + * @code + * void somefunc(const paramtype&); + * ... + * somefunc(..., LLSDParam<paramtype>(someLLSD), ...); + * @endcode + */ +template <typename T> +class LLSDParam +{ +public: + /** + * Default implementation converts to T on construction, saves converted + * value for later retrieval + */ + LLSDParam(const LLSD& value): + _value(value) + {} + + operator T() const { return _value; } + +private: + T _value; +}; + +/** + * Turns out that several target types could accept an LLSD param using any of + * a few different conversions, e.g. LLUUID's constructor can accept LLUUID or + * std::string. Therefore, the compiler can't decide which LLSD conversion + * operator to choose, even though to us it seems obvious. But that's okay, we + * can specialize LLSDParam for such target types, explicitly specifying the + * desired conversion -- that's part of what LLSDParam is all about. Turns out + * we have to do that enough to make it worthwhile generalizing. Use a macro + * because I need to specify one of the asReal, etc., explicit conversion + * methods as well as a type. If I'm overlooking a clever way to implement + * that using a template instead, feel free to reimplement. + */ +#define LLSDParam_for(T, AS) \ +template <> \ +class LLSDParam<T> \ +{ \ +public: \ + LLSDParam(const LLSD& value): \ + _value((T)value.AS()) \ + {} \ + \ + operator T() const { return _value; } \ + \ +private: \ + T _value; \ +} + +LLSDParam_for(float, asReal); +LLSDParam_for(LLUUID, asUUID); +LLSDParam_for(LLDate, asDate); +LLSDParam_for(LLURI, asURI); +LLSDParam_for(LLSD::Binary, asBinary); + +/** + * LLSDParam<const char*> is an example of the kind of conversion you can + * support with LLSDParam beyond native LLSD conversions. Normally you can't + * pass an LLSD object to a function accepting const char* -- but you can + * safely pass an LLSDParam<const char*>(yourLLSD). + */ +template <> +class LLSDParam<const char*> +{ +private: + // The difference here is that we store a std::string rather than a const + // char*. It's important that the LLSDParam object own the std::string. + std::string _value; + // We don't bother storing the incoming LLSD object, but we do have to + // distinguish whether _value is an empty string because the LLSD object + // contains an empty string or because it's isUndefined(). + bool _undefined; + +public: + LLSDParam(const LLSD& value): + _value(value), + _undefined(value.isUndefined()) + {} + + // The const char* we retrieve is for storage owned by our _value member. + // That's how we guarantee that the const char* is valid for the lifetime + // of this LLSDParam object. Constructing your LLSDParam in the argument + // list should ensure that the LLSDParam object will persist for the + // duration of the function call. + operator const char*() const + { + if (_undefined) + { + // By default, an isUndefined() LLSD object's asString() method + // will produce an empty string. But for a function accepting + // const char*, it's often important to be able to pass NULL, and + // isUndefined() seems like the best way. If you want to pass an + // empty string, you can still pass LLSD(""). Without this special + // case, though, no LLSD value could pass NULL. + return NULL; + } + return _value.c_str(); + } +}; + +namespace llsd +{ + +/***************************************************************************** +* BOOST_FOREACH() helpers for LLSD +*****************************************************************************/ +/// Usage: BOOST_FOREACH(LLSD item, inArray(someLLSDarray)) { ... } +class inArray +{ +public: + inArray(const LLSD& array): + _array(array) + {} + + typedef LLSD::array_const_iterator const_iterator; + typedef LLSD::array_iterator iterator; + + iterator begin() { return _array.beginArray(); } + iterator end() { return _array.endArray(); } + const_iterator begin() const { return _array.beginArray(); } + const_iterator end() const { return _array.endArray(); } + +private: + LLSD _array; +}; + +/// MapEntry is what you get from dereferencing an LLSD::map_[const_]iterator. +typedef std::map<LLSD::String, LLSD>::value_type MapEntry; + +/// Usage: BOOST_FOREACH([const] MapEntry& e, inMap(someLLSDmap)) { ... } +class inMap +{ +public: + inMap(const LLSD& map): + _map(map) + {} + + typedef LLSD::map_const_iterator const_iterator; + typedef LLSD::map_iterator iterator; + + iterator begin() { return _map.beginMap(); } + iterator end() { return _map.endMap(); } + const_iterator begin() const { return _map.beginMap(); } + const_iterator end() const { return _map.endMap(); } + +private: + LLSD _map; +}; + +} // namespace llsd + #endif // LL_LLSDUTIL_H diff --git a/indra/llcommon/llsingleton.h b/indra/llcommon/llsingleton.h index 7aee1bb85f..49d99f2cd0 100644 --- a/indra/llcommon/llsingleton.h +++ b/indra/llcommon/llsingleton.h @@ -100,12 +100,6 @@ private: DELETED } EInitState; - static void deleteSingleton() - { - delete getData().mSingletonInstance; - getData().mSingletonInstance = NULL; - } - // stores pointer to singleton instance // and tracks initialization state of singleton struct SingletonInstanceData @@ -120,7 +114,10 @@ private: ~SingletonInstanceData() { - deleteSingleton(); + if (mInitState != DELETED) + { + deleteSingleton(); + } } }; @@ -132,6 +129,33 @@ public: data.mInitState = DELETED; } + /** + * @brief Immediately delete the singleton. + * + * A subsequent call to LLProxy::getInstance() will construct a new + * instance of the class. + * + * LLSingletons are normally destroyed after main() has exited and the C++ + * runtime is cleaning up statically-constructed objects. Some classes + * derived from LLSingleton have objects that are part of a runtime system + * that is terminated before main() exits. Calling the destructor of those + * objects after the termination of their respective systems can cause + * crashes and other problems during termination of the project. Using this + * method to destroy the singleton early can prevent these crashes. + * + * An example where this is needed is for a LLSingleton that has an APR + * object as a member that makes APR calls on destruction. The APR system is + * shut down explicitly before main() exits. This causes a crash on exit. + * Using this method before the call to apr_terminate() and NOT calling + * getInstance() again will prevent the crash. + */ + static void deleteSingleton() + { + delete getData().mSingletonInstance; + getData().mSingletonInstance = NULL; + getData().mInitState = DELETED; + } + static SingletonInstanceData& getData() { // this is static to cache the lookup results diff --git a/indra/llcommon/llstat.cpp b/indra/llcommon/llstat.cpp index 8ba97d7730..057257057f 100644 --- a/indra/llcommon/llstat.cpp +++ b/indra/llcommon/llstat.cpp @@ -593,7 +593,7 @@ void LLStatTime::stop() { if ( LLStatAccum::SCALE_PER_FRAME == scale ) { - return mTotalTimeInFrame; + return (F32)mTotalTimeInFrame; } else { @@ -737,7 +737,7 @@ void LLPerfBlock::addStatsToLLSDandReset( LLSD & stats, } } else - { // WTF? Shouldn't have a NULL pointer in the map. + { // Shouldn't have a NULL pointer in the map. llwarns << "Unexpected NULL dynamic stat at '" << stats_full_path << "'" << llendl; } } diff --git a/indra/llcommon/llstring.cpp b/indra/llcommon/llstring.cpp index f3b48b0156..e7fe656808 100644 --- a/indra/llcommon/llstring.cpp +++ b/indra/llcommon/llstring.cpp @@ -936,13 +936,18 @@ LLStringUtil::size_type LLStringUtil::getSubstitution(const std::string& instr, { const std::string delims (","); - // Find the first ] - size_type pos2 = instr.find(']', start); + // Find the first [ + size_type pos1 = instr.find('[', start); + if (pos1 == std::string::npos) + return std::string::npos; + + //Find the first ] after the initial [ + size_type pos2 = instr.find(']', pos1); if (pos2 == std::string::npos) return std::string::npos; - // Find the last [ before ] - size_type pos1 = instr.find_last_of('[', pos2-1); + // Find the last [ before ] in case of nested [[]] + pos1 = instr.find_last_of('[', pos2-1); if (pos1 == std::string::npos || pos1 < start) return std::string::npos; diff --git a/indra/llcommon/llsys.cpp b/indra/llcommon/llsys.cpp index 10cdc7087b..6073bcd0a6 100644 --- a/indra/llcommon/llsys.cpp +++ b/indra/llcommon/llsys.cpp @@ -1,6 +1,6 @@ /** * @file llsys.cpp - * @brief Impelementation of the basic system query functions. + * @brief Implementation of the basic system query functions. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -24,6 +24,10 @@ * $/LicenseInfo$ */ +#if LL_WINDOWS +#pragma warning (disable : 4355) // 'this' used in initializer list: yes, intentionally +#endif + #include "linden_common.h" #include "llsys.h" @@ -36,21 +40,45 @@ #endif #include "llprocessor.h" +#include "llerrorcontrol.h" +#include "llevents.h" +#include "lltimer.h" +#include "llsdserialize.h" +#include "llsdutil.h" +#include <boost/bind.hpp> +#include <boost/circular_buffer.hpp> +#include <boost/regex.hpp> +#include <boost/foreach.hpp> +#include <boost/lexical_cast.hpp> +#include <boost/range.hpp> +#include <boost/utility/enable_if.hpp> +#include <boost/type_traits/is_integral.hpp> +#include <boost/type_traits/is_float.hpp> + +using namespace llsd; #if LL_WINDOWS # define WIN32_LEAN_AND_MEAN # include <winsock2.h> # include <windows.h> +# include <psapi.h> // GetPerformanceInfo() et al. #elif LL_DARWIN # include <errno.h> # include <sys/sysctl.h> # include <sys/utsname.h> # include <stdint.h> +# include <Carbon/Carbon.h> +# include <stdexcept> +# include <mach/host_info.h> +# include <mach/mach_host.h> +# include <mach/task.h> +# include <mach/task_info.h> #elif LL_LINUX # include <errno.h> # include <sys/utsname.h> # include <unistd.h> # include <sys/sysinfo.h> +# include <stdexcept> const char MEMINFO_FILE[] = "/proc/meminfo"; #elif LL_SOLARIS # include <stdio.h> @@ -69,6 +97,15 @@ extern int errno; static const S32 CPUINFO_BUFFER_SIZE = 16383; LLCPUInfo gSysCPU; +// Don't log memory info any more often than this. It also serves as our +// framerate sample size. +static const F32 MEM_INFO_THROTTLE = 20; +// Sliding window of samples. We intentionally limit the length of time we +// remember "the slowest" framerate because framerate is very slow at login. +// If we only triggered FrameWatcher logging when the session framerate +// dropped below the login framerate, we'd have very little additional data. +static const F32 MEM_INFO_WINDOW = 10*60; + #if LL_WINDOWS #ifndef DLLVERSIONINFO typedef struct _DllVersionInfo @@ -187,22 +224,30 @@ LLOSInfo::LLOSInfo() : if(osvi.wProductType == VER_NT_WORKSTATION) mOSStringSimple = "Microsoft Windows XP x64 Edition "; else - mOSStringSimple = "Microsoft Windows Server 2003 "; + mOSStringSimple = "Microsoft Windows Server 2003 "; } - else if(osvi.dwMajorVersion == 6 && osvi.dwMinorVersion <= 1) + else if(osvi.dwMajorVersion == 6 && osvi.dwMinorVersion <= 2) { if(osvi.dwMinorVersion == 0) { - mOSStringSimple = "Microsoft Windows Vista "; + if(osvi.wProductType == VER_NT_WORKSTATION) + mOSStringSimple = "Microsoft Windows Vista "; + else + mOSStringSimple = "Windows Server 2008 "; } else if(osvi.dwMinorVersion == 1) { - mOSStringSimple = "Microsoft Windows 7 "; + if(osvi.wProductType == VER_NT_WORKSTATION) + mOSStringSimple = "Microsoft Windows 7 "; + else + mOSStringSimple = "Windows Server 2008 R2 "; } - - if(osvi.wProductType != VER_NT_WORKSTATION) + else if(osvi.dwMinorVersion == 2) { - mOSStringSimple += "Server "; + if(osvi.wProductType == VER_NT_WORKSTATION) + mOSStringSimple = "Microsoft Windows 8 "; + else + mOSStringSimple = "Windows Server 2012 "; } ///get native system info if available.. @@ -307,8 +352,7 @@ LLOSInfo::LLOSInfo() : std::string compatibility_mode; if(got_shell32_version) { - if(osvi.dwMajorVersion != shell32_major - || osvi.dwMinorVersion != shell32_minor) + if(osvi.dwMajorVersion != shell32_major || osvi.dwMinorVersion != shell32_minor) { compatibility_mode = llformat(" compatibility mode. real ver: %d.%d (Build %d)", shell32_major, @@ -318,7 +362,58 @@ LLOSInfo::LLOSInfo() : } mOSString += compatibility_mode; +#elif LL_DARWIN + + // Initialize mOSStringSimple to something like: + // "Mac OS X 10.6.7" + { + const char * DARWIN_PRODUCT_NAME = "Mac OS X"; + + SInt32 major_version, minor_version, bugfix_version; + OSErr r1 = Gestalt(gestaltSystemVersionMajor, &major_version); + OSErr r2 = Gestalt(gestaltSystemVersionMinor, &minor_version); + OSErr r3 = Gestalt(gestaltSystemVersionBugFix, &bugfix_version); + + if((r1 == noErr) && (r2 == noErr) && (r3 == noErr)) + { + mMajorVer = major_version; + mMinorVer = minor_version; + mBuild = bugfix_version; + + std::stringstream os_version_string; + os_version_string << DARWIN_PRODUCT_NAME << " " << mMajorVer << "." << mMinorVer << "." << mBuild; + + // Put it in the OS string we are compiling + mOSStringSimple.append(os_version_string.str()); + } + else + { + mOSStringSimple.append("Unable to collect OS info"); + } + } + + // Initialize mOSString to something like: + // "Mac OS X 10.6.7 Darwin Kernel Version 10.7.0: Sat Jan 29 15:17:16 PST 2011; root:xnu-1504.9.37~1/RELEASE_I386 i386" + struct utsname un; + if(uname(&un) != -1) + { + mOSString = mOSStringSimple; + mOSString.append(" "); + mOSString.append(un.sysname); + mOSString.append(" "); + mOSString.append(un.release); + mOSString.append(" "); + mOSString.append(un.version); + mOSString.append(" "); + mOSString.append(un.machine); + } + else + { + mOSString = mOSStringSimple; + } + #else + struct utsname un; if(uname(&un) != -1) { @@ -334,15 +429,7 @@ LLOSInfo::LLOSInfo() : // Simplify 'Simple' std::string ostype = mOSStringSimple.substr(0, mOSStringSimple.find_first_of(" ", 0)); - if (ostype == "Darwin") - { - // Only care about major Darwin versions, truncate at first '.' - S32 idx1 = mOSStringSimple.find_first_of(".", 0); - std::string simple = mOSStringSimple.substr(0, idx1); - if (simple.length() > 0) - mOSStringSimple = simple; - } - else if (ostype == "Linux") + if (ostype == "Linux") { // Only care about major and minor Linux versions, truncate at second '.' std::string::size_type idx1 = mOSStringSimple.find_first_of(".", 0); @@ -521,6 +608,7 @@ LLCPUInfo::LLCPUInfo() out << " (" << mCPUMHz << " MHz)"; } mCPUString = out.str(); + LLStringUtil::trim(mCPUString); } bool LLCPUInfo::hasAltivec() const @@ -562,8 +650,78 @@ void LLCPUInfo::stream(std::ostream& s) const s << "->mCPUString: " << mCPUString << std::endl; } +// Helper class for LLMemoryInfo: accumulate stats in the form we store for +// LLMemoryInfo::getStatsMap(). +class Stats +{ +public: + Stats(): + mStats(LLSD::emptyMap()) + {} + + // Store every integer type as LLSD::Integer. + template <class T> + void add(const LLSD::String& name, const T& value, + typename boost::enable_if<boost::is_integral<T> >::type* = 0) + { + mStats[name] = LLSD::Integer(value); + } + + // Store every floating-point type as LLSD::Real. + template <class T> + void add(const LLSD::String& name, const T& value, + typename boost::enable_if<boost::is_float<T> >::type* = 0) + { + mStats[name] = LLSD::Real(value); + } + + // Hope that LLSD::Date values are sufficiently unambiguous. + void add(const LLSD::String& name, const LLSD::Date& value) + { + mStats[name] = value; + } + + LLSD get() const { return mStats; } + +private: + LLSD mStats; +}; + +// Wrap boost::regex_match() with a function that doesn't throw. +template <typename S, typename M, typename R> +static bool regex_match_no_exc(const S& string, M& match, const R& regex) +{ + try + { + return boost::regex_match(string, match, regex); + } + catch (const std::runtime_error& e) + { + LL_WARNS("LLMemoryInfo") << "error matching with '" << regex.str() << "': " + << e.what() << ":\n'" << string << "'" << LL_ENDL; + return false; + } +} + +// Wrap boost::regex_search() with a function that doesn't throw. +template <typename S, typename M, typename R> +static bool regex_search_no_exc(const S& string, M& match, const R& regex) +{ + try + { + return boost::regex_search(string, match, regex); + } + catch (const std::runtime_error& e) + { + LL_WARNS("LLMemoryInfo") << "error searching with '" << regex.str() << "': " + << e.what() << ":\n'" << string << "'" << LL_ENDL; + return false; + } +} + LLMemoryInfo::LLMemoryInfo() { + refresh(); } #if LL_WINDOWS @@ -587,11 +745,7 @@ static U32 LLMemoryAdjustKBResult(U32 inKB) U32 LLMemoryInfo::getPhysicalMemoryKB() const { #if LL_WINDOWS - MEMORYSTATUSEX state; - state.dwLength = sizeof(state); - GlobalMemoryStatusEx(&state); - - return LLMemoryAdjustKBResult((U32)(state.ullTotalPhys >> 10)); + return LLMemoryAdjustKBResult(mStatsMap["Total Physical KB"].asInteger()); #elif LL_DARWIN // This might work on Linux as well. Someone check... @@ -639,12 +793,82 @@ U32 LLMemoryInfo::getPhysicalMemoryClamped() const void LLMemoryInfo::getAvailableMemoryKB(U32& avail_physical_mem_kb, U32& avail_virtual_mem_kb) { #if LL_WINDOWS - MEMORYSTATUSEX state; - state.dwLength = sizeof(state); - GlobalMemoryStatusEx(&state); + // Sigh, this shouldn't be a static method, then we wouldn't have to + // reload this data separately from refresh() + LLSD statsMap(loadStatsMap()); + + avail_physical_mem_kb = statsMap["Avail Physical KB"].asInteger(); + avail_virtual_mem_kb = statsMap["Avail Virtual KB"].asInteger(); + +#elif LL_DARWIN + // mStatsMap is derived from vm_stat, look for (e.g.) "kb free": + // $ vm_stat + // Mach Virtual Memory Statistics: (page size of 4096 bytes) + // Pages free: 462078. + // Pages active: 142010. + // Pages inactive: 220007. + // Pages wired down: 159552. + // "Translation faults": 220825184. + // Pages copy-on-write: 2104153. + // Pages zero filled: 167034876. + // Pages reactivated: 65153. + // Pageins: 2097212. + // Pageouts: 41759. + // Object cache: 841598 hits of 7629869 lookups (11% hit rate) + avail_physical_mem_kb = -1 ; + avail_virtual_mem_kb = -1 ; - avail_physical_mem_kb = (U32)(state.ullAvailPhys/1024) ; - avail_virtual_mem_kb = (U32)(state.ullAvailVirtual/1024) ; +#elif LL_LINUX + // mStatsMap is derived from MEMINFO_FILE: + // $ cat /proc/meminfo + // MemTotal: 4108424 kB + // MemFree: 1244064 kB + // Buffers: 85164 kB + // Cached: 1990264 kB + // SwapCached: 0 kB + // Active: 1176648 kB + // Inactive: 1427532 kB + // Active(anon): 529152 kB + // Inactive(anon): 15924 kB + // Active(file): 647496 kB + // Inactive(file): 1411608 kB + // Unevictable: 16 kB + // Mlocked: 16 kB + // HighTotal: 3266316 kB + // HighFree: 721308 kB + // LowTotal: 842108 kB + // LowFree: 522756 kB + // SwapTotal: 6384632 kB + // SwapFree: 6384632 kB + // Dirty: 28 kB + // Writeback: 0 kB + // AnonPages: 528820 kB + // Mapped: 89472 kB + // Shmem: 16324 kB + // Slab: 159624 kB + // SReclaimable: 145168 kB + // SUnreclaim: 14456 kB + // KernelStack: 2560 kB + // PageTables: 5560 kB + // NFS_Unstable: 0 kB + // Bounce: 0 kB + // WritebackTmp: 0 kB + // CommitLimit: 8438844 kB + // Committed_AS: 1271596 kB + // VmallocTotal: 122880 kB + // VmallocUsed: 65252 kB + // VmallocChunk: 52356 kB + // HardwareCorrupted: 0 kB + // HugePages_Total: 0 + // HugePages_Free: 0 + // HugePages_Rsvd: 0 + // HugePages_Surp: 0 + // Hugepagesize: 2048 kB + // DirectMap4k: 434168 kB + // DirectMap2M: 477184 kB + // (could also run 'free', but easier to read a file than run a program) + avail_physical_mem_kb = -1 ; + avail_virtual_mem_kb = -1 ; #else //do not know how to collect available memory info for other systems. @@ -657,56 +881,283 @@ void LLMemoryInfo::getAvailableMemoryKB(U32& avail_physical_mem_kb, U32& avail_v void LLMemoryInfo::stream(std::ostream& s) const { + // We want these memory stats to be easy to grep from the log, along with + // the timestamp. So preface each line with the timestamp and a + // distinctive marker. Without that, we'd have to search the log for the + // introducer line, then read subsequent lines, etc... + std::string pfx(LLError::utcTime() + " <mem> "); + + // Max key length + size_t key_width(0); + BOOST_FOREACH(const MapEntry& pair, inMap(mStatsMap)) + { + size_t len(pair.first.length()); + if (len > key_width) + { + key_width = len; + } + } + + // Now stream stats + BOOST_FOREACH(const MapEntry& pair, inMap(mStatsMap)) + { + s << pfx << std::setw(key_width+1) << (pair.first + ':') << ' '; + LLSD value(pair.second); + if (value.isInteger()) + s << std::setw(12) << value.asInteger(); + else if (value.isReal()) + s << std::fixed << std::setprecision(1) << value.asReal(); + else if (value.isDate()) + value.asDate().toStream(s); + else + s << value; // just use default LLSD formatting + s << std::endl; + } +} + +LLSD LLMemoryInfo::getStatsMap() const +{ + return mStatsMap; +} + +LLMemoryInfo& LLMemoryInfo::refresh() +{ + mStatsMap = loadStatsMap(); + + LL_DEBUGS("LLMemoryInfo") << "Populated mStatsMap:\n"; + LLSDSerialize::toPrettyXML(mStatsMap, LL_CONT); + LL_ENDL; + + return *this; +} + +LLSD LLMemoryInfo::loadStatsMap() +{ + // This implementation is derived from stream() code (as of 2011-06-29). + Stats stats; + + // associate timestamp for analysis over time + stats.add("timestamp", LLDate::now()); + #if LL_WINDOWS MEMORYSTATUSEX state; state.dwLength = sizeof(state); GlobalMemoryStatusEx(&state); - s << "Percent Memory use: " << (U32)state.dwMemoryLoad << '%' << std::endl; - s << "Total Physical KB: " << (U32)(state.ullTotalPhys/1024) << std::endl; - s << "Avail Physical KB: " << (U32)(state.ullAvailPhys/1024) << std::endl; - s << "Total page KB: " << (U32)(state.ullTotalPageFile/1024) << std::endl; - s << "Avail page KB: " << (U32)(state.ullAvailPageFile/1024) << std::endl; - s << "Total Virtual KB: " << (U32)(state.ullTotalVirtual/1024) << std::endl; - s << "Avail Virtual KB: " << (U32)(state.ullAvailVirtual/1024) << std::endl; + stats.add("Percent Memory use", state.dwMemoryLoad); + stats.add("Total Physical KB", state.ullTotalPhys/1024); + stats.add("Avail Physical KB", state.ullAvailPhys/1024); + stats.add("Total page KB", state.ullTotalPageFile/1024); + stats.add("Avail page KB", state.ullAvailPageFile/1024); + stats.add("Total Virtual KB", state.ullTotalVirtual/1024); + stats.add("Avail Virtual KB", state.ullAvailVirtual/1024); + + PERFORMANCE_INFORMATION perf; + perf.cb = sizeof(perf); + GetPerformanceInfo(&perf, sizeof(perf)); + + SIZE_T pagekb(perf.PageSize/1024); + stats.add("CommitTotal KB", perf.CommitTotal * pagekb); + stats.add("CommitLimit KB", perf.CommitLimit * pagekb); + stats.add("CommitPeak KB", perf.CommitPeak * pagekb); + stats.add("PhysicalTotal KB", perf.PhysicalTotal * pagekb); + stats.add("PhysicalAvail KB", perf.PhysicalAvailable * pagekb); + stats.add("SystemCache KB", perf.SystemCache * pagekb); + stats.add("KernelTotal KB", perf.KernelTotal * pagekb); + stats.add("KernelPaged KB", perf.KernelPaged * pagekb); + stats.add("KernelNonpaged KB", perf.KernelNonpaged * pagekb); + stats.add("PageSize KB", pagekb); + stats.add("HandleCount", perf.HandleCount); + stats.add("ProcessCount", perf.ProcessCount); + stats.add("ThreadCount", perf.ThreadCount); + + PROCESS_MEMORY_COUNTERS_EX pmem; + pmem.cb = sizeof(pmem); + // GetProcessMemoryInfo() is documented to accept either + // PROCESS_MEMORY_COUNTERS* or PROCESS_MEMORY_COUNTERS_EX*, presumably + // using the redundant size info to distinguish. But its prototype + // specifically accepts PROCESS_MEMORY_COUNTERS*, and since this is a + // classic-C API, PROCESS_MEMORY_COUNTERS_EX isn't a subclass. Cast the + // pointer. + GetProcessMemoryInfo(GetCurrentProcess(), PPROCESS_MEMORY_COUNTERS(&pmem), sizeof(pmem)); + + stats.add("Page Fault Count", pmem.PageFaultCount); + stats.add("PeakWorkingSetSize KB", pmem.PeakWorkingSetSize/1024); + stats.add("WorkingSetSize KB", pmem.WorkingSetSize/1024); + stats.add("QutaPeakPagedPoolUsage KB", pmem.QuotaPeakPagedPoolUsage/1024); + stats.add("QuotaPagedPoolUsage KB", pmem.QuotaPagedPoolUsage/1024); + stats.add("QuotaPeakNonPagedPoolUsage KB", pmem.QuotaPeakNonPagedPoolUsage/1024); + stats.add("QuotaNonPagedPoolUsage KB", pmem.QuotaNonPagedPoolUsage/1024); + stats.add("PagefileUsage KB", pmem.PagefileUsage/1024); + stats.add("PeakPagefileUsage KB", pmem.PeakPagefileUsage/1024); + stats.add("PrivateUsage KB", pmem.PrivateUsage/1024); + #elif LL_DARWIN - uint64_t phys = 0; - size_t len = sizeof(phys); + const vm_size_t pagekb(vm_page_size / 1024); + + // + // Collect the vm_stat's + // - if(sysctlbyname("hw.memsize", &phys, &len, NULL, 0) == 0) { - s << "Total Physical KB: " << phys/1024 << std::endl; - } - else + vm_statistics_data_t vmstat; + mach_msg_type_number_t vmstatCount = HOST_VM_INFO_COUNT; + + if (host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t) &vmstat, &vmstatCount) != KERN_SUCCESS) { - s << "Unable to collect memory information"; + LL_WARNS("LLMemoryInfo") << "Unable to collect memory information" << LL_ENDL; + } + else + { + stats.add("Pages free KB", pagekb * vmstat.free_count); + stats.add("Pages active KB", pagekb * vmstat.active_count); + stats.add("Pages inactive KB", pagekb * vmstat.inactive_count); + stats.add("Pages wired KB", pagekb * vmstat.wire_count); + + stats.add("Pages zero fill", vmstat.zero_fill_count); + stats.add("Page reactivations", vmstat.reactivations); + stats.add("Page-ins", vmstat.pageins); + stats.add("Page-outs", vmstat.pageouts); + + stats.add("Faults", vmstat.faults); + stats.add("Faults copy-on-write", vmstat.cow_faults); + + stats.add("Cache lookups", vmstat.lookups); + stats.add("Cache hits", vmstat.hits); + + stats.add("Page purgeable count", vmstat.purgeable_count); + stats.add("Page purges", vmstat.purges); + + stats.add("Page speculative reads", vmstat.speculative_count); + } } + + // + // Collect the misc task info + // + + { + task_events_info_data_t taskinfo; + unsigned taskinfoSize = sizeof(taskinfo); + + if (task_info(mach_task_self(), TASK_EVENTS_INFO, (task_info_t) &taskinfo, &taskinfoSize) != KERN_SUCCESS) + { + LL_WARNS("LLMemoryInfo") << "Unable to collect task information" << LL_ENDL; + } + else + { + stats.add("Task page-ins", taskinfo.pageins); + stats.add("Task copy-on-write faults", taskinfo.cow_faults); + stats.add("Task messages sent", taskinfo.messages_sent); + stats.add("Task messages received", taskinfo.messages_received); + stats.add("Task mach system call count", taskinfo.syscalls_mach); + stats.add("Task unix system call count", taskinfo.syscalls_unix); + stats.add("Task context switch count", taskinfo.csw); + } + } + + // + // Collect the basic task info + // + + { + task_basic_info_64_data_t taskinfo; + unsigned taskinfoSize = sizeof(taskinfo); + + if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t) &taskinfo, &taskinfoSize) != KERN_SUCCESS) + { + LL_WARNS("LLMemoryInfo") << "Unable to collect task information" << LL_ENDL; + } + else + { + stats.add("Basic suspend count", taskinfo.suspend_count); + stats.add("Basic virtual memory KB", taskinfo.virtual_size / 1024); + stats.add("Basic resident memory KB", taskinfo.resident_size / 1024); + stats.add("Basic new thread policy", taskinfo.policy); + } + } + #elif LL_SOLARIS - U64 phys = 0; + U64 phys = 0; - phys = (U64)(sysconf(_SC_PHYS_PAGES)) * (U64)(sysconf(_SC_PAGESIZE)/1024); + phys = (U64)(sysconf(_SC_PHYS_PAGES)) * (U64)(sysconf(_SC_PAGESIZE)/1024); - s << "Total Physical KB: " << phys << std::endl; -#else - // *NOTE: This works on linux. What will it do on other systems? - LLFILE* meminfo = LLFile::fopen(MEMINFO_FILE,"rb"); - if(meminfo) + stats.add("Total Physical KB", phys); + +#elif LL_LINUX + std::ifstream meminfo(MEMINFO_FILE); + if (meminfo.is_open()) { - char line[MAX_STRING]; /* Flawfinder: ignore */ - memset(line, 0, MAX_STRING); - while(fgets(line, MAX_STRING, meminfo)) + // MemTotal: 4108424 kB + // MemFree: 1244064 kB + // Buffers: 85164 kB + // Cached: 1990264 kB + // SwapCached: 0 kB + // Active: 1176648 kB + // Inactive: 1427532 kB + // ... + // VmallocTotal: 122880 kB + // VmallocUsed: 65252 kB + // VmallocChunk: 52356 kB + // HardwareCorrupted: 0 kB + // HugePages_Total: 0 + // HugePages_Free: 0 + // HugePages_Rsvd: 0 + // HugePages_Surp: 0 + // Hugepagesize: 2048 kB + // DirectMap4k: 434168 kB + // DirectMap2M: 477184 kB + + // Intentionally don't pass the boost::no_except flag. This + // boost::regex object is constructed with a string literal, so it + // should be valid every time. If it becomes invalid, we WANT an + // exception, hopefully even before the dev checks in. + boost::regex stat_rx("(.+): +([0-9]+)( kB)?"); + boost::smatch matched; + + std::string line; + while (std::getline(meminfo, line)) { - line[strlen(line)-1] = ' '; /*Flawfinder: ignore*/ - s << line; + LL_DEBUGS("LLMemoryInfo") << line << LL_ENDL; + if (regex_match_no_exc(line, matched, stat_rx)) + { + // e.g. "MemTotal: 4108424 kB" + LLSD::String key(matched[1].first, matched[1].second); + LLSD::String value_str(matched[2].first, matched[2].second); + LLSD::Integer value(0); + try + { + value = boost::lexical_cast<LLSD::Integer>(value_str); + } + catch (const boost::bad_lexical_cast&) + { + LL_WARNS("LLMemoryInfo") << "couldn't parse '" << value_str + << "' in " << MEMINFO_FILE << " line: " + << line << LL_ENDL; + continue; + } + // Store this statistic. + stats.add(key, value); + } + else + { + LL_WARNS("LLMemoryInfo") << "unrecognized " << MEMINFO_FILE << " line: " + << line << LL_ENDL; + } } - fclose(meminfo); } else { - s << "Unable to collect memory information"; + LL_WARNS("LLMemoryInfo") << "Unable to collect memory information" << LL_ENDL; } + +#else + LL_WARNS("LLMemoryInfo") << "Unknown system; unable to collect memory information" << LL_ENDL; + #endif + + return stats.get(); } std::ostream& operator<<(std::ostream& s, const LLOSInfo& info) @@ -727,6 +1178,143 @@ std::ostream& operator<<(std::ostream& s, const LLMemoryInfo& info) return s; } +class FrameWatcher +{ +public: + FrameWatcher(): + // Hooking onto the "mainloop" event pump gets us one call per frame. + mConnection(LLEventPumps::instance() + .obtain("mainloop") + .listen("FrameWatcher", boost::bind(&FrameWatcher::tick, this, _1))), + // Initializing mSampleStart to an invalid timestamp alerts us to skip + // trying to compute framerate on the first call. + mSampleStart(-1), + // Initializing mSampleEnd to 0 ensures that we treat the first call + // as the completion of a sample window. + mSampleEnd(0), + mFrames(0), + // Both MEM_INFO_WINDOW and MEM_INFO_THROTTLE are in seconds. We need + // the number of integer MEM_INFO_THROTTLE sample slots that will fit + // in MEM_INFO_WINDOW. Round up. + mSamples(int((MEM_INFO_WINDOW / MEM_INFO_THROTTLE) + 0.7)), + // Initializing to F32_MAX means that the first real frame will become + // the slowest ever, which sounds like a good idea. + mSlowest(F32_MAX) + {} + + bool tick(const LLSD&) + { + F32 timestamp(mTimer.getElapsedTimeF32()); + + // Count this frame in the interval just completed. + ++mFrames; + + // Have we finished a sample window yet? + if (timestamp < mSampleEnd) + { + // no, just keep waiting + return false; + } + + // Set up for next sample window. Capture values for previous frame in + // local variables and reset data members. + U32 frames(mFrames); + F32 sampleStart(mSampleStart); + // No frames yet in next window + mFrames = 0; + // which starts right now + mSampleStart = timestamp; + // and ends MEM_INFO_THROTTLE seconds in the future + mSampleEnd = mSampleStart + MEM_INFO_THROTTLE; + + // On the very first call, that's all we can do, no framerate + // computation is possible. + if (sampleStart < 0) + { + return false; + } + + // How long did this actually take? As framerate slows, the duration + // of the frame we just finished could push us WELL beyond our desired + // sample window size. + F32 elapsed(timestamp - sampleStart); + F32 framerate(frames/elapsed); + + // Remember previous slowest framerate because we're just about to + // update it. + F32 slowest(mSlowest); + // Remember previous number of samples. + boost::circular_buffer<F32>::size_type prevSize(mSamples.size()); + + // Capture new framerate in our samples buffer. Once the buffer is + // full (after MEM_INFO_WINDOW seconds), this will displace the oldest + // sample. ("So they all rolled over, and one fell out...") + mSamples.push_back(framerate); + + // Calculate the new minimum framerate. I know of no way to update a + // rolling minimum without ever rescanning the buffer. But since there + // are only a few tens of items in this buffer, rescanning it is + // probably cheaper (and certainly easier to reason about) than + // attempting to optimize away some of the scans. + mSlowest = framerate; // pick an arbitrary entry to start + for (boost::circular_buffer<F32>::const_iterator si(mSamples.begin()), send(mSamples.end()); + si != send; ++si) + { + if (*si < mSlowest) + { + mSlowest = *si; + } + } + + // We're especially interested in memory as framerate drops. Only log + // when framerate drops below the slowest framerate we remember. + // (Should always be true for the end of the very first sample + // window.) + if (framerate >= slowest) + { + return false; + } + // Congratulations, we've hit a new low. :-P + + LL_INFOS("FrameWatcher") << ' '; + if (! prevSize) + { + LL_CONT << "initial framerate "; + } + else + { + LL_CONT << "slowest framerate for last " << int(prevSize * MEM_INFO_THROTTLE) + << " seconds "; + } + LL_CONT << std::fixed << std::setprecision(1) << framerate << '\n' + << LLMemoryInfo() << LL_ENDL; + + return false; + } + +private: + // Storing the connection in an LLTempBoundListener ensures it will be + // disconnected when we're destroyed. + LLTempBoundListener mConnection; + // Track elapsed time + LLTimer mTimer; + // Some of what you see here is in fact redundant with functionality you + // can get from LLTimer. Unfortunately the LLTimer API is missing the + // feature we need: has at least the stated interval elapsed, and if so, + // exactly how long has passed? So we have to do it by hand, sigh. + // Time at start, end of sample window + F32 mSampleStart, mSampleEnd; + // Frames this sample window + U32 mFrames; + // Sliding window of framerate samples + boost::circular_buffer<F32> mSamples; + // Slowest framerate in mSamples + F32 mSlowest; +}; + +// Need an instance of FrameWatcher before it does any good +static FrameWatcher sFrameWatcher; + BOOL gunzip_file(const std::string& srcfile, const std::string& dstfile) { std::string tmpfile; @@ -776,11 +1364,21 @@ BOOL gzip_file(const std::string& srcfile, const std::string& dstfile) src = LLFile::fopen(srcfile, "rb"); /* Flawfinder: ignore */ if (! src) goto err; - do + while ((bytes = (S32)fread(buffer, sizeof(U8), COMPRESS_BUFFER_SIZE, src)) > 0) { - bytes = (S32)fread(buffer, sizeof(U8), COMPRESS_BUFFER_SIZE,src); - gzwrite(dst, buffer, bytes); - } while(feof(src) == 0); + if (gzwrite(dst, buffer, bytes) <= 0) + { + llwarns << "gzwrite failed: " << gzerror(dst, NULL) << llendl; + goto err; + } + } + + if (ferror(src)) + { + llwarns << "Error reading " << srcfile << llendl; + goto err; + } + gzclose(dst); dst = NULL; #if LL_WINDOWS diff --git a/indra/llcommon/llsys.h b/indra/llcommon/llsys.h index 41a4f25000..739e795d3a 100644 --- a/indra/llcommon/llsys.h +++ b/indra/llcommon/llsys.h @@ -36,6 +36,7 @@ // llinfos << info << llendl; // +#include "llsd.h" #include <iosfwd> #include <string> @@ -117,6 +118,27 @@ public: //get the available memory infomation in KiloBytes. static void getAvailableMemoryKB(U32& avail_physical_mem_kb, U32& avail_virtual_mem_kb); + + // Retrieve a map of memory statistics. The keys of the map are platform- + // dependent. The values are in kilobytes to try to avoid integer overflow. + LLSD getStatsMap() const; + + // Re-fetch memory data (as reported by stream() and getStatsMap()) from the + // system. Normally this is fetched at construction time. Return (*this) + // to permit usage of the form: + // @code + // LLMemoryInfo info; + // ... + // info.refresh().getStatsMap(); + // @endcode + LLMemoryInfo& refresh(); + +private: + // set mStatsMap + static LLSD loadStatsMap(); + + // Memory stats for getStatsMap(). + LLSD mStatsMap; }; diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index 49d05ef411..a6ad6b125c 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -56,6 +56,21 @@ // //---------------------------------------------------------------------------- +#if !LL_DARWIN +U32 ll_thread_local sThreadID = 0; +#endif + +U32 LLThread::sIDIter = 0; + +LL_COMMON_API void assert_main_thread() +{ + static U32 s_thread_id = LLThread::currentID(); + if (LLThread::currentID() != s_thread_id) + { + llerrs << "Illegal execution outside main thread." << llendl; + } +} + // // Handed to the APR thread creation function // @@ -63,10 +78,14 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap { LLThread *threadp = (LLThread *)datap; +#if !LL_DARWIN + sThreadID = threadp->mID; +#endif + // Run the user supplied function threadp->run(); - llinfos << "LLThread::staticRun() Exiting: " << threadp->mName << llendl; + //llinfos << "LLThread::staticRun() Exiting: " << threadp->mName << llendl; // We're done with the run function, this thread is done executing now. threadp->mStatus = STOPPED; @@ -81,6 +100,8 @@ LLThread::LLThread(const std::string& name, apr_pool_t *poolp) : mAPRThreadp(NULL), mStatus(STOPPED) { + mID = ++sIDIter; + // Thread creation probably CAN be paranoid about APR being initialized, if necessary if (poolp) { @@ -121,7 +142,7 @@ void LLThread::shutdown() // First, set the flag that indicates that we're ready to die setQuitting(); - llinfos << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << llendl; + //llinfos << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << llendl; // Now wait a bit for the thread to exit // It's unclear whether I should even bother doing this - this destructor // should netver get called unless we're already stopped, really... @@ -143,7 +164,7 @@ void LLThread::shutdown() if (!isStopped()) { // This thread just wouldn't stop, even though we gave it time - llwarns << "LLThread::~LLThread() exiting thread before clean exit!" << llendl; + //llwarns << "LLThread::~LLThread() exiting thread before clean exit!" << llendl; // Put a stake in its heart. apr_thread_exit(mAPRThreadp, -1); return; @@ -283,7 +304,7 @@ void LLThread::wakeLocked() //============================================================================ LLMutex::LLMutex(apr_pool_t *poolp) : - mAPRMutexp(NULL) + mAPRMutexp(NULL), mCount(0), mLockingThread(NO_THREAD) { //if (poolp) //{ @@ -302,7 +323,8 @@ LLMutex::LLMutex(apr_pool_t *poolp) : LLMutex::~LLMutex() { #if MUTEX_DEBUG - llassert_always(!isLocked()); // better not be locked! + //bad assertion, the subclass LLSignal might be "locked", and that's OK + //llassert_always(!isLocked()); // better not be locked! #endif apr_thread_mutex_destroy(mAPRMutexp); mAPRMutexp = NULL; @@ -315,7 +337,14 @@ LLMutex::~LLMutex() void LLMutex::lock() { + if(isSelfLocked()) + { //redundant lock + mCount++; + return; + } + apr_thread_mutex_lock(mAPRMutexp); + #if MUTEX_DEBUG // Have to have the lock before we can access the debug info U32 id = LLThread::currentID(); @@ -323,10 +352,22 @@ void LLMutex::lock() llerrs << "Already locked in Thread: " << id << llendl; mIsLocked[id] = TRUE; #endif + +#if LL_DARWIN + mLockingThread = LLThread::currentID(); +#else + mLockingThread = sThreadID; +#endif } void LLMutex::unlock() { + if (mCount > 0) + { //not the root unlock + mCount--; + return; + } + #if MUTEX_DEBUG // Access the debug info while we have the lock U32 id = LLThread::currentID(); @@ -334,6 +375,8 @@ void LLMutex::unlock() llerrs << "Not locked in Thread: " << id << llendl; mIsLocked[id] = FALSE; #endif + + mLockingThread = NO_THREAD; apr_thread_mutex_unlock(mAPRMutexp); } @@ -351,6 +394,20 @@ bool LLMutex::isLocked() } } +bool LLMutex::isSelfLocked() +{ +#if LL_DARWIN + return mLockingThread == LLThread::currentID(); +#else + return mLockingThread == sThreadID; +#endif +} + +U32 LLMutex::lockingThread() const +{ + return mLockingThread; +} + //============================================================================ LLCondition::LLCondition(apr_pool_t *poolp) : @@ -371,6 +428,15 @@ LLCondition::~LLCondition() void LLCondition::wait() { + if (!isLocked()) + { //mAPRMutexp MUST be locked before calling apr_thread_cond_wait + apr_thread_mutex_lock(mAPRMutexp); +#if MUTEX_DEBUG + // avoid asserts on destruction in non-release builds + U32 id = LLThread::currentID(); + mIsLocked[id] = TRUE; +#endif + } apr_thread_cond_wait(mAPRCondp, mAPRMutexp); } diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index f1c6cd75af..b52e70ab2e 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -35,8 +35,17 @@ class LLThread; class LLMutex; class LLCondition; +#if LL_WINDOWS +#define ll_thread_local __declspec(thread) +#else +#define ll_thread_local __thread +#endif + class LL_COMMON_API LLThread { +private: + static U32 sIDIter; + public: typedef enum e_thread_status { @@ -77,6 +86,8 @@ public: apr_pool_t *getAPRPool() { return mAPRPoolp; } LLVolatileAPRPool* getLocalAPRFilePool() { return mLocalAPRFilePoolp ; } + U32 getID() const { return mID; } + private: BOOL mPaused; @@ -91,6 +102,7 @@ protected: apr_pool_t *mAPRPoolp; BOOL mIsLocalPool; EThreadStatus mStatus; + U32 mID; //a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used. //Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes. @@ -128,17 +140,28 @@ protected: class LL_COMMON_API LLMutex { public: + typedef enum + { + NO_THREAD = 0xFFFFFFFF + } e_locking_thread; + LLMutex(apr_pool_t *apr_poolp); // NULL pool constructs a new pool for the mutex virtual ~LLMutex(); void lock(); // blocks void unlock(); bool isLocked(); // non-blocking, but does do a lock/unlock so not free + bool isSelfLocked(); //return true if locked in a same thread + U32 lockingThread() const; //get ID of locking thread protected: apr_thread_mutex_t *mAPRMutexp; + mutable U32 mCount; + mutable U32 mLockingThread; + apr_pool_t *mAPRPoolp; BOOL mIsLocalPool; + #if MUTEX_DEBUG std::map<U32, BOOL> mIsLocked; #endif @@ -165,11 +188,14 @@ public: LLMutexLock(LLMutex* mutex) { mMutex = mutex; - mMutex->lock(); + + if(mMutex) + mMutex->lock(); } ~LLMutexLock() { - mMutex->unlock(); + if(mMutex) + mMutex->unlock(); } private: LLMutex* mMutex; diff --git a/indra/llcommon/lltimer.cpp b/indra/llcommon/lltimer.cpp index 38054b636e..9ebc6de7f4 100644 --- a/indra/llcommon/lltimer.cpp +++ b/indra/llcommon/lltimer.cpp @@ -83,7 +83,7 @@ U32 micro_sleep(U64 us, U32 max_yields) { // max_yields is unused; just fiddle with it to avoid warnings. max_yields = 0; - ms_sleep(us / 1000); + ms_sleep((U32)(us / 1000)); return 0; } #elif LL_LINUX || LL_SOLARIS || LL_DARWIN diff --git a/indra/llcommon/llversionviewer.h b/indra/llcommon/llversionviewer.h index d6fa5b1997..fafc750690 100644 --- a/indra/llcommon/llversionviewer.h +++ b/indra/llcommon/llversionviewer.h @@ -27,15 +27,15 @@ #ifndef LL_LLVERSIONVIEWER_H #define LL_LLVERSIONVIEWER_H -const S32 LL_VERSION_MAJOR = 2; -const S32 LL_VERSION_MINOR = 5; -const S32 LL_VERSION_PATCH = 0; +const S32 LL_VERSION_MAJOR = 3; +const S32 LL_VERSION_MINOR = 3; +const S32 LL_VERSION_PATCH = 2; const S32 LL_VERSION_BUILD = 0; const char * const LL_CHANNEL = "Second Life Developer"; #if LL_DARWIN -const char * const LL_VERSION_BUNDLE_ID = "com.secondlife.snowglobe.viewer"; +const char * const LL_VERSION_BUNDLE_ID = "com.secondlife.indra.viewer"; #endif #endif diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index 3ac50832fd..3d05a30ac2 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -34,8 +34,8 @@ //============================================================================ // Run on MAIN thread -LLWorkerThread::LLWorkerThread(const std::string& name, bool threaded) : - LLQueuedThread(name, threaded) +LLWorkerThread::LLWorkerThread(const std::string& name, bool threaded, bool should_pause) : + LLQueuedThread(name, threaded, should_pause) { mDeleteMutex = new LLMutex(NULL); @@ -81,7 +81,7 @@ void LLWorkerThread::clearDeleteList() } // virtual -S32 LLWorkerThread::update(U32 max_time_ms) +S32 LLWorkerThread::update(F32 max_time_ms) { S32 res = LLQueuedThread::update(max_time_ms); // Delete scheduled workers diff --git a/indra/llcommon/llworkerthread.h b/indra/llcommon/llworkerthread.h index 9bff18303e..be46394d6e 100644 --- a/indra/llcommon/llworkerthread.h +++ b/indra/llcommon/llworkerthread.h @@ -83,10 +83,10 @@ private: LLMutex* mDeleteMutex; public: - LLWorkerThread(const std::string& name, bool threaded = true); + LLWorkerThread(const std::string& name, bool threaded = true, bool should_pause = false); ~LLWorkerThread(); - /*virtual*/ S32 update(U32 max_time_ms); + /*virtual*/ S32 update(F32 max_time_ms); handle_t addWorkRequest(LLWorkerClass* workerclass, S32 param, U32 priority = PRIORITY_NORMAL); diff --git a/indra/llcommon/stdenums.h b/indra/llcommon/stdenums.h index 9f86de124e..40b3364b36 100644 --- a/indra/llcommon/stdenums.h +++ b/indra/llcommon/stdenums.h @@ -49,7 +49,9 @@ enum EDragAndDropType DAD_ANIMATION = 12, DAD_GESTURE = 13, DAD_LINK = 14, - DAD_COUNT = 15, // number of types in this enum + DAD_MESH = 15, + DAD_WIDGET = 16, + DAD_COUNT = 17, // number of types in this enum }; // Reasons for drags to be denied. diff --git a/indra/llcommon/tests/lldependencies_test.cpp b/indra/llcommon/tests/lldependencies_test.cpp index e40743ccf7..5395d785b6 100644 --- a/indra/llcommon/tests/lldependencies_test.cpp +++ b/indra/llcommon/tests/lldependencies_test.cpp @@ -258,10 +258,10 @@ namespace tut ++const_iterator; ensure_equals(const_iterator->first, "def"); ensure_equals(const_iterator->second, 2); - NameIndexDeps::node_range node_range(nideps.get_node_range()); - ensure_equals(instance_from_range<std::vector<int> >(node_range), make< std::vector<int> >(list_of(1)(2)(3))); - *node_range.begin() = 0; - *node_range.begin() = 1; +// NameIndexDeps::node_range node_range(nideps.get_node_range()); +// ensure_equals(instance_from_range<std::vector<int> >(node_range), make< std::vector<int> >(list_of(1)(2)(3))); +// *node_range.begin() = 0; +// *node_range.begin() = 1; NameIndexDeps::const_node_range const_node_range(const_nideps.get_node_range()); ensure_equals(instance_from_range<std::vector<int> >(const_node_range), make< std::vector<int> >(list_of(1)(2)(3))); NameIndexDeps::const_key_range const_key_range(const_nideps.get_key_range()); @@ -278,8 +278,8 @@ namespace tut def); ensure_equals(instance_from_range<StringList>(const_nideps.get_after_range(const_nideps.get_range().begin())), def); - ensure_equals(instance_from_range<StringList>(nideps.get_after_range(nideps.get_node_range().begin())), - def); +// ensure_equals(instance_from_range<StringList>(nideps.get_after_range(nideps.get_node_range().begin())), +// def); ensure_equals(instance_from_range<StringList>(const_nideps.get_after_range(const_nideps.get_node_range().begin())), def); ensure_equals(instance_from_range<StringList>(nideps.get_after_range(nideps.get_key_range().begin())), diff --git a/indra/llcommon/tests/llerror_test.cpp b/indra/llcommon/tests/llerror_test.cpp index 1ef8fc9712..09a20231de 100644 --- a/indra/llcommon/tests/llerror_test.cpp +++ b/indra/llcommon/tests/llerror_test.cpp @@ -48,7 +48,10 @@ namespace { static bool fatalWasCalled; void fatalCall(const std::string&) { fatalWasCalled = true; } +} +namespace tut +{ class TestRecorder : public LLError::Recorder { public: @@ -56,7 +59,7 @@ namespace ~TestRecorder() { LLError::removeRecorder(this); } void recordMessage(LLError::ELevel level, - const std::string& message) + const std::string& message) { mMessages.push_back(message); } @@ -66,12 +69,12 @@ namespace void setWantsTime(bool t) { mWantsTime = t; } bool wantsTime() { return mWantsTime; } - + std::string message(int n) { std::ostringstream test_name; test_name << "testing message " << n << ", not enough messages"; - + tut::ensure(test_name.str(), n < countMessages()); return mMessages[n]; } @@ -82,10 +85,7 @@ namespace bool mWantsTime; }; -} - -namespace tut -{ + struct ErrorTestData { TestRecorder mRecorder; @@ -381,7 +381,7 @@ namespace } typedef std::string (*LogFromFunction)(bool); - void testLogName(TestRecorder& recorder, LogFromFunction f, + void testLogName(tut::TestRecorder& recorder, LogFromFunction f, const std::string& class_name = "") { recorder.clearMessages(); diff --git a/indra/llcommon/tests/lleventdispatcher_test.cpp b/indra/llcommon/tests/lleventdispatcher_test.cpp new file mode 100644 index 0000000000..263c9b171f --- /dev/null +++ b/indra/llcommon/tests/lleventdispatcher_test.cpp @@ -0,0 +1,1324 @@ +/** + * @file lleventdispatcher_test.cpp + * @author Nat Goodspeed + * @date 2011-01-20 + * @brief Test for lleventdispatcher. + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Copyright (c) 2011, Linden Research, Inc. + * $/LicenseInfo$ + */ + +// Precompiled header +#include "linden_common.h" +// associated header +#include "lleventdispatcher.h" +// STL headers +// std headers +// external library headers +// other Linden headers +#include "../test/lltut.h" +#include "llsd.h" +#include "llsdutil.h" +#include "stringize.h" +#include "tests/wrapllerrs.h" + +#include <map> +#include <string> +#include <stdexcept> + +#include <boost/bind.hpp> +#include <boost/function.hpp> +#include <boost/range.hpp> +#include <boost/foreach.hpp> +#define foreach BOOST_FOREACH + +#include <boost/lambda/lambda.hpp> + +#include <iostream> +#include <iomanip> + +using boost::lambda::constant; +using boost::lambda::constant_ref; +using boost::lambda::var; + +using namespace llsd; + +/***************************************************************************** +* Output control +*****************************************************************************/ +#ifdef DEBUG_ON +using std::cout; +#else +static std::ostringstream cout; +#endif + +/***************************************************************************** +* Example data, functions, classes +*****************************************************************************/ +// We don't need a whole lot of different arbitrary-params methods, just (no | +// (const LLSD&) | arbitrary) args (function | static method | non-static +// method), where 'arbitrary' is (every LLSD datatype + (const char*)). +// But we need to register each one under different names for the different +// registration styles. Don't forget LLEventDispatcher subclass methods(const +// LLSD&). + +// However, the number of target parameter conversions we want to try exceeds +// boost::fusion::invoke()'s supported parameter-list size. Break out two +// different lists. +#define NPARAMSa bool b, int i, float f, double d, const char* cp +#define NPARAMSb const std::string& s, const LLUUID& uuid, const LLDate& date, \ + const LLURI& uri, const std::vector<U8>& bin +#define NARGSa b, i, f, d, cp +#define NARGSb s, uuid, date, uri, bin + +// For some registration methods we need methods on a subclass of +// LLEventDispatcher. To simplify things, we'll use this Dispatcher subclass +// for all our testing, including testing its own methods. +class Dispatcher: public LLEventDispatcher +{ +public: + Dispatcher(const std::string& name, const std::string& key): + LLEventDispatcher(name, key) + {} + + // sensing member, mutable because we want to know when we've reached our + // const method too + mutable LLSD llsd; + + void method1(const LLSD& obj) { llsd = obj; } + void cmethod1(const LLSD& obj) const { llsd = obj; } +}; + +// sensing vars, captured in a struct to make it convenient to clear them +struct Vars +{ + LLSD llsd; + bool b; + int i; + float f; + double d; + // Capture param passed as char*. But merely storing a char* received from + // our caller, possibly the .c_str() from a concatenation expression, + // would be Bad: the pointer will be invalidated long before we can query + // it. We could allocate a new chunk of memory, copy the string data and + // point to that instead -- but hey, guess what, we already have a class + // that does that! + std::string cp; + std::string s; + LLUUID uuid; + LLDate date; + LLURI uri; + std::vector<U8> bin; + + Vars(): + // Only need to initialize the POD types, the rest should take care of + // default-constructing themselves. + b(false), + i(0), + f(0), + d(0) + {} + + // Detect any non-default values for convenient testing + LLSD inspect() const + { + LLSD result; + + if (llsd.isDefined()) + result["llsd"] = llsd; + if (b) + result["b"] = b; + if (i) + result["i"] = i; + if (f) + result["f"] = f; + if (d) + result["d"] = d; + if (! cp.empty()) + result["cp"] = cp; + if (! s.empty()) + result["s"] = s; + if (uuid != LLUUID()) + result["uuid"] = uuid; + if (date != LLDate()) + result["date"] = date; + if (uri != LLURI()) + result["uri"] = uri; + if (! bin.empty()) + result["bin"] = bin; + + return result; + } + + /*------------- no-args (non-const, const, static) methods -------------*/ + void method0() + { + cout << "method0()\n"; + i = 17; + } + + void cmethod0() const + { + cout << 'c'; + const_cast<Vars*>(this)->method0(); + } + + static void smethod0(); + + /*------------ Callable (non-const, const, static) methods -------------*/ + void method1(const LLSD& obj) + { + cout << "method1(" << obj << ")\n"; + llsd = obj; + } + + void cmethod1(const LLSD& obj) const + { + cout << 'c'; + const_cast<Vars*>(this)->method1(obj); + } + + static void smethod1(const LLSD& obj); + + /*-------- Arbitrary-params (non-const, const, static) methods ---------*/ + void methodna(NPARAMSa) + { + // Because our const char* param cp might be NULL, and because we + // intend to capture the value in a std::string, have to distinguish + // between the NULL value and any non-NULL value. Use a convention + // easy for a human reader: enclose any non-NULL value in single + // quotes, reserving the unquoted string "NULL" to represent a NULL ptr. + std::string vcp; + if (cp == NULL) + vcp = "NULL"; + else + vcp = std::string("'") + cp + "'"; + + cout << "methodna(" << b + << ", " << i + << ", " << f + << ", " << d + << ", " << vcp + << ")\n"; + + this->b = b; + this->i = i; + this->f = f; + this->d = d; + this->cp = vcp; + } + + void methodnb(NPARAMSb) + { + std::ostringstream vbin; + foreach(U8 byte, bin) + { + vbin << std::hex << std::setfill('0') << std::setw(2) << unsigned(byte); + } + + cout << "methodnb(" << "'" << s << "'" + << ", " << uuid + << ", " << date + << ", '" << uri << "'" + << ", " << vbin.str() + << ")\n"; + + this->s = s; + this->uuid = uuid; + this->date = date; + this->uri = uri; + this->bin = bin; + } + + void cmethodna(NPARAMSa) const + { + cout << 'c'; + const_cast<Vars*>(this)->methodna(NARGSa); + } + + void cmethodnb(NPARAMSb) const + { + cout << 'c'; + const_cast<Vars*>(this)->methodnb(NARGSb); + } + + static void smethodna(NPARAMSa); + static void smethodnb(NPARAMSb); +}; +/*------- Global Vars instance for free functions and static methods -------*/ +static Vars g; + +/*------------ Static Vars method implementations reference 'g' ------------*/ +void Vars::smethod0() +{ + cout << "smethod0() -> "; + g.method0(); +} + +void Vars::smethod1(const LLSD& obj) +{ + cout << "smethod1(" << obj << ") -> "; + g.method1(obj); +} + +void Vars::smethodna(NPARAMSa) +{ + cout << "smethodna(...) -> "; + g.methodna(NARGSa); +} + +void Vars::smethodnb(NPARAMSb) +{ + cout << "smethodnb(...) -> "; + g.methodnb(NARGSb); +} + +/*--------------------------- Reset global Vars ----------------------------*/ +void clear() +{ + g = Vars(); +} + +/*------------------- Free functions also reference 'g' --------------------*/ +void free0() +{ + cout << "free0() -> "; + g.method0(); +} + +void free1(const LLSD& obj) +{ + cout << "free1(" << obj << ") -> "; + g.method1(obj); +} + +void freena(NPARAMSa) +{ + cout << "freena(...) -> "; + g.methodna(NARGSa); +} + +void freenb(NPARAMSb) +{ + cout << "freenb(...) -> "; + g.methodnb(NARGSb); +} + +/***************************************************************************** +* TUT +*****************************************************************************/ +namespace tut +{ + struct lleventdispatcher_data + { + WrapLL_ERRS redirect; + Dispatcher work; + Vars v; + std::string name, desc; + // Capture our own copy of all registered functions' descriptions + typedef std::map<std::string, std::string> DescMap; + DescMap descs; + // Capture the Vars instance on which we expect each function to operate + typedef std::map<std::string, Vars*> VarsMap; + VarsMap funcvars; + // Required structure for Callables with requirements + LLSD required; + // Parameter names for freena(), freenb() + LLSD params; + // Full, partial defaults arrays for params for freena(), freenb() + LLSD dft_array_full, dft_array_partial; + // Start index of partial defaults arrays + const LLSD::Integer partial_offset; + // Full, partial defaults maps for params for freena(), freenb() + LLSD dft_map_full, dft_map_partial; + // Most of the above are indexed by "a" or "b". Useful to have an + // array containing those strings for iterating. + std::vector<LLSD::String> ab; + + lleventdispatcher_data(): + work("test dispatcher", "op"), + // map {d=double, array=[3 elements]} + required(LLSDMap("d", LLSD::Real(0))("array", LLSDArray(LLSD())(LLSD())(LLSD()))), + // first several params are required, last couple optional + partial_offset(3) + { + // This object is reconstructed for every test<n> method. But + // clear global variables every time too. + ::clear(); + + const char* abs[] = { "a", "b" }; + ab.assign(boost::begin(abs), boost::end(abs)); + + // Registration cases: + // - (Callable | subclass const method | subclass non-const method | + // non-subclass method) (with | without) required + // - (Free function | static method | non-static method), (no | arbitrary) params, + // array style + // - (Free function | static method | non-static method), (no | arbitrary) params, + // map style, (empty | partial | full) (array | map) defaults + // - Map-style errors: + // - (scalar | map) param names + // - defaults scalar + // - defaults array longer than params array + // - defaults map with plural unknown param names + + // I hate to have to write things twice, because of having to keep + // them consistent. If we had variadic functions, addf() would be + // a variadic method, capturing the name and desc and passing them + // plus "everything else" to work.add(). If I could return a pair + // and use that pair as the first two args to work.add(), I'd do + // that. But the best I can do with present C++ is to set two + // instance variables as a side effect of addf(), and pass those + // variables to each work.add() call. :-P + + /*------------------------- Callables --------------------------*/ + + // Arbitrary Callable with/out required params + addf("free1", "free1", &g); + work.add(name, desc, free1); + addf("free1_req", "free1", &g); + work.add(name, desc, free1, required); + // Subclass non-const method with/out required params + addf("Dmethod1", "method1", NULL); + work.add(name, desc, &Dispatcher::method1); + addf("Dmethod1_req", "method1", NULL); + work.add(name, desc, &Dispatcher::method1, required); + // Subclass const method with/out required params + addf("Dcmethod1", "cmethod1", NULL); + work.add(name, desc, &Dispatcher::cmethod1); + addf("Dcmethod1_req", "cmethod1", NULL); + work.add(name, desc, &Dispatcher::cmethod1, required); + // Non-subclass method with/out required params + addf("method1", "method1", &v); + work.add(name, desc, boost::bind(&Vars::method1, boost::ref(v), _1)); + addf("method1_req", "method1", &v); + work.add(name, desc, boost::bind(&Vars::method1, boost::ref(v), _1), required); + + /*--------------- Arbitrary params, array style ----------------*/ + + // (Free function | static method) with (no | arbitrary) params, array style + addf("free0_array", "free0", &g); + work.add(name, desc, free0); + addf("freena_array", "freena", &g); + work.add(name, desc, freena); + addf("freenb_array", "freenb", &g); + work.add(name, desc, freenb); + addf("smethod0_array", "smethod0", &g); + work.add(name, desc, &Vars::smethod0); + addf("smethodna_array", "smethodna", &g); + work.add(name, desc, &Vars::smethodna); + addf("smethodnb_array", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb); + // Non-static method with (no | arbitrary) params, array style + addf("method0_array", "method0", &v); + work.add(name, desc, &Vars::method0, boost::lambda::var(v)); + addf("methodna_array", "methodna", &v); + work.add(name, desc, &Vars::methodna, boost::lambda::var(v)); + addf("methodnb_array", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, boost::lambda::var(v)); + + /*---------------- Arbitrary params, map style -----------------*/ + + // We lay out each params list as an array, also each array of + // default values we'll register. We'll zip these into + // (param=value) maps. Why not define them as maps and just + // extract the keys and values to arrays? Because that wouldn't + // give us the right params-list order. + + // freena(), methodna(), cmethodna(), smethodna() all take same param list. + // Same for freenb() et al. + params = LLSDMap("a", LLSDArray("b")("i")("f")("d")("cp")) + ("b", LLSDArray("s")("uuid")("date")("uri")("bin")); + cout << "params:\n" << params << "\nparams[\"a\"]:\n" << params["a"] << "\nparams[\"b\"]:\n" << params["b"] << std::endl; + // default LLSD::Binary value + std::vector<U8> binary; + for (size_t ix = 0, h = 0xaa; ix < 6; ++ix, h += 0x11) + { + binary.push_back(h); + } + // Full defaults arrays. We actually don't care what the LLUUID or + // LLDate values are, as long as they're different from the + // LLUUID() and LLDate() default values so inspect() will report + // them. + dft_array_full = LLSDMap("a", LLSDArray(true)(17)(3.14)(123456.78)("classic")) + ("b", LLSDArray("string") + (LLUUID::generateNewID()) + (LLDate::now()) + (LLURI("http://www.ietf.org/rfc/rfc3986.txt")) + (binary)); + cout << "dft_array_full:\n" << dft_array_full << std::endl; + // Partial defaults arrays. + foreach(LLSD::String a, ab) + { + LLSD::Integer partition(std::min(partial_offset, dft_array_full[a].size())); + dft_array_partial[a] = + llsd_copy_array(dft_array_full[a].beginArray() + partition, + dft_array_full[a].endArray()); + } + cout << "dft_array_partial:\n" << dft_array_partial << std::endl; + + foreach(LLSD::String a, ab) + { + // Generate full defaults maps by zipping (params, dft_array_full). + dft_map_full[a] = zipmap(params[a], dft_array_full[a]); + + // Generate partial defaults map by zipping alternate entries from + // (params, dft_array_full). Part of the point of using map-style + // defaults is to allow any subset of the target function's + // parameters to be optional, not just the rightmost. + for (LLSD::Integer ix = 0, ixend = params[a].size(); ix < ixend; ix += 2) + { + dft_map_partial[a][params[a][ix].asString()] = dft_array_full[a][ix]; + } + } + cout << "dft_map_full:\n" << dft_map_full << "\ndft_map_partial:\n" << dft_map_partial << '\n'; + + // (Free function | static method) with (no | arbitrary) params, + // map style, no (empty array) defaults + addf("free0_map", "free0", &g); + work.add(name, desc, free0, LLSD::emptyArray()); + addf("smethod0_map", "smethod0", &g); + work.add(name, desc, &Vars::smethod0, LLSD::emptyArray()); + addf("freena_map_allreq", "freena", &g); + work.add(name, desc, freena, params["a"]); + addf("freenb_map_allreq", "freenb", &g); + work.add(name, desc, freenb, params["b"]); + addf("smethodna_map_allreq", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"]); + addf("smethodnb_map_allreq", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"]); + // Non-static method with (no | arbitrary) params, map style, no + // (empty array) defaults + addf("method0_map", "method0", &v); + work.add(name, desc, &Vars::method0, var(v), LLSD::emptyArray()); + addf("methodna_map_allreq", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"]); + addf("methodnb_map_allreq", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"]); + + // Except for the "more (array | map) defaults than params" error + // cases, tested separately below, the (partial | full)(array | + // map) defaults cases don't apply to no-params functions/methods. + // So eliminate free0, smethod0, method0 from the cases below. + + // (Free function | static method) with arbitrary params, map + // style, partial (array | map) defaults + addf("freena_map_leftreq", "freena", &g); + work.add(name, desc, freena, params["a"], dft_array_partial["a"]); + addf("freenb_map_leftreq", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_array_partial["b"]); + addf("smethodna_map_leftreq", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_array_partial["a"]); + addf("smethodnb_map_leftreq", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_array_partial["b"]); + addf("freena_map_skipreq", "freena", &g); + work.add(name, desc, freena, params["a"], dft_map_partial["a"]); + addf("freenb_map_skipreq", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_map_partial["b"]); + addf("smethodna_map_skipreq", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_map_partial["a"]); + addf("smethodnb_map_skipreq", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_map_partial["b"]); + // Non-static method with arbitrary params, map style, partial + // (array | map) defaults + addf("methodna_map_leftreq", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_array_partial["a"]); + addf("methodnb_map_leftreq", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_array_partial["b"]); + addf("methodna_map_skipreq", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_map_partial["a"]); + addf("methodnb_map_skipreq", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_map_partial["b"]); + + // (Free function | static method) with arbitrary params, map + // style, full (array | map) defaults + addf("freena_map_adft", "freena", &g); + work.add(name, desc, freena, params["a"], dft_array_full["a"]); + addf("freenb_map_adft", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_array_full["b"]); + addf("smethodna_map_adft", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_array_full["a"]); + addf("smethodnb_map_adft", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_array_full["b"]); + addf("freena_map_mdft", "freena", &g); + work.add(name, desc, freena, params["a"], dft_map_full["a"]); + addf("freenb_map_mdft", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_map_full["b"]); + addf("smethodna_map_mdft", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_map_full["a"]); + addf("smethodnb_map_mdft", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_map_full["b"]); + // Non-static method with arbitrary params, map style, full + // (array | map) defaults + addf("methodna_map_adft", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_array_full["a"]); + addf("methodnb_map_adft", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_array_full["b"]); + addf("methodna_map_mdft", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_map_full["a"]); + addf("methodnb_map_mdft", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_map_full["b"]); + + // All the above are expected to succeed, and are setup for the + // tests to follow. Registration error cases are exercised as + // tests rather than as test setup. + } + + void addf(const std::string& n, const std::string& d, Vars* v) + { + // This method is to capture in our own DescMap the name and + // description of every registered function, for metadata query + // testing. + descs[n] = d; + // Also capture the Vars instance on which each function should operate. + funcvars[n] = v; + // See constructor for rationale for setting these instance vars. + this->name = n; + this->desc = d; + } + + void verify_descs() + { + // Copy descs to a temp map of same type. + DescMap forgotten(descs.begin(), descs.end()); + // LLEventDispatcher intentionally provides only const_iterator: + // since dereferencing that iterator generates values on the fly, + // it's meaningless to have a modifiable iterator. But since our + // 'work' object isn't const, by default BOOST_FOREACH() wants to + // use non-const iterators. Persuade it to use the const_iterator. + foreach(LLEventDispatcher::NameDesc nd, const_cast<const Dispatcher&>(work)) + { + DescMap::iterator found = forgotten.find(nd.first); + ensure(STRINGIZE("LLEventDispatcher records function '" << nd.first + << "' we didn't enter"), + found != forgotten.end()); + ensure_equals(STRINGIZE("LLEventDispatcher desc '" << nd.second << + "' doesn't match what we entered: '" << found->second << "'"), + nd.second, found->second); + // found in our map the name from LLEventDispatcher, good, erase + // our map entry + forgotten.erase(found); + } + if (! forgotten.empty()) + { + std::ostringstream out; + out << "LLEventDispatcher failed to report"; + const char* delim = ": "; + foreach(const DescMap::value_type& fme, forgotten) + { + out << delim << fme.first; + delim = ", "; + } + ensure(out.str(), false); + } + } + + Vars* varsfor(const std::string& name) + { + VarsMap::const_iterator found = funcvars.find(name); + ensure(STRINGIZE("No Vars* for " << name), found != funcvars.end()); + ensure(STRINGIZE("NULL Vars* for " << name), found->second); + return found->second; + } + + void ensure_has(const std::string& outer, const std::string& inner) + { + ensure(STRINGIZE("'" << outer << "' does not contain '" << inner << "'").c_str(), + outer.find(inner) != std::string::npos); + } + + void call_exc(const std::string& func, const LLSD& args, const std::string& exc_frag) + { + std::string threw; + try + { + work(func, args); + } + catch (const std::runtime_error& e) + { + cout << "*** " << e.what() << '\n'; + threw = e.what(); + } + ensure_has(threw, exc_frag); + } + + LLSD getMetadata(const std::string& name) + { + LLSD meta(work.getMetadata(name)); + ensure(STRINGIZE("No metadata for " << name), meta.isDefined()); + return meta; + } + + // From two related LLSD arrays, e.g. a param-names array and a values + // array, zip them together into an LLSD map. + LLSD zipmap(const LLSD& keys, const LLSD& values) + { + LLSD map; + for (LLSD::Integer i = 0, iend = keys.size(); i < iend; ++i) + { + // Have to select asString() since you can index an LLSD + // object with either String or Integer. + map[keys[i].asString()] = values[i]; + } + return map; + } + + // If I call this ensure_equals(), it blocks visibility of all other + // ensure_equals() overloads. Normally I could say 'using + // baseclass::ensure_equals;' and fix that, but I don't know what the + // base class is! + void ensure_llsd(const std::string& msg, const LLSD& actual, const LLSD& expected, U32 bits) + { + std::ostringstream out; + if (! msg.empty()) + { + out << msg << ": "; + } + out << "expected " << expected << ", actual " << actual; + ensure(out.str(), llsd_equals(actual, expected, bits)); + } + + void ensure_llsd(const LLSD& actual, const LLSD& expected, U32 bits) + { + ensure_llsd("", actual, expected, bits); + } + }; + typedef test_group<lleventdispatcher_data> lleventdispatcher_group; + typedef lleventdispatcher_group::object object; + lleventdispatcher_group lleventdispatchergrp("lleventdispatcher"); + + // Call cases: + // - (try_call | call) (explicit name | event key) (real | bogus) name + // - Callable with args that (do | do not) match required + // - (Free function | non-static method), no args, (array | map) style + // - (Free function | non-static method), arbitrary args, + // (array style with (scalar | map) | map style with scalar) + // - (Free function | non-static method), arbitrary args, array style with + // array (too short | too long | just right) + // [trap LL_WARNS for too-long case?] + // - (Free function | non-static method), arbitrary args, map style with + // (array | map) (all | too many | holes (with | without) defaults) + // - const char* param gets ("" | NULL) + + // Query cases: + // - Iterate over all (with | without) remove() + // - getDispatchKey() + // - Callable style (with | without) required + // - (Free function | non-static method), array style, (no | arbitrary) params + // - (Free function | non-static method), map style, (no | arbitrary) params, + // (empty | full | partial (array | map)) defaults + + template<> template<> + void object::test<1>() + { + set_test_name("map-style registration with non-array params"); + // Pass "param names" as scalar or as map + LLSD attempts(LLSDArray(17)(LLSDMap("pi", 3.14)("two", 2))); + foreach(LLSD ae, inArray(attempts)) + { + std::string threw; + try + { + work.add("freena_err", "freena", freena, ae); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "must be an array"); + } + } + + template<> template<> + void object::test<2>() + { + set_test_name("map-style registration with badly-formed defaults"); + std::string threw; + try + { + work.add("freena_err", "freena", freena, LLSDArray("a")("b"), 17); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "must be a map or an array"); + } + + template<> template<> + void object::test<3>() + { + set_test_name("map-style registration with too many array defaults"); + std::string threw; + try + { + work.add("freena_err", "freena", freena, + LLSDArray("a")("b"), + LLSDArray(17)(0.9)("gack")); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "shorter than"); + } + + template<> template<> + void object::test<4>() + { + set_test_name("map-style registration with too many map defaults"); + std::string threw; + try + { + work.add("freena_err", "freena", freena, + LLSDArray("a")("b"), + LLSDMap("b", 17)("foo", 3.14)("bar", "sinister")); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "nonexistent params"); + ensure_has(threw, "foo"); + ensure_has(threw, "bar"); + } + + template<> template<> + void object::test<5>() + { + set_test_name("query all"); + verify_descs(); + } + + template<> template<> + void object::test<6>() + { + set_test_name("query all with remove()"); + ensure("remove('bogus') returned true", ! work.remove("bogus")); + ensure("remove('real') returned false", work.remove("free1")); + // Of course, remove that from 'descs' too... + descs.erase("free1"); + verify_descs(); + } + + template<> template<> + void object::test<7>() + { + set_test_name("getDispatchKey()"); + ensure_equals(work.getDispatchKey(), "op"); + } + + template<> template<> + void object::test<8>() + { + set_test_name("query Callables with/out required params"); + LLSD names(LLSDArray("free1")("Dmethod1")("Dcmethod1")("method1")); + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(metadata["desc"].asString(), descs[nm]); + ensure("should not have required structure", metadata["required"].isUndefined()); + ensure("should not have optional", metadata["optional"].isUndefined()); + + std::string name_req(nm.asString() + "_req"); + metadata = getMetadata(name_req); + ensure_equals(metadata["name"].asString(), name_req); + ensure_equals(metadata["desc"].asString(), descs[name_req]); + ensure_equals("required mismatch", required, metadata["required"]); + ensure("should not have optional", metadata["optional"].isUndefined()); + } + } + + template<> template<> + void object::test<9>() + { + set_test_name("query array-style functions/methods"); + // Associate each registered name with expected arity. + LLSD expected(LLSDArray + (LLSDArray + (0)(LLSDArray("free0_array")("smethod0_array")("method0_array"))) + (LLSDArray + (5)(LLSDArray("freena_array")("smethodna_array")("methodna_array"))) + (LLSDArray + (5)(LLSDArray("freenb_array")("smethodnb_array")("methodnb_array")))); + foreach(LLSD ae, inArray(expected)) + { + LLSD::Integer arity(ae[0].asInteger()); + LLSD names(ae[1]); + LLSD req(LLSD::emptyArray()); + if (arity) + req[arity - 1] = LLSD(); + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(metadata["desc"].asString(), descs[nm]); + ensure_equals(STRINGIZE("mismatched required for " << nm.asString()), + metadata["required"], req); + ensure("should not have optional", metadata["optional"].isUndefined()); + } + } + } + + template<> template<> + void object::test<10>() + { + set_test_name("query map-style no-params functions/methods"); + // - (Free function | non-static method), map style, no params (ergo + // no defaults) + LLSD names(LLSDArray("free0_map")("smethod0_map")("method0_map")); + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(metadata["desc"].asString(), descs[nm]); + ensure("should not have required", + (metadata["required"].isUndefined() || metadata["required"].size() == 0)); + ensure("should not have optional", metadata["optional"].isUndefined()); + } + } + + template<> template<> + void object::test<11>() + { + set_test_name("query map-style arbitrary-params functions/methods: " + "full array defaults vs. full map defaults"); + // With functions registered with no defaults ("_allreq" suffixes), + // there is of course no difference between array defaults and map + // defaults. (We don't even bother registering with LLSD::emptyArray() + // vs. LLSD::emptyMap().) With functions registered with all defaults, + // there should (!) be no difference beween array defaults and map + // defaults. Verify, so we can ignore the distinction for all other + // tests. + LLSD equivalences(LLSDArray + (LLSDArray("freena_map_adft")("freena_map_mdft")) + (LLSDArray("freenb_map_adft")("freenb_map_mdft")) + (LLSDArray("smethodna_map_adft")("smethodna_map_mdft")) + (LLSDArray("smethodnb_map_adft")("smethodnb_map_mdft")) + (LLSDArray("methodna_map_adft")("methodna_map_mdft")) + (LLSDArray("methodnb_map_adft")("methodnb_map_mdft"))); + foreach(LLSD eq, inArray(equivalences)) + { + LLSD adft(eq[0]); + LLSD mdft(eq[1]); + // We can't just compare the results of the two getMetadata() + // calls, because they contain ["name"], which are different. So + // capture them, verify that each ["name"] is as expected, then + // remove for comparing the rest. + LLSD ameta(getMetadata(adft)); + LLSD mmeta(getMetadata(mdft)); + ensure_equals("adft name", adft, ameta["name"]); + ensure_equals("mdft name", mdft, mmeta["name"]); + ameta.erase("name"); + mmeta.erase("name"); + ensure_equals(STRINGIZE("metadata for " << adft.asString() + << " vs. " << mdft.asString()), + ameta, mmeta); + } + } + + template<> template<> + void object::test<12>() + { + set_test_name("query map-style arbitrary-params functions/methods"); + // - (Free function | non-static method), map style, arbitrary params, + // (empty | full | partial (array | map)) defaults + + // Generate maps containing all parameter names for cases in which all + // params are required. Also maps containing left requirements for + // partial defaults arrays. Also defaults maps from defaults arrays. + LLSD allreq, leftreq, rightdft; + foreach(LLSD::String a, ab) + { + // The map in which all params are required uses params[a] as + // keys, with all isUndefined() as values. We can accomplish that + // by passing zipmap() an empty values array. + allreq[a] = zipmap(params[a], LLSD::emptyArray()); + // Same for leftreq, save that we use the subset of the params not + // supplied by dft_array_partial[a]. + LLSD::Integer partition(params[a].size() - dft_array_partial[a].size()); + leftreq[a] = zipmap(llsd_copy_array(params[a].beginArray(), + params[a].beginArray() + partition), + LLSD::emptyArray()); + // Generate map pairing dft_array_partial[a] values with their + // param names. + rightdft[a] = zipmap(llsd_copy_array(params[a].beginArray() + partition, + params[a].endArray()), + dft_array_partial[a]); + } + cout << "allreq:\n" << allreq << "\nleftreq:\n" << leftreq << "\nrightdft:\n" << rightdft << std::endl; + + // Generate maps containing parameter names not provided by the + // dft_map_partial maps. + LLSD skipreq(allreq); + foreach(LLSD::String a, ab) + { + foreach(const MapEntry& me, inMap(dft_map_partial[a])) + { + skipreq[a].erase(me.first); + } + } + cout << "skipreq:\n" << skipreq << std::endl; + + LLSD groups(LLSDArray // array of groups + + (LLSDArray // group + (LLSDArray("freena_map_allreq")("smethodna_map_allreq")("methodna_map_allreq")) + (LLSDArray(allreq["a"])(LLSD()))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_allreq")("smethodnb_map_allreq")("methodnb_map_allreq")) + (LLSDArray(allreq["b"])(LLSD()))) // required, optional + + (LLSDArray // group + (LLSDArray("freena_map_leftreq")("smethodna_map_leftreq")("methodna_map_leftreq")) + (LLSDArray(leftreq["a"])(rightdft["a"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_leftreq")("smethodnb_map_leftreq")("methodnb_map_leftreq")) + (LLSDArray(leftreq["b"])(rightdft["b"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freena_map_skipreq")("smethodna_map_skipreq")("methodna_map_skipreq")) + (LLSDArray(skipreq["a"])(dft_map_partial["a"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_skipreq")("smethodnb_map_skipreq")("methodnb_map_skipreq")) + (LLSDArray(skipreq["b"])(dft_map_partial["b"]))) // required, optional + + // We only need mention the full-map-defaults ("_mdft" suffix) + // registrations, having established their equivalence with the + // full-array-defaults ("_adft" suffix) registrations in another test. + (LLSDArray // group + (LLSDArray("freena_map_mdft")("smethodna_map_mdft")("methodna_map_mdft")) + (LLSDArray(LLSD::emptyMap())(dft_map_full["a"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_mdft")("smethodnb_map_mdft")("methodnb_map_mdft")) + (LLSDArray(LLSD::emptyMap())(dft_map_full["b"])))); // required, optional + + foreach(LLSD grp, inArray(groups)) + { + // Internal structure of each group in 'groups': + LLSD names(grp[0]); + LLSD required(grp[1][0]); + LLSD optional(grp[1][1]); + cout << "For " << names << ",\n" << "required:\n" << required << "\noptional:\n" << optional << std::endl; + + // Loop through 'names' + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(nm.asString(), metadata["desc"].asString(), descs[nm]); + ensure_equals(STRINGIZE(nm << " required mismatch"), + metadata["required"], required); + ensure_equals(STRINGIZE(nm << " optional mismatch"), + metadata["optional"], optional); + } + } + } + + template<> template<> + void object::test<13>() + { + set_test_name("try_call()"); + ensure("try_call(bogus name, LLSD()) returned true", ! work.try_call("freek", LLSD())); + ensure("try_call(bogus name) returned true", ! work.try_call(LLSDMap("op", "freek"))); + ensure("try_call(real name, LLSD()) returned false", work.try_call("free0_array", LLSD())); + ensure("try_call(real name) returned false", work.try_call(LLSDMap("op", "free0_map"))); + } + + template<> template<> + void object::test<14>() + { + set_test_name("call with bad name"); + call_exc("freek", LLSD(), "not found"); + // We don't have a comparable helper function for the one-arg + // operator() method, and it's not worth building one just for this + // case. Write it out. + std::string threw; + try + { + work(LLSDMap("op", "freek")); + } + catch (const std::runtime_error& e) + { + cout << "*** " << e.what() << "\n"; + threw = e.what(); + } + ensure_has(threw, "bad"); + ensure_has(threw, "op"); + ensure_has(threw, "freek"); + } + + template<> template<> + void object::test<15>() + { + set_test_name("call with event key"); + // We don't need a separate test for operator()(string, LLSD) with + // valid name, because all the rest of the tests exercise that case. + // The one we don't exercise elsewhere is operator()(LLSD) with valid + // name, so here it is. + work(LLSDMap("op", "free0_map")); + ensure_equals(g.i, 17); + } + + // Cannot be defined inside function body... remind me again why we use C++... :-P + struct CallablesTriple + { + std::string name, name_req; + LLSD& llsd; + }; + + template<> template<> + void object::test<16>() + { + set_test_name("call Callables"); + CallablesTriple tests[] = + { + { "free1", "free1_req", g.llsd }, + { "Dmethod1", "Dmethod1_req", work.llsd }, + { "Dcmethod1", "Dcmethod1_req", work.llsd }, + { "method1", "method1_req", v.llsd } + }; + // Arbitrary LLSD value that we should be able to pass to Callables + // without 'required', but should not be able to pass to Callables + // with 'required'. + LLSD answer(42); + // LLSD value matching 'required' according to llsd_matches() rules. + LLSD matching(LLSDMap("d", 3.14)("array", LLSDArray("answer")(true)(answer))); + // Okay, walk through 'tests'. + foreach(const CallablesTriple& tr, tests) + { + // Should be able to pass 'answer' to Callables registered + // without 'required'. + work(tr.name, answer); + ensure_equals("answer mismatch", tr.llsd, answer); + // Should NOT be able to pass 'answer' to Callables registered + // with 'required'. + call_exc(tr.name_req, answer, "bad request"); + // But SHOULD be able to pass 'matching' to Callables registered + // with 'required'. + work(tr.name_req, matching); + ensure_equals("matching mismatch", tr.llsd, matching); + } + } + + template<> template<> + void object::test<17>() + { + set_test_name("passing wrong args to (map | array)-style registrations"); + + // Pass scalar/map to array-style functions, scalar/array to map-style + // functions. As that validation happens well before we engage the + // argument magic, it seems pointless to repeat this with every + // variation: (free function | non-static method), (no | arbitrary) + // args. We should only need to engage it for one map-style + // registration and one array-style registration. + std::string array_exc("needs an args array"); + call_exc("free0_array", 17, array_exc); + call_exc("free0_array", LLSDMap("pi", 3.14), array_exc); + + std::string map_exc("needs a map"); + call_exc("free0_map", 17, map_exc); + // Passing an array to a map-style function works now! No longer an + // error case! +// call_exc("free0_map", LLSDArray("a")("b"), map_exc); + } + + template<> template<> + void object::test<18>() + { + set_test_name("call no-args functions"); + LLSD names(LLSDArray + ("free0_array")("free0_map") + ("smethod0_array")("smethod0_map") + ("method0_array")("method0_map")); + foreach(LLSD name, inArray(names)) + { + // Look up the Vars instance for this function. + Vars* vars(varsfor(name)); + // Both the global and stack Vars instances are automatically + // cleared at the start of each test<n> method. But since we're + // calling these things several different times in the same + // test<n> method, manually reset the Vars between each. + *vars = Vars(); + ensure_equals(vars->i, 0); + // call function with empty array (or LLSD(), should be equivalent) + work(name, LLSD()); + ensure_equals(vars->i, 17); + } + } + + // Break out this data because we use it in a couple different tests. + LLSD array_funcs(LLSDArray + (LLSDMap("a", "freena_array") ("b", "freenb_array")) + (LLSDMap("a", "smethodna_array")("b", "smethodnb_array")) + (LLSDMap("a", "methodna_array") ("b", "methodnb_array"))); + + template<> template<> + void object::test<19>() + { + set_test_name("call array-style functions with too-short arrays"); + // Could have two different too-short arrays, one for *na and one for + // *nb, but since they both take 5 params... + LLSD tooshort(LLSDArray("this")("array")("too")("short")); + foreach(const LLSD& funcsab, inArray(array_funcs)) + { + foreach(const llsd::MapEntry& e, inMap(funcsab)) + { + call_exc(e.second, tooshort, "requires more arguments"); + } + } + } + + template<> template<> + void object::test<20>() + { + set_test_name("call array-style functions with (just right | too long) arrays"); + std::vector<U8> binary; + for (size_t h(0x01), i(0); i < 5; h+= 0x22, ++i) + { + binary.push_back(h); + } + LLSD args(LLSDMap("a", LLSDArray(true)(17)(3.14)(123.456)("char*")) + ("b", LLSDArray("string") + (LLUUID("01234567-89ab-cdef-0123-456789abcdef")) + (LLDate("2011-02-03T15:07:00Z")) + (LLURI("http://secondlife.com")) + (binary))); + LLSD argsplus(args); + argsplus["a"].append("bogus"); + argsplus["b"].append("bogus"); + LLSD expect; + foreach(LLSD::String a, ab) + { + expect[a] = zipmap(params[a], args[a]); + } + // Adjust expect["a"]["cp"] for special Vars::cp treatment. + expect["a"]["cp"] = std::string("'") + expect["a"]["cp"].asString() + "'"; + cout << "expect: " << expect << '\n'; + + // Use substantially the same logic for args and argsplus + LLSD argsarrays(LLSDArray(args)(argsplus)); + // So i==0 selects 'args', i==1 selects argsplus + for (LLSD::Integer i(0), iend(argsarrays.size()); i < iend; ++i) + { + foreach(const LLSD& funcsab, inArray(array_funcs)) + { + foreach(LLSD::String a, ab) + { + // Reset the Vars instance before each call + Vars* vars(varsfor(funcsab[a])); + *vars = Vars(); + work(funcsab[a], argsarrays[i][a]); + ensure_llsd(STRINGIZE(funcsab[a].asString() << + ": expect[\"" << a << "\"] mismatch"), + vars->inspect(), expect[a], 7); // 7 bits ~= 2 decimal digits + + // TODO: in the i==1 or argsplus case, intercept LL_WARNS + // output? Even without that, using argsplus verifies that + // passing too many args isn't fatal; it works -- but + // would be nice to notice the warning too. + } + } + } + } + + template<> template<> + void object::test<21>() + { + set_test_name("verify that passing LLSD() to const char* sends NULL"); + + ensure_equals("Vars::cp init", v.cp, ""); + work("methodna_map_mdft", LLSDMap("cp", LLSD())); + ensure_equals("passing LLSD()", v.cp, "NULL"); + work("methodna_map_mdft", LLSDMap("cp", "")); + ensure_equals("passing \"\"", v.cp, "''"); + work("methodna_map_mdft", LLSDMap("cp", "non-NULL")); + ensure_equals("passing \"non-NULL\"", v.cp, "'non-NULL'"); + } + + template<> template<> + void object::test<22>() + { + set_test_name("call map-style functions with (full | oversized) (arrays | maps)"); + const char binary[] = "\x99\x88\x77\x66\x55"; + LLSD array_full(LLSDMap + ("a", LLSDArray(false)(255)(98.6)(1024.5)("pointer")) + ("b", LLSDArray("object")(LLUUID::generateNewID())(LLDate::now())(LLURI("http://wiki.lindenlab.com/wiki"))(LLSD::Binary(boost::begin(binary), boost::end(binary))))); + LLSD array_overfull(array_full); + foreach(LLSD::String a, ab) + { + array_overfull[a].append("bogus"); + } + cout << "array_full: " << array_full << "\narray_overfull: " << array_overfull << std::endl; + // We rather hope that LLDate::now() will generate a timestamp + // distinct from the one it generated in the constructor, moments ago. + ensure_not_equals("Timestamps too close", + array_full["b"][2].asDate(), dft_array_full["b"][2].asDate()); + // We /insist/ that LLUUID::generateNewID() do so. + ensure_not_equals("UUID collision", + array_full["b"][1].asUUID(), dft_array_full["b"][1].asUUID()); + LLSD map_full, map_overfull; + foreach(LLSD::String a, ab) + { + map_full[a] = zipmap(params[a], array_full[a]); + map_overfull[a] = map_full[a]; + map_overfull[a]["extra"] = "ignore"; + } + cout << "map_full: " << map_full << "\nmap_overfull: " << map_overfull << std::endl; + LLSD expect(map_full); + // Twiddle the const char* param. + expect["a"]["cp"] = std::string("'") + expect["a"]["cp"].asString() + "'"; + // Another adjustment. For each data type, we're trying to distinguish + // three values: the Vars member's initial value (member wasn't + // stored; control never reached the set function), the registered + // default param value from dft_array_full, and the array_full value + // in this test. But bool can only distinguish two values. In this + // case, we want to differentiate the local array_full value from the + // dft_array_full value, so we use 'false'. However, that means + // Vars::inspect() doesn't differentiate it from the initial value, + // so won't bother returning it. Predict that behavior to match the + // LLSD values. + expect["a"].erase("b"); + cout << "expect: " << expect << std::endl; + // For this test, calling functions registered with different sets of + // parameter defaults should make NO DIFFERENCE WHATSOEVER. Every call + // should pass all params. + LLSD names(LLSDMap + ("a", LLSDArray + ("freena_map_allreq") ("smethodna_map_allreq") ("methodna_map_allreq") + ("freena_map_leftreq")("smethodna_map_leftreq")("methodna_map_leftreq") + ("freena_map_skipreq")("smethodna_map_skipreq")("methodna_map_skipreq") + ("freena_map_adft") ("smethodna_map_adft") ("methodna_map_adft") + ("freena_map_mdft") ("smethodna_map_mdft") ("methodna_map_mdft")) + ("b", LLSDArray + ("freenb_map_allreq") ("smethodnb_map_allreq") ("methodnb_map_allreq") + ("freenb_map_leftreq")("smethodnb_map_leftreq")("methodnb_map_leftreq") + ("freenb_map_skipreq")("smethodnb_map_skipreq")("methodnb_map_skipreq") + ("freenb_map_adft") ("smethodnb_map_adft") ("methodnb_map_adft") + ("freenb_map_mdft") ("smethodnb_map_mdft") ("methodnb_map_mdft"))); + // Treat (full | overfull) (array | map) the same. + LLSD argssets(LLSDArray(array_full)(array_overfull)(map_full)(map_overfull)); + foreach(const LLSD& args, inArray(argssets)) + { + foreach(LLSD::String a, ab) + { + foreach(LLSD::String name, inArray(names[a])) + { + // Reset the Vars instance + Vars* vars(varsfor(name)); + *vars = Vars(); + work(name, args[a]); + ensure_llsd(STRINGIZE(name << ": expect[\"" << a << "\"] mismatch"), + vars->inspect(), expect[a], 7); // 7 bits, 2 decimal digits + // intercept LL_WARNS for the two overfull cases? + } + } + } + } +} // namespace tut diff --git a/indra/llcommon/tests/llinstancetracker_test.cpp b/indra/llcommon/tests/llinstancetracker_test.cpp index c7cb488ca1..b34d1c5fd3 100644 --- a/indra/llcommon/tests/llinstancetracker_test.cpp +++ b/indra/llcommon/tests/llinstancetracker_test.cpp @@ -40,6 +40,7 @@ #include <boost/scoped_ptr.hpp> // other Linden headers #include "../test/lltut.h" +#include "wrapllerrs.h" struct Keyed: public LLInstanceTracker<Keyed, std::string> { @@ -151,33 +152,81 @@ namespace tut { Unkeyed one, two, three; typedef std::set<Unkeyed*> KeySet; - KeySet keys; - keys.insert(&one); - keys.insert(&two); - keys.insert(&three); - { - Unkeyed::LLInstanceTrackerScopedGuard guard; - for (Unkeyed::key_iter ki(guard.beginKeys()), kend(guard.endKeys()); - ki != kend; ++ki) - { - ensure_equals("spurious key", keys.erase(*ki), 1); - } - } - ensure_equals("unreported key", keys.size(), 0); - + KeySet instances; instances.insert(&one); instances.insert(&two); instances.insert(&three); - { - Unkeyed::LLInstanceTrackerScopedGuard guard; - for (Unkeyed::instance_iter ii(guard.beginInstances()), iend(guard.endInstances()); - ii != iend; ++ii) + + for (Unkeyed::instance_iter ii(Unkeyed::beginInstances()), iend(Unkeyed::endInstances()); ii != iend; ++ii) { Unkeyed& ref = *ii; ensure_equals("spurious instance", instances.erase(&ref), 1); } - } + ensure_equals("unreported instance", instances.size(), 0); } + + template<> template<> + void object::test<5>() + { + set_test_name("delete Keyed with outstanding instance_iter"); + std::string what; + Keyed* keyed = new Keyed("one"); + { + WrapLL_ERRS wrapper; + Keyed::instance_iter i(Keyed::beginInstances()); + try + { + delete keyed; + } + catch (const WrapLL_ERRS::FatalException& e) + { + what = e.what(); + } + } + ensure(! what.empty()); + } + + template<> template<> + void object::test<6>() + { + set_test_name("delete Keyed with outstanding key_iter"); + std::string what; + Keyed* keyed = new Keyed("one"); + { + WrapLL_ERRS wrapper; + Keyed::key_iter i(Keyed::beginKeys()); + try + { + delete keyed; + } + catch (const WrapLL_ERRS::FatalException& e) + { + what = e.what(); + } + } + ensure(! what.empty()); + } + + template<> template<> + void object::test<7>() + { + set_test_name("delete Unkeyed with outstanding instance_iter"); + std::string what; + Unkeyed* unkeyed = new Unkeyed; + { + WrapLL_ERRS wrapper; + Unkeyed::instance_iter i(Unkeyed::beginInstances()); + try + { + delete unkeyed; + } + catch (const WrapLL_ERRS::FatalException& e) + { + what = e.what(); + } + } + ensure(! what.empty()); + } } // namespace tut diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 770443da1d..72322c3b72 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -25,35 +25,293 @@ * $/LicenseInfo$ */ -#if !LL_WINDOWS + +#include "linden_common.h" + +#if LL_WINDOWS +#include <winsock2.h> +typedef U32 uint32_t; +#include <process.h> +#include <io.h> +#else +#include <unistd.h> #include <netinet/in.h> +#include <errno.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include "llprocesslauncher.h" #endif -#include "linden_common.h" +#include <sstream> + +/*==========================================================================*| +// Whoops, seems Linden's Boost package and the viewer are built with +// different settings of VC's /Zc:wchar_t switch! Using Boost.Filesystem +// pathname operations produces Windows link errors: +// unresolved external symbol "private: static class std::codecvt<unsigned short, +// char,int> const * & __cdecl boost::filesystem3::path::wchar_t_codecvt_facet()" +// unresolved external symbol "void __cdecl boost::filesystem3::path_traits::convert()" +// See: +// http://boost.2283326.n4.nabble.com/filesystem-v3-unicode-and-std-codecvt-linker-error-td3455549.html +// which points to: +// http://msdn.microsoft.com/en-us/library/dh8che7s%28v=VS.100%29.aspx + +// As we're not trying to preserve compatibility with old Boost.Filesystem +// code, but rather writing brand-new code, use the newest available +// Filesystem API. +#define BOOST_FILESYSTEM_VERSION 3 +#include "boost/filesystem.hpp" +#include "boost/filesystem/v3/fstream.hpp" +|*==========================================================================*/ +#include "boost/range.hpp" +#include "boost/foreach.hpp" +#include "boost/function.hpp" +#include "boost/lambda/lambda.hpp" +#include "boost/lambda/bind.hpp" +namespace lambda = boost::lambda; +/*==========================================================================*| +// Aaaarrgh, Linden's Boost package doesn't even include Boost.Iostreams! +#include "boost/iostreams/stream.hpp" +#include "boost/iostreams/device/file_descriptor.hpp" +|*==========================================================================*/ + #include "../llsd.h" #include "../llsdserialize.h" +#include "llsdutil.h" #include "../llformat.h" #include "../test/lltut.h" +#include "stringize.h" +std::vector<U8> string_to_vector(const std::string& str) +{ + return std::vector<U8>(str.begin(), str.end()); +} -#if LL_WINDOWS -#include <winsock2.h> -typedef U32 uint32_t; -#endif +#if ! LL_WINDOWS +// We want to call strerror_r(), but alarmingly, there are two different +// variants. The one that returns int always populates the passed buffer +// (except in case of error), whereas the other one always returns a valid +// char* but might or might not populate the passed buffer. How do we know +// which one we're getting? Define adapters for each and let the compiler +// select the applicable adapter. -std::vector<U8> string_to_vector(std::string str) +// strerror_r() returns char* +std::string message_from(int /*orig_errno*/, const char* /*buffer*/, const char* strerror_ret) { - // bc LLSD can't... - size_t len = (size_t)str.length(); - std::vector<U8> v(len); - for (size_t i = 0; i < len ; i++) - { - v[i] = str[i]; - } - return v; + return strerror_ret; } +// strerror_r() returns int +std::string message_from(int orig_errno, const char* buffer, int strerror_ret) +{ + if (strerror_ret == 0) + { + return buffer; + } + // Here strerror_r() has set errno. Since strerror_r() has already failed, + // seems like a poor bet to call it again to diagnose its own error... + int stre_errno = errno; + if (stre_errno == ERANGE) + { + return STRINGIZE("strerror_r() can't explain errno " << orig_errno + << " (buffer too small)"); + } + if (stre_errno == EINVAL) + { + return STRINGIZE("unknown errno " << orig_errno); + } + // Here we don't even understand the errno from strerror_r()! + return STRINGIZE("strerror_r() can't explain errno " << orig_errno + << " (error " << stre_errno << ')'); +} +#endif // ! LL_WINDOWS + +// boost::filesystem::temp_directory_path() isn't yet in Boost 1.45! :-( +std::string temp_directory_path() +{ +#if LL_WINDOWS + char buffer[4096]; + GetTempPathA(sizeof(buffer), buffer); + return buffer; + +#else // LL_DARWIN, LL_LINUX + static const char* vars[] = { "TMPDIR", "TMP", "TEMP", "TEMPDIR" }; + BOOST_FOREACH(const char* var, vars) + { + const char* found = getenv(var); + if (found) + return found; + } + return "/tmp"; +#endif // LL_DARWIN, LL_LINUX +} + +// Windows presents a kinda sorta compatibility layer. Code to the yucky +// Windows names because they're less likely than the Posix names to collide +// with any other names in this source. +#if LL_WINDOWS +#define _remove DeleteFileA +#else // ! LL_WINDOWS +#define _open open +#define _write write +#define _close close +#define _remove remove +#endif // ! LL_WINDOWS + +// Create a text file with specified content "somewhere in the +// filesystem," cleaning up when it goes out of scope. +class NamedTempFile +{ +public: + // Function that accepts an ostream ref and (presumably) writes stuff to + // it, e.g.: + // (lambda::_1 << "the value is " << 17 << '\n') + typedef boost::function<void(std::ostream&)> Streamer; + + NamedTempFile(const std::string& ext, const std::string& content): + mPath(temp_directory_path()) + { + createFile(ext, lambda::_1 << content); + } + + // Disambiguate when passing string literal + NamedTempFile(const std::string& ext, const char* content): + mPath(temp_directory_path()) + { + createFile(ext, lambda::_1 << content); + } + + NamedTempFile(const std::string& ext, const Streamer& func): + mPath(temp_directory_path()) + { + createFile(ext, func); + } + + ~NamedTempFile() + { + _remove(mPath.c_str()); + } + + std::string getName() const { return mPath; } + +private: + void createFile(const std::string& ext, const Streamer& func) + { + // Silly maybe, but use 'ext' as the name prefix. Strip off a leading + // '.' if present. + int pfx_offset = ((! ext.empty()) && ext[0] == '.')? 1 : 0; + +#if ! LL_WINDOWS + // Make sure mPath ends with a directory separator, if it doesn't already. + if (mPath.empty() || + ! (mPath[mPath.length() - 1] == '\\' || mPath[mPath.length() - 1] == '/')) + { + mPath.append("/"); + } + + // mkstemp() accepts and modifies a char* template string. Generate + // the template string, then copy to modifiable storage. + // mkstemp() requires its template string to end in six X's. + mPath += ext.substr(pfx_offset) + "XXXXXX"; + // Copy to vector<char> + std::vector<char> pathtemplate(mPath.begin(), mPath.end()); + // append a nul byte for classic-C semantics + pathtemplate.push_back('\0'); + // std::vector promises that a pointer to the 0th element is the same + // as a pointer to a contiguous classic-C array + int fd(mkstemp(&pathtemplate[0])); + if (fd == -1) + { + // The documented errno values (http://linux.die.net/man/3/mkstemp) + // are used in a somewhat unusual way, so provide context-specific + // errors. + if (errno == EEXIST) + { + LL_ERRS("NamedTempFile") << "mkstemp(\"" << mPath + << "\") could not create unique file " << LL_ENDL; + } + if (errno == EINVAL) + { + LL_ERRS("NamedTempFile") << "bad mkstemp() file path template '" + << mPath << "'" << LL_ENDL; + } + // Shrug, something else + int mkst_errno = errno; + char buffer[256]; + LL_ERRS("NamedTempFile") << "mkstemp(\"" << mPath << "\") failed: " + << message_from(mkst_errno, buffer, + strerror_r(mkst_errno, buffer, sizeof(buffer))) + << LL_ENDL; + } + // mkstemp() seems to have worked! Capture the modified filename. + // Avoid the nul byte we appended. + mPath.assign(pathtemplate.begin(), (pathtemplate.end()-1)); + +/*==========================================================================*| + // Define an ostream on the open fd. Tell it to close fd on destruction. + boost::iostreams::stream<boost::iostreams::file_descriptor_sink> + out(fd, boost::iostreams::close_handle); +|*==========================================================================*/ + + // Write desired content. + std::ostringstream out; + // Stream stuff to it. + func(out); + + std::string data(out.str()); + int written(_write(fd, data.c_str(), data.length())); + int closed(_close(fd)); + llassert_always(written == data.length() && closed == 0); + +#else // LL_WINDOWS + // GetTempFileName() is documented to require a MAX_PATH buffer. + char tempname[MAX_PATH]; + // Use 'ext' as filename prefix, but skip leading '.' if any. + // The 0 param is very important: requests iterating until we get a + // unique name. + if (0 == GetTempFileNameA(mPath.c_str(), ext.c_str() + pfx_offset, 0, tempname)) + { + // I always have to look up this call... :-P + LPSTR msgptr; + FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + GetLastError(), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + LPSTR(&msgptr), // have to cast (char**) to (char*) + 0, NULL ); + LL_ERRS("NamedTempFile") << "GetTempFileName(\"" << mPath << "\", \"" + << (ext.c_str() + pfx_offset) << "\") failed: " + << msgptr << LL_ENDL; + LocalFree(msgptr); + } + // GetTempFileName() appears to have worked! Capture the actual + // filename. + mPath = tempname; + // Open the file and stream content to it. Destructor will close. + std::ofstream out(tempname); + func(out); + +#endif // LL_WINDOWS + } + + void peep() + { + std::cout << "File '" << mPath << "' contains:\n"; + std::ifstream reader(mPath.c_str()); + std::string line; + while (std::getline(reader, line)) + std::cout << line << '\n'; + std::cout << "---\n"; + } + + std::string mPath; +}; + namespace tut { struct sd_xml_data @@ -452,7 +710,7 @@ namespace tut checkRoundTrip(msg + " nested arrays", v); v = LLSD::emptyMap(); - fillmap(v, 10, 6); // 10^6 maps + fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } @@ -1494,5 +1752,223 @@ namespace tut ensureBinaryAndNotation("map", test); ensureBinaryAndXML("map", test); } -} + struct TestPythonCompatible + { + TestPythonCompatible(): + // Note the peculiar insertion of __FILE__ into this string. Since + // this script is being written into a platform-dependent temp + // directory, we can't locate indra/lib/python relative to + // Python's __file__. Use __FILE__ instead, navigating relative + // to this C++ source file. Use Python raw-string syntax so + // Windows pathname backslashes won't mislead Python's string + // scanner. + import_llsd("import os.path\n" + "import sys\n" + "sys.path.insert(0,\n" + " os.path.join(os.path.dirname(r'" __FILE__ "'),\n" + " os.pardir, os.pardir, 'lib', 'python'))\n" + "try:\n" + " from llbase import llsd\n" + "except ImportError:\n" + " from indra.base import llsd\n") + {} + ~TestPythonCompatible() {} + + std::string import_llsd; + + template <typename CONTENT> + void python(const std::string& desc, const CONTENT& script, int expect=0) + { + const char* PYTHON(getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", PYTHON); + + NamedTempFile scriptfile(".py", script); + +#if LL_WINDOWS + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON, qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) + { + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } + +#else // LL_DARWIN, LL_LINUX + LLProcessLauncher py; + py.setExecutable(PYTHON); + py.addArgument(scriptfile.getName()); + ensure_equals(STRINGIZE("Couldn't launch " << desc << " script"), py.launch(), 0); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py.getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) + { + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); + } + else if (WIFSIGNALED(status)) + { + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); + } + else + { + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); + } + } +#endif + } + }; + + typedef tut::test_group<TestPythonCompatible> TestPythonCompatibleGroup; + typedef TestPythonCompatibleGroup::object TestPythonCompatibleObject; + TestPythonCompatibleGroup pycompat("LLSD serialize Python compatibility"); + + template<> template<> + void TestPythonCompatibleObject::test<1>() + { + set_test_name("verify python()"); + python("hello", + "import sys\n" + "sys.exit(17)\n", + 17); // expect nonzero rc + } + + template<> template<> + void TestPythonCompatibleObject::test<2>() + { + set_test_name("verify NamedTempFile"); + python("platform", + "import sys\n" + "print 'Running on', sys.platform\n"); + } + + template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("verify sequence to Python"); + + LLSD cdata(LLSDArray(17)(3.14) + ("This string\n" + "has several\n" + "lines.")); + + const char pydata[] = + "def verify(iterable):\n" + " it = iter(iterable)\n" + " assert it.next() == 17\n" + " assert abs(it.next() - 3.14) < 0.01\n" + " assert it.next() == '''\\\n" + "This string\n" + "has several\n" + "lines.'''\n" + " try:\n" + " it.next()\n" + " except StopIteration:\n" + " pass\n" + " else:\n" + " assert False, 'Too many data items'\n"; + + // Create a something.llsd file containing 'data' serialized to + // notation. It's important to separate with newlines because Python's + // llsd module doesn't support parsing from a file stream, only from a + // string, so we have to know how much of the file to read into a + // string. + NamedTempFile file(".llsd", + // NamedTempFile's boost::function constructor + // takes a callable. To this callable it passes the + // std::ostream with which it's writing the + // NamedTempFile. This lambda-based expression + // first calls LLSD::Serialize() with that ostream, + // then streams a newline to it, etc. + (lambda::bind(LLSDSerialize::toNotation, cdata[0], lambda::_1), + lambda::_1 << '\n', + lambda::bind(LLSDSerialize::toNotation, cdata[1], lambda::_1), + lambda::_1 << '\n', + lambda::bind(LLSDSerialize::toNotation, cdata[2], lambda::_1), + lambda::_1 << '\n')); + + python("read C++ notation", + lambda::_1 << + import_llsd << + "def parse_each(iterable):\n" + " for item in iterable:\n" + " yield llsd.parse(item)\n" << + pydata << + // Don't forget raw-string syntax for Windows pathnames. + "verify(parse_each(open(r'" << file.getName() << "')))\n"); + } + + template<> template<> + void TestPythonCompatibleObject::test<4>() + { + set_test_name("verify sequence from Python"); + + // Create an empty data file. This is just a placeholder for our + // script to write into. Create it to establish a unique name that + // we know. + NamedTempFile file(".llsd", ""); + + python("write Python notation", + lambda::_1 << + "from __future__ import with_statement\n" << + import_llsd << + "DATA = [\n" + " 17,\n" + " 3.14,\n" + " '''\\\n" + "This string\n" + "has several\n" + "lines.''',\n" + "]\n" + // Don't forget raw-string syntax for Windows pathnames. + // N.B. Using 'print' implicitly adds newlines. + "with open(r'" << file.getName() << "', 'w') as f:\n" + " for item in DATA:\n" + " print >>f, llsd.format_notation(item)\n"); + + std::ifstream inf(file.getName().c_str()); + LLSD item; + // Notice that we're not doing anything special to parse out the + // newlines: LLSDSerialize::fromNotation ignores them. While it would + // seem they're not strictly necessary, going in this direction, we + // want to ensure that notation-separated-by-newlines works in both + // directions -- since in practice, a given file might be read by + // either language. + ensure_equals("Failed to read LLSD::Integer from Python", + LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), + 1); + ensure_equals(item.asInteger(), 17); + ensure_equals("Failed to read LLSD::Real from Python", + LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), + 1); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure_equals("Failed to read LLSD::String from Python", + LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), + 1); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } +} diff --git a/indra/llcommon/tests/llsingleton_test.cpp b/indra/llcommon/tests/llsingleton_test.cpp new file mode 100644 index 0000000000..385289aefe --- /dev/null +++ b/indra/llcommon/tests/llsingleton_test.cpp @@ -0,0 +1,76 @@ +/** + * @file llsingleton_test.cpp + * @date 2011-08-11 + * @brief Unit test for the LLSingleton class + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llsingleton.h" +#include "../test/lltut.h" + +namespace tut +{ + struct singleton + { + // We need a class created with the LLSingleton template to test with. + class LLSingletonTest: public LLSingleton<LLSingletonTest> + { + + }; + }; + + typedef test_group<singleton> singleton_t; + typedef singleton_t::object singleton_object_t; + tut::singleton_t tut_singleton("LLSingleton"); + + template<> template<> + void singleton_object_t::test<1>() + { + + } + template<> template<> + void singleton_object_t::test<2>() + { + LLSingletonTest* singleton_test = LLSingletonTest::getInstance(); + ensure(singleton_test); + } + template<> template<> + void singleton_object_t::test<3>() + { + //Construct the instance + LLSingletonTest::getInstance(); + ensure(LLSingletonTest::instanceExists()); + + //Delete the instance + LLSingletonTest::deleteSingleton(); + ensure(LLSingletonTest::destroyed()); + ensure(!LLSingletonTest::instanceExists()); + + //Construct it again. + LLSingletonTest* singleton_test = LLSingletonTest::getInstance(); + ensure(singleton_test); + ensure(LLSingletonTest::instanceExists()); + } +} diff --git a/indra/llcommon/tests/llstring_test.cpp b/indra/llcommon/tests/llstring_test.cpp index 304e91ed92..6a1cbf652a 100644 --- a/indra/llcommon/tests/llstring_test.cpp +++ b/indra/llcommon/tests/llstring_test.cpp @@ -624,6 +624,14 @@ namespace tut subcount = LLStringUtil::format(s, fmt_map); ensure_equals("LLStringUtil::format: Assorted Test2 result", s, "?Am I not a long string?short[A]bbbaaaba[A]"); ensure_equals("LLStringUtil::format: Assorted Test2 result count", 9, subcount); + + // Test on nested brackets + std::string srcs6 = "[[TRICK1]][[A]][[B]][[AAA]][[BBB]][[TRICK2]][[KEYLONGER]][[KEYSHORTER]]?[[DELETE]]"; + s = srcs6; + subcount = LLStringUtil::format(s, fmt_map); + ensure_equals("LLStringUtil::format: Assorted Test2 result", s, "[[A]][a][b][aaa][bbb][[A]][short][Am I not a long string?]?[]"); + ensure_equals("LLStringUtil::format: Assorted Test2 result count", 9, subcount); + // Test an assorted substitution std::string srcs8 = "foo[DELETE]bar?"; diff --git a/indra/llcommon/tests/setpython.py b/indra/llcommon/tests/setpython.py new file mode 100644 index 0000000000..df7b90428e --- /dev/null +++ b/indra/llcommon/tests/setpython.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +"""\ +@file setpython.py +@author Nat Goodspeed +@date 2011-07-13 +@brief Set PYTHON environment variable for tests that care. + +$LicenseInfo:firstyear=2011&license=viewerlgpl$ +Copyright (c) 2011, Linden Research, Inc. +$/LicenseInfo$ +""" + +import os +import sys +import subprocess + +if __name__ == "__main__": + os.environ["PYTHON"] = sys.executable + sys.exit(subprocess.call(sys.argv[1:])) |