diff options
Diffstat (limited to 'indra/llcommon')
59 files changed, 7322 insertions, 532 deletions
diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 9342a22d46..0a3eaec5c5 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -7,17 +7,16 @@ include(00-Common) include(LLCommon) include(Linking) include(Boost) -include(Pth) include(LLSharedLibs) include(GoogleBreakpad) include(GooglePerfTools) include(Copy3rdPartyLibs) +include(ZLIB) include_directories( ${EXPAT_INCLUDE_DIRS} ${LLCOMMON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS} - ${PTH_INCLUDE_DIRS} ) # add_executable(lltreeiterators lltreeiterators.cpp) @@ -62,6 +61,7 @@ set(llcommon_SOURCE_FILES llformat.cpp llframetimer.cpp llheartbeat.cpp + llinstancetracker.cpp llliveappconfig.cpp lllivefile.cpp lllog.cpp @@ -115,6 +115,7 @@ set(llcommon_HEADER_FILES indra_constants.h linden_common.h linked_lists.h + llaccountingcost.h llallocator.h llallocator_heap_profile.h llagentconstants.h @@ -266,6 +267,10 @@ if(LLCOMMON_LINK_SHARED) add_definitions(-fPIC) endif(WINDOWS) endif(NOT WORD_SIZE EQUAL 32) + if(WINDOWS) + # always generate llcommon.pdb, even for "Release" builds + set_target_properties(llcommon PROPERTIES LINK_FLAGS "/DEBUG") + endif(WINDOWS) ll_stage_sharedlib(llcommon) else(LLCOMMON_LINK_SHARED) add_library (llcommon ${llcommon_SOURCE_FILES}) @@ -281,10 +286,15 @@ target_link_libraries( ${WINDOWS_LIBRARIES} ${BOOST_PROGRAM_OPTIONS_LIBRARY} ${BOOST_REGEX_LIBRARY} - ${PTH_LIBRARIES} ${GOOGLE_PERFTOOLS_LIBRARIES} ) +if (DARWIN) + include(CMakeFindFrameworks) + find_library(CARBON_LIBRARY Carbon) + target_link_libraries(llcommon ${CARBON_LIBRARY}) +endif (DARWIN) + add_dependencies(llcommon stage_third_party_libs) if (LL_TESTS) @@ -307,12 +317,15 @@ if (LL_TESTS) LL_ADD_INTEGRATION_TEST(lllazy "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}") - LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}" + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/setpython.py") + LL_ADD_INTEGRATION_TEST(llsingleton "" "${test_libs}") LL_ADD_INTEGRATION_TEST(llstring "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lltreeiterators "" "${test_libs}") LL_ADD_INTEGRATION_TEST(lluri "" "${test_libs}") LL_ADD_INTEGRATION_TEST(reflection "" "${test_libs}") LL_ADD_INTEGRATION_TEST(stringize "" "${test_libs}") + LL_ADD_INTEGRATION_TEST(lleventdispatcher "" "${test_libs}") # *TODO - reenable these once tcmalloc libs no longer break the build. #ADD_BUILD_TEST(llallocator llcommon) diff --git a/indra/llcommon/indra_constants.h b/indra/llcommon/indra_constants.h index 95cb606240..0745696ef3 100644 --- a/indra/llcommon/indra_constants.h +++ b/indra/llcommon/indra_constants.h @@ -312,6 +312,14 @@ const F32 CHAT_SHOUT_RADIUS = 100.f; const F32 CHAT_MAX_RADIUS = CHAT_SHOUT_RADIUS; const F32 CHAT_MAX_RADIUS_BY_TWO = CHAT_MAX_RADIUS / 2.f; +// squared editions of the above for distance checks +const F32 CHAT_WHISPER_RADIUS_SQUARED = CHAT_WHISPER_RADIUS * CHAT_WHISPER_RADIUS; +const F32 CHAT_NORMAL_RADIUS_SQUARED = CHAT_NORMAL_RADIUS * CHAT_NORMAL_RADIUS; +const F32 CHAT_SHOUT_RADIUS_SQUARED = CHAT_SHOUT_RADIUS * CHAT_SHOUT_RADIUS; +const F32 CHAT_MAX_RADIUS_SQUARED = CHAT_SHOUT_RADIUS_SQUARED; +const F32 CHAT_MAX_RADIUS_BY_TWO_SQUARED = CHAT_MAX_RADIUS_BY_TWO * CHAT_MAX_RADIUS_BY_TWO; + + // this times above gives barely audible radius const F32 CHAT_BARELY_AUDIBLE_FACTOR = 2.0f; @@ -379,8 +387,6 @@ const S32 MAP_SIM_RETURN_NULL_SIMS = 0x00010000; const S32 MAP_SIM_PRELUDE = 0x00020000; // Crash reporter behavior -const char* const CRASH_SETTINGS_FILE = "settings_crash_behavior.xml"; -const char* const CRASH_BEHAVIOR_SETTING = "CrashSubmitBehavior"; const S32 CRASH_BEHAVIOR_ASK = 0; const S32 CRASH_BEHAVIOR_ALWAYS_SEND = 1; const S32 CRASH_BEHAVIOR_NEVER_SEND = 2; diff --git a/indra/llcommon/llaccountingcost.h b/indra/llcommon/llaccountingcost.h new file mode 100644 index 0000000000..0ef3b50c6d --- /dev/null +++ b/indra/llcommon/llaccountingcost.h @@ -0,0 +1,86 @@ +/** + * @file llaccountingcost.h + * @ + * + * $LicenseInfo:firstyear=2001&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_ACCOUNTINGQUOTA_H +#define LL_ACCOUNTINGQUOTA_H + +struct ParcelQuota +{ + ParcelQuota( F32 ownerRenderCost, F32 ownerPhysicsCost, F32 ownerNetworkCost, F32 ownerSimulationCost, + F32 groupRenderCost, F32 groupPhysicsCost, F32 groupNetworkCost, F32 groupSimulationCost, + F32 otherRenderCost, F32 otherPhysicsCost, F32 otherNetworkCost, F32 otherSimulationCost, + F32 tempRenderCost, F32 tempPhysicsCost, F32 tempNetworkCost, F32 tempSimulationCost, + F32 selectedRenderCost, F32 selectedPhysicsCost, F32 selectedNetworkCost, F32 selectedSimulationCost, + F32 parcelCapacity ) + : mOwnerRenderCost( ownerRenderCost ), mOwnerPhysicsCost( ownerPhysicsCost ) + , mOwnerNetworkCost( ownerNetworkCost ), mOwnerSimulationCost( ownerSimulationCost ) + , mGroupRenderCost( groupRenderCost ), mGroupPhysicsCost( groupPhysicsCost ) + , mGroupNetworkCost( groupNetworkCost ), mGroupSimulationCost( groupSimulationCost ) + , mOtherRenderCost( otherRenderCost ), mOtherPhysicsCost( otherPhysicsCost ) + , mOtherNetworkCost( otherNetworkCost ), mOtherSimulationCost( otherSimulationCost ) + , mTempRenderCost( tempRenderCost ), mTempPhysicsCost( tempPhysicsCost ) + , mTempNetworkCost( tempNetworkCost ), mTempSimulationCost( tempSimulationCost ) + , mSelectedRenderCost( tempRenderCost ), mSelectedPhysicsCost( tempPhysicsCost ) + , mSelectedNetworkCost( tempNetworkCost ), mSelectedSimulationCost( selectedSimulationCost ) + , mParcelCapacity( parcelCapacity ) + { + } + + ParcelQuota(){} + F32 mOwnerRenderCost, mOwnerPhysicsCost, mOwnerNetworkCost, mOwnerSimulationCost; + F32 mGroupRenderCost, mGroupPhysicsCost, mGroupNetworkCost, mGroupSimulationCost; + F32 mOtherRenderCost, mOtherPhysicsCost, mOtherNetworkCost, mOtherSimulationCost; + F32 mTempRenderCost, mTempPhysicsCost, mTempNetworkCost, mTempSimulationCost; + F32 mSelectedRenderCost, mSelectedPhysicsCost, mSelectedNetworkCost, mSelectedSimulationCost; + F32 mParcelCapacity; +}; + +//SelectionQuota atm does not require a id +struct SelectionCost +{ + SelectionCost( /*LLTransactionID transactionId, */ F32 physicsCost, F32 networkCost, F32 simulationCost ) + //: mTransactionId( transactionId) + : mPhysicsCost( physicsCost ) + , mNetworkCost( networkCost ) + , mSimulationCost( simulationCost ) + { + } + SelectionCost() + : mPhysicsCost( 0.0f ) + , mNetworkCost( 0.0f ) + , mSimulationCost( 0.0f ) + {} + + F32 mPhysicsCost, mNetworkCost, mSimulationCost; + //LLTransactionID mTransactionId; +}; + +typedef enum { Roots = 0 , Prims } eSelectionType; + +#endif + + + diff --git a/indra/llcommon/llapp.cpp b/indra/llcommon/llapp.cpp index 39daefd1ad..ed192a9975 100644 --- a/indra/llcommon/llapp.cpp +++ b/indra/llcommon/llapp.cpp @@ -24,6 +24,10 @@ * $/LicenseInfo$ */ +#include "linden_common.h" + +#include "llapp.h" + #include <cstdlib> #ifdef LL_DARWIN @@ -32,9 +36,6 @@ #include <sys/sysctl.h> #endif -#include "linden_common.h" -#include "llapp.h" - #include "llcommon.h" #include "llapr.h" #include "llerrorcontrol.h" diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index eb610f625a..145dddd543 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -93,8 +93,9 @@ LLAssetDictionary::LLAssetDictionary() addEntry(LLAssetType::AT_LINK, new AssetEntry("LINK", "link", "sym link", false, false, true)); addEntry(LLAssetType::AT_LINK_FOLDER, new AssetEntry("FOLDER_LINK", "link_f", "sym folder link", false, false, true)); + addEntry(LLAssetType::AT_MESH, new AssetEntry("MESH", "mesh", "mesh", false, false, false)); + addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, FALSE, FALSE, FALSE)); - addEntry(LLAssetType::AT_NONE, new AssetEntry("NONE", "-1", NULL, false, false, false)); }; // static diff --git a/indra/llcommon/llassettype.h b/indra/llcommon/llassettype.h index c5ff2364cc..74ccd00324 100644 --- a/indra/llcommon/llassettype.h +++ b/indra/llcommon/llassettype.h @@ -108,8 +108,10 @@ public: AT_LINK_FOLDER = 25, // Inventory folder link - - AT_COUNT = 26, + AT_MESH = 49, + // Mesh data in our proprietary SLM format + + AT_COUNT = 50, // +*********************************************************+ // | TO ADD AN ELEMENT TO THIS ENUM: | diff --git a/indra/llcommon/llavatarconstants.h b/indra/llcommon/llavatarconstants.h index 596b0643ef..f47f447b45 100644 --- a/indra/llcommon/llavatarconstants.h +++ b/indra/llcommon/llavatarconstants.h @@ -46,10 +46,10 @@ const U32 AVATAR_TRANSACTED = 0x1 << 3; // whether avatar has actively used p const U32 AVATAR_ONLINE = 0x1 << 4; // the online status of this avatar, if known. const U32 AVATAR_AGEVERIFIED = 0x1 << 5; // whether avatar has been age-verified -static const std::string VISIBILITY_DEFAULT("default"); -static const std::string VISIBILITY_HIDDEN("hidden"); -static const std::string VISIBILITY_VISIBLE("visible"); -static const std::string VISIBILITY_INVISIBLE("invisible"); +char const* const VISIBILITY_DEFAULT = "default"; +char const* const VISIBILITY_HIDDEN = "hidden"; +char const* const VISIBILITY_VISIBLE = "visible"; +char const* const VISIBILITY_INVISIBLE = "invisible"; #endif diff --git a/indra/llcommon/llavatarname.cpp b/indra/llcommon/llavatarname.cpp index ad1845d387..ba3dd6d6b4 100644 --- a/indra/llcommon/llavatarname.cpp +++ b/indra/llcommon/llavatarname.cpp @@ -90,14 +90,16 @@ void LLAvatarName::fromLLSD(const LLSD& sd) std::string LLAvatarName::getCompleteName() const { std::string name; - if (!mUsername.empty()) + if (mUsername.empty() || mIsDisplayNameDefault) + // If the display name feature is off + // OR this particular display name is defaulted (i.e. based on user name), + // then display only the easier to read instance of the person's name. { - name = mDisplayName + " (" + mUsername + ")"; + name = mDisplayName; } else { - // ...display names are off, legacy name is in mDisplayName - name = mDisplayName; + name = mDisplayName + " (" + mUsername + ")"; } return name; } diff --git a/indra/llcommon/llchat.h b/indra/llcommon/llchat.h index 87c2d6775b..f5b242fdfc 100644 --- a/indra/llcommon/llchat.h +++ b/indra/llcommon/llchat.h @@ -49,7 +49,8 @@ typedef enum e_chat_type CHAT_TYPE_STOP = 5, CHAT_TYPE_DEBUG_MSG = 6, CHAT_TYPE_REGION = 7, - CHAT_TYPE_OWNER = 8 + CHAT_TYPE_OWNER = 8, + CHAT_TYPE_DIRECT = 9 // From llRegionSayTo() } EChatType; typedef enum e_chat_audible_level diff --git a/indra/llcommon/lldefs.h b/indra/llcommon/lldefs.h index 6b38de6500..5a4b8325f4 100644 --- a/indra/llcommon/lldefs.h +++ b/indra/llcommon/lldefs.h @@ -236,5 +236,13 @@ inline LLDATATYPE llclampb(const LLDATATYPE& a) return llmin(llmax(a, (LLDATATYPE)0), (LLDATATYPE)255); } +template <class LLDATATYPE> +inline void llswap(LLDATATYPE& lhs, LLDATATYPE& rhs) +{ + LLDATATYPE tmp = lhs; + lhs = rhs; + rhs = tmp; +} + #endif // LL_LLDEFS_H diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index bb64152407..c35799bbb9 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -379,7 +379,7 @@ namespace { /* This pattern, of returning a reference to a static function variable, is to ensure that this global is constructed before - it is used, no matter what the global initializeation sequence + it is used, no matter what the global initialization sequence is. See C++ FAQ Lite, sections 10.12 through 10.14 */ diff --git a/indra/llcommon/llerror.h b/indra/llcommon/llerror.h index 4a42241c4f..b3e604f8e8 100644 --- a/indra/llcommon/llerror.h +++ b/indra/llcommon/llerror.h @@ -39,7 +39,7 @@ Information for most users: - Code can log messages with constuctions like this: + Code can log messages with constructions like this: LL_INFOS("StringTag") << "request to fizzbip agent " << agent_id << " denied due to timeout" << LL_ENDL; @@ -47,9 +47,9 @@ Messages can be logged to one of four increasing levels of concern, using one of four "streams": - LL_DEBUGS("StringTag") - debug messages that are normally supressed - LL_INFOS("StringTag") - informational messages that are normall shown - LL_WARNS("StringTag") - warning messages that singal a problem + LL_DEBUGS("StringTag") - debug messages that are normally suppressed + LL_INFOS("StringTag") - informational messages that are normal shown + LL_WARNS("StringTag") - warning messages that signal a problem LL_ERRS("StringTag") - error messages that are major, unrecoverable failures The later (LL_ERRS("StringTag")) automatically crashes the process after the message @@ -90,7 +90,7 @@ WARN: LLFoo::doSomething: called with a big value for i: 283 - Which messages are logged and which are supressed can be controled at run + Which messages are logged and which are suppressed can be controlled at run time from the live file logcontrol.xml based on function, class and/or source file. See etc/logcontrol-dev.xml for details. @@ -106,7 +106,7 @@ namespace LLError enum ELevel { LEVEL_ALL = 0, - // used to indicate that all messagess should be logged + // used to indicate that all messages should be logged LEVEL_DEBUG = 0, LEVEL_INFO = 1, @@ -220,7 +220,7 @@ namespace LLError // See top of file for example of how to use this typedef LLError::NoClassInfo _LL_CLASS_TO_LOG; - // Outside a class declartion, or in class without LOG_CLASS(), this + // Outside a class declaration, or in class without LOG_CLASS(), this // typedef causes the messages to not be associated with any class. diff --git a/indra/llcommon/lleventdispatcher.cpp b/indra/llcommon/lleventdispatcher.cpp index d6e820d793..5b6d4efbe9 100644 --- a/indra/llcommon/lleventdispatcher.cpp +++ b/indra/llcommon/lleventdispatcher.cpp @@ -41,6 +41,354 @@ #include "llevents.h" #include "llerror.h" #include "llsdutil.h" +#include "stringize.h" +#include <memory> // std::auto_ptr + +/***************************************************************************** +* LLSDArgsSource +*****************************************************************************/ +/** + * Store an LLSD array, producing its elements one at a time. Die with LL_ERRS + * if the consumer requests more elements than the array contains. + */ +class LL_COMMON_API LLSDArgsSource +{ +public: + LLSDArgsSource(const std::string function, const LLSD& args); + ~LLSDArgsSource(); + + LLSD next(); + + void done() const; + +private: + std::string _function; + LLSD _args; + LLSD::Integer _index; +}; + +LLSDArgsSource::LLSDArgsSource(const std::string function, const LLSD& args): + _function(function), + _args(args), + _index(0) +{ + if (! (_args.isUndefined() || _args.isArray())) + { + LL_ERRS("LLSDArgsSource") << _function << " needs an args array instead of " + << _args << LL_ENDL; + } +} + +LLSDArgsSource::~LLSDArgsSource() +{ + done(); +} + +LLSD LLSDArgsSource::next() +{ + if (_index >= _args.size()) + { + LL_ERRS("LLSDArgsSource") << _function << " requires more arguments than the " + << _args.size() << " provided: " << _args << LL_ENDL; + } + return _args[_index++]; +} + +void LLSDArgsSource::done() const +{ + if (_index < _args.size()) + { + LL_WARNS("LLSDArgsSource") << _function << " only consumed " << _index + << " of the " << _args.size() << " arguments provided: " + << _args << LL_ENDL; + } +} + +/***************************************************************************** +* LLSDArgsMapper +*****************************************************************************/ +/** + * From a formal parameters description and a map of arguments, construct an + * arguments array. + * + * That is, given: + * - an LLSD array of length n containing parameter-name strings, + * corresponding to the arguments of a function of interest + * - an LLSD collection specifying default parameter values, either: + * - an LLSD array of length m <= n, matching the rightmost m params, or + * - an LLSD map explicitly stating default name=value pairs + * - an LLSD map of parameter names and actual values for a particular + * function call + * construct an LLSD array of actual argument values for this function call. + * + * The parameter-names array and the defaults collection describe the function + * being called. The map might vary with every call, providing argument values + * for the described parameters. + * + * The array of parameter names must match the number of parameters expected + * by the function of interest. + * + * If you pass a map of default parameter values, it provides default values + * as you might expect. It is an error to specify a default value for a name + * not listed in the parameters array. + * + * If you pass an array of default parameter values, it is mapped to the + * rightmost m of the n parameter names. It is an error if the default-values + * array is longer than the parameter-names array. Consider the following + * parameter names: ["a", "b", "c", "d"]. + * + * - An empty array of default values (or an isUndefined() value) asserts that + * every one of the above parameter names is required. + * - An array of four default values [1, 2, 3, 4] asserts that every one of + * the above parameters is optional. If the current parameter map is empty, + * they will be passed to the function as [1, 2, 3, 4]. + * - An array of two default values [11, 12] asserts that parameters "a" and + * "b" are required, while "c" and "d" are optional, having default values + * "c"=11 and "d"=12. + * + * The arguments array is constructed as follows: + * + * - Arguments-map keys not found in the parameter-names array are ignored. + * - Entries from the map provide values for an improper subset of the + * parameters named in the parameter-names array. This results in a + * tentative values array with "holes." (size of map) + (number of holes) = + * (size of names array) + * - Holes are filled with the default values. + * - Any remaining holes constitute an error. + */ +class LL_COMMON_API LLSDArgsMapper +{ +public: + /// Accept description of function: function name, param names, param + /// default values + LLSDArgsMapper(const std::string& function, const LLSD& names, const LLSD& defaults); + + /// Given arguments map, return LLSD::Array of parameter values, or LL_ERRS. + LLSD map(const LLSD& argsmap) const; + +private: + static std::string formatlist(const LLSD&); + + // The function-name string is purely descriptive. We want error messages + // to be able to indicate which function's LLSDArgsMapper has the problem. + std::string _function; + // Store the names array pretty much as given. + LLSD _names; + // Though we're handed an array of name strings, it's more useful to us to + // store it as a map from name string to position index. Of course that's + // easy to generate from the incoming names array, but why do it more than + // once? + typedef std::map<LLSD::String, LLSD::Integer> IndexMap; + IndexMap _indexes; + // Generated array of default values, aligned with the array of param names. + LLSD _defaults; + // Indicate whether we have a default value for each param. + typedef std::vector<char> FilledVector; + FilledVector _has_dft; +}; + +LLSDArgsMapper::LLSDArgsMapper(const std::string& function, + const LLSD& names, const LLSD& defaults): + _function(function), + _names(names), + _has_dft(names.size()) +{ + if (! (_names.isUndefined() || _names.isArray())) + { + LL_ERRS("LLSDArgsMapper") << function << " names must be an array, not " << names << LL_ENDL; + } + LLSD::Integer nparams(_names.size()); + // From _names generate _indexes. + for (LLSD::Integer ni = 0, nend = _names.size(); ni < nend; ++ni) + { + _indexes[_names[ni]] = ni; + } + + // Presize _defaults() array so we don't have to resize it more than once. + // All entries are initialized to LLSD(); but since _has_dft is still all + // 0, they're all "holes" for now. + if (nparams) + { + _defaults[nparams - 1] = LLSD(); + } + + if (defaults.isUndefined() || defaults.isArray()) + { + LLSD::Integer ndefaults = defaults.size(); + // defaults is a (possibly empty) array. Right-align it with names. + if (ndefaults > nparams) + { + LL_ERRS("LLSDArgsMapper") << function << " names array " << names + << " shorter than defaults array " << defaults << LL_ENDL; + } + + // Offset by which we slide defaults array right to right-align with + // _names array + LLSD::Integer offset = nparams - ndefaults; + // Fill rightmost _defaults entries from defaults, and mark them as + // filled + for (LLSD::Integer i = 0, iend = ndefaults; i < iend; ++i) + { + _defaults[i + offset] = defaults[i]; + _has_dft[i + offset] = 1; + } + } + else if (defaults.isMap()) + { + // defaults is a map. Use it to populate the _defaults array. + LLSD bogus; + for (LLSD::map_const_iterator mi(defaults.beginMap()), mend(defaults.endMap()); + mi != mend; ++mi) + { + IndexMap::const_iterator ixit(_indexes.find(mi->first)); + if (ixit == _indexes.end()) + { + bogus.append(mi->first); + continue; + } + + LLSD::Integer pos = ixit->second; + // Store default value at that position in the _defaults array. + _defaults[pos] = mi->second; + // Don't forget to record the fact that we've filled this + // position. + _has_dft[pos] = 1; + } + if (bogus.size()) + { + LL_ERRS("LLSDArgsMapper") << function << " defaults specified for nonexistent params " + << formatlist(bogus) << LL_ENDL; + } + } + else + { + LL_ERRS("LLSDArgsMapper") << function << " defaults must be a map or an array, not " + << defaults << LL_ENDL; + } +} + +LLSD LLSDArgsMapper::map(const LLSD& argsmap) const +{ + if (! (argsmap.isUndefined() || argsmap.isMap() || argsmap.isArray())) + { + LL_ERRS("LLSDArgsMapper") << _function << " map() needs a map or array, not " + << argsmap << LL_ENDL; + } + // Initialize the args array. Indexing a non-const LLSD array grows it + // to appropriate size, but we don't want to resize this one on each + // new operation. Just make it as big as we need before we start + // stuffing values into it. + LLSD args(LLSD::emptyArray()); + if (_defaults.size() == 0) + { + // If this function requires no arguments, fast exit. (Don't try to + // assign to args[-1].) + return args; + } + args[_defaults.size() - 1] = LLSD(); + + // Get a vector of chars to indicate holes. It's tempting to just scan + // for LLSD::isUndefined() values after filling the args array from + // the map, but it's plausible for caller to explicitly pass + // isUndefined() as the value of some parameter name. That's legal + // since isUndefined() has well-defined conversions (default value) + // for LLSD data types. So use a whole separate array for detecting + // holes. (Avoid std::vector<bool> which is known to be odd -- can we + // iterate?) + FilledVector filled(args.size()); + + if (argsmap.isArray()) + { + // Fill args from array. If there are too many args in passed array, + // ignore the rest. + LLSD::Integer size(argsmap.size()); + if (size > args.size()) + { + // We don't just use std::min() because we want to sneak in this + // warning if caller passes too many args. + LL_WARNS("LLSDArgsMapper") << _function << " needs " << args.size() + << " params, ignoring last " << (size - args.size()) + << " of passed " << size << ": " << argsmap << LL_ENDL; + size = args.size(); + } + for (LLSD::Integer i(0); i < size; ++i) + { + // Copy the actual argument from argsmap + args[i] = argsmap[i]; + // Note that it's been filled + filled[i] = 1; + } + } + else + { + // argsmap is in fact a map. Walk the map. + for (LLSD::map_const_iterator mi(argsmap.beginMap()), mend(argsmap.endMap()); + mi != mend; ++mi) + { + // mi->first is a parameter-name string, with mi->second its + // value. Look up the name's position index in _indexes. + IndexMap::const_iterator ixit(_indexes.find(mi->first)); + if (ixit == _indexes.end()) + { + // Allow for a map containing more params than were passed in + // our names array. Caller typically receives a map containing + // the function name, cruft such as reqid, etc. Ignore keys + // not defined in _indexes. + LL_DEBUGS("LLSDArgsMapper") << _function << " ignoring " + << mi->first << "=" << mi->second << LL_ENDL; + continue; + } + LLSD::Integer pos = ixit->second; + // Store the value at that position in the args array. + args[pos] = mi->second; + // Don't forget to record the fact that we've filled this + // position. + filled[pos] = 1; + } + } + + // Fill any remaining holes from _defaults. + LLSD unfilled(LLSD::emptyArray()); + for (LLSD::Integer i = 0, iend = args.size(); i < iend; ++i) + { + if (! filled[i]) + { + // If there's no default value for this parameter, that's an + // error. + if (! _has_dft[i]) + { + unfilled.append(_names[i]); + } + else + { + args[i] = _defaults[i]; + } + } + } + // If any required args -- args without defaults -- were left unfilled + // by argsmap, that's a problem. + if (unfilled.size()) + { + LL_ERRS("LLSDArgsMapper") << _function << " missing required arguments " + << formatlist(unfilled) << " from " << argsmap << LL_ENDL; + } + + // done + return args; +} + +std::string LLSDArgsMapper::formatlist(const LLSD& list) +{ + std::ostringstream out; + const char* delim = ""; + for (LLSD::array_const_iterator li(list.beginArray()), lend(list.endArray()); + li != lend; ++li) + { + out << delim << li->asString(); + delim = ", "; + } + return out.str(); +} LLEventDispatcher::LLEventDispatcher(const std::string& desc, const std::string& key): mDesc(desc), @@ -52,12 +400,181 @@ LLEventDispatcher::~LLEventDispatcher() { } +/** + * DispatchEntry subclass used for callables accepting(const LLSD&) + */ +struct LLEventDispatcher::LLSDDispatchEntry: public LLEventDispatcher::DispatchEntry +{ + LLSDDispatchEntry(const std::string& desc, const Callable& func, const LLSD& required): + DispatchEntry(desc), + mFunc(func), + mRequired(required) + {} + + Callable mFunc; + LLSD mRequired; + + virtual void call(const std::string& desc, const LLSD& event) const + { + // Validate the syntax of the event itself. + std::string mismatch(llsd_matches(mRequired, event)); + if (! mismatch.empty()) + { + LL_ERRS("LLEventDispatcher") << desc << ": bad request: " << mismatch << LL_ENDL; + } + // Event syntax looks good, go for it! + mFunc(event); + } + + virtual LLSD addMetadata(LLSD meta) const + { + meta["required"] = mRequired; + return meta; + } +}; + +/** + * DispatchEntry subclass for passing LLSD to functions accepting + * arbitrary argument types (convertible via LLSDParam) + */ +struct LLEventDispatcher::ParamsDispatchEntry: public LLEventDispatcher::DispatchEntry +{ + ParamsDispatchEntry(const std::string& desc, const invoker_function& func): + DispatchEntry(desc), + mInvoker(func) + {} + + invoker_function mInvoker; + + virtual void call(const std::string& desc, const LLSD& event) const + { + LLSDArgsSource src(desc, event); + mInvoker(boost::bind(&LLSDArgsSource::next, boost::ref(src))); + } +}; + +/** + * DispatchEntry subclass for dispatching LLSD::Array to functions accepting + * arbitrary argument types (convertible via LLSDParam) + */ +struct LLEventDispatcher::ArrayParamsDispatchEntry: public LLEventDispatcher::ParamsDispatchEntry +{ + ArrayParamsDispatchEntry(const std::string& desc, const invoker_function& func, + LLSD::Integer arity): + ParamsDispatchEntry(desc, func), + mArity(arity) + {} + + LLSD::Integer mArity; + + virtual LLSD addMetadata(LLSD meta) const + { + LLSD array(LLSD::emptyArray()); + // Resize to number of arguments required + if (mArity) + array[mArity - 1] = LLSD(); + llassert_always(array.size() == mArity); + meta["required"] = array; + return meta; + } +}; + +/** + * DispatchEntry subclass for dispatching LLSD::Map to functions accepting + * arbitrary argument types (convertible via LLSDParam) + */ +struct LLEventDispatcher::MapParamsDispatchEntry: public LLEventDispatcher::ParamsDispatchEntry +{ + MapParamsDispatchEntry(const std::string& name, const std::string& desc, + const invoker_function& func, + const LLSD& params, const LLSD& defaults): + ParamsDispatchEntry(desc, func), + mMapper(name, params, defaults), + mRequired(LLSD::emptyMap()) + { + // Build the set of all param keys, then delete the ones that are + // optional. What's left are the ones that are required. + for (LLSD::array_const_iterator pi(params.beginArray()), pend(params.endArray()); + pi != pend; ++pi) + { + mRequired[pi->asString()] = LLSD(); + } + + if (defaults.isArray() || defaults.isUndefined()) + { + // Right-align the params and defaults arrays. + LLSD::Integer offset = params.size() - defaults.size(); + // Now the name of every defaults[i] is at params[i + offset]. + for (LLSD::Integer i(0), iend(defaults.size()); i < iend; ++i) + { + // Erase this optional param from mRequired. + mRequired.erase(params[i + offset].asString()); + // Instead, make an entry in mOptional with the default + // param's name and value. + mOptional[params[i + offset].asString()] = defaults[i]; + } + } + else if (defaults.isMap()) + { + // if defaults is already a map, then it's already in the form we + // intend to deliver in metadata + mOptional = defaults; + // Just delete from mRequired every key appearing in mOptional. + for (LLSD::map_const_iterator mi(mOptional.beginMap()), mend(mOptional.endMap()); + mi != mend; ++mi) + { + mRequired.erase(mi->first); + } + } + } + + LLSDArgsMapper mMapper; + LLSD mRequired; + LLSD mOptional; + + virtual void call(const std::string& desc, const LLSD& event) const + { + // Just convert from LLSD::Map to LLSD::Array using mMapper, then pass + // to base-class call() method. + ParamsDispatchEntry::call(desc, mMapper.map(event)); + } + + virtual LLSD addMetadata(LLSD meta) const + { + meta["required"] = mRequired; + meta["optional"] = mOptional; + return meta; + } +}; + +void LLEventDispatcher::addArrayParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + LLSD::Integer arity) +{ + mDispatch.insert( + DispatchMap::value_type(name, DispatchMap::mapped_type( + new ArrayParamsDispatchEntry(desc, invoker, arity)))); +} + +void LLEventDispatcher::addMapParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + const LLSD& params, + const LLSD& defaults) +{ + mDispatch.insert( + DispatchMap::value_type(name, DispatchMap::mapped_type( + new MapParamsDispatchEntry(name, desc, invoker, params, defaults)))); +} + /// Register a callable by name void LLEventDispatcher::add(const std::string& name, const std::string& desc, const Callable& callable, const LLSD& required) { - mDispatch.insert(DispatchMap::value_type(name, - DispatchMap::mapped_type(callable, desc, required))); + mDispatch.insert( + DispatchMap::value_type(name, DispatchMap::mapped_type( + new LLSDDispatchEntry(desc, callable, required)))); } void LLEventDispatcher::addFail(const std::string& name, const std::string& classname) const @@ -83,7 +600,7 @@ bool LLEventDispatcher::remove(const std::string& name) /// such callable exists, die with LL_ERRS. void LLEventDispatcher::operator()(const std::string& name, const LLSD& event) const { - if (! attemptCall(name, event)) + if (! try_call(name, event)) { LL_ERRS("LLEventDispatcher") << "LLEventDispatcher(" << mDesc << "): '" << name << "' not found" << LL_ENDL; @@ -98,44 +615,29 @@ void LLEventDispatcher::operator()(const LLSD& event) const // This could/should be implemented in terms of the two-arg overload. // However -- we can produce a more informative error message. std::string name(event[mKey]); - if (! attemptCall(name, event)) + if (! try_call(name, event)) { LL_ERRS("LLEventDispatcher") << "LLEventDispatcher(" << mDesc << "): bad " << mKey << " value '" << name << "'" << LL_ENDL; } } -bool LLEventDispatcher::attemptCall(const std::string& name, const LLSD& event) const +bool LLEventDispatcher::try_call(const LLSD& event) const { - DispatchMap::const_iterator found = mDispatch.find(name); - if (found == mDispatch.end()) - { - // The reason we only return false, leaving it up to our caller to die - // with LL_ERRS, is that different callers have different amounts of - // available information. - return false; - } - // Found the name, so it's plausible to even attempt the call. But first, - // validate the syntax of the event itself. - std::string mismatch(llsd_matches(found->second.mRequired, event)); - if (! mismatch.empty()) - { - LL_ERRS("LLEventDispatcher") << "LLEventDispatcher(" << mDesc << ") calling '" << name - << "': bad request: " << mismatch << LL_ENDL; - } - // Event syntax looks good, go for it! - (found->second.mFunc)(event); - return true; // tell caller we were able to call + return try_call(event[mKey], event); } -LLEventDispatcher::Callable LLEventDispatcher::get(const std::string& name) const +bool LLEventDispatcher::try_call(const std::string& name, const LLSD& event) const { DispatchMap::const_iterator found = mDispatch.find(name); if (found == mDispatch.end()) { - return Callable(); + return false; } - return found->second.mFunc; + // Found the name, so it's plausible to even attempt the call. + found->second->call(STRINGIZE("LLEventDispatcher(" << mDesc << ") calling '" << name << "'"), + event); + return true; // tell caller we were able to call } LLSD LLEventDispatcher::getMetadata(const std::string& name) const @@ -147,9 +649,8 @@ LLSD LLEventDispatcher::getMetadata(const std::string& name) const } LLSD meta; meta["name"] = name; - meta["desc"] = found->second.mDesc; - meta["required"] = found->second.mRequired; - return meta; + meta["desc"] = found->second->mDesc; + return found->second->addMetadata(meta); } LLDispatchListener::LLDispatchListener(const std::string& pumpname, const std::string& key): @@ -164,3 +665,8 @@ bool LLDispatchListener::process(const LLSD& event) (*this)(event); return false; } + +LLEventDispatcher::DispatchEntry::DispatchEntry(const std::string& desc): + mDesc(desc) +{} + diff --git a/indra/llcommon/lleventdispatcher.h b/indra/llcommon/lleventdispatcher.h index dfffd59eb6..7acc61de4e 100644 --- a/indra/llcommon/lleventdispatcher.h +++ b/indra/llcommon/lleventdispatcher.h @@ -27,18 +27,56 @@ * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ + * + * The invoker machinery that constructs a boost::fusion argument list for use + * with boost::fusion::invoke() is derived from + * http://www.boost.org/doc/libs/1_45_0/libs/function_types/example/interpreter.hpp + * whose license information is copied below: + * + * "(C) Copyright Tobias Schwinger + * + * Use modification and distribution are subject to the boost Software License, + * Version 1.0. (See http://www.boost.org/LICENSE_1_0.txt)." */ #if ! defined(LL_LLEVENTDISPATCHER_H) #define LL_LLEVENTDISPATCHER_H +// nil is too generic a term to be allowed to be a global macro. In +// particular, boost::fusion defines a 'class nil' (properly encapsulated in a +// namespace) that a global 'nil' macro breaks badly. +#if defined(nil) +// Capture the value of the macro 'nil', hoping int is an appropriate type. +static const int nil_(nil); +// Now forget the macro. +#undef nil +// Finally, reintroduce 'nil' as a properly-scoped alias for the previously- +// defined const 'nil_'. Make it static since otherwise it produces duplicate- +// symbol link errors later. +static const int& nil(nil_); +#endif + #include <string> -#include <map> +#include <boost/shared_ptr.hpp> #include <boost/function.hpp> #include <boost/bind.hpp> #include <boost/iterator/transform_iterator.hpp> +#include <boost/utility/enable_if.hpp> +#include <boost/function_types/is_nonmember_callable_builtin.hpp> +#include <boost/function_types/parameter_types.hpp> +#include <boost/function_types/function_arity.hpp> +#include <boost/type_traits/remove_cv.hpp> +#include <boost/type_traits/remove_reference.hpp> +#include <boost/fusion/include/push_back.hpp> +#include <boost/fusion/include/cons.hpp> +#include <boost/fusion/include/invoke.hpp> +#include <boost/mpl/begin.hpp> +#include <boost/mpl/end.hpp> +#include <boost/mpl/next.hpp> +#include <boost/mpl/deref.hpp> #include <typeinfo> #include "llevents.h" +#include "llsdutil.h" class LLSD; @@ -54,12 +92,18 @@ public: LLEventDispatcher(const std::string& desc, const std::string& key); virtual ~LLEventDispatcher(); - /// Accept any C++ callable, typically a boost::bind() expression + /// @name Register functions accepting(const LLSD&) + //@{ + + /// Accept any C++ callable with the right signature, typically a + /// boost::bind() expression typedef boost::function<void(const LLSD&)> Callable; /** - * Register a @a callable by @a name. The optional @a required parameter - * is used to validate the structure of each incoming event (see + * Register a @a callable by @a name. The passed @a callable accepts a + * single LLSD value and uses it in any way desired, e.g. extract + * parameters and call some other function. The optional @a required + * parameter is used to validate the structure of each incoming event (see * llsd_matches()). */ void add(const std::string& name, @@ -68,9 +112,23 @@ public: const LLSD& required=LLSD()); /** + * The case of a free function (or static method) accepting(const LLSD&) + * could also be intercepted by the arbitrary-args overload below. Ensure + * that it's directed to the Callable overload above instead. + */ + void add(const std::string& name, + const std::string& desc, + void (*f)(const LLSD&), + const LLSD& required=LLSD()) + { + add(name, desc, Callable(f), required); + } + + /** * Special case: a subclass of this class can pass an unbound member - * function pointer without explicitly specifying the - * <tt>boost::bind()</tt> expression. + * function pointer (of an LLEventDispatcher subclass) without explicitly + * specifying the <tt>boost::bind()</tt> expression. The passed @a method + * accepts a single LLSD value, presumably containing other parameters. */ template <class CLASS> void add(const std::string& name, @@ -81,7 +139,8 @@ public: addMethod<CLASS>(name, desc, method, required); } - /// Overload for both const and non-const methods + /// Overload for both const and non-const methods. The passed @a method + /// accepts a single LLSD value, presumably containing other parameters. template <class CLASS> void add(const std::string& name, const std::string& desc, @@ -91,15 +150,106 @@ public: addMethod<CLASS>(name, desc, method, required); } - /// Convenience: for LLEventDispatcher, not every callable needs a - /// documentation string. - template <typename CALLABLE> - void add(const std::string& name, - CALLABLE callable, - const LLSD& required=LLSD()) - { - add(name, "", callable, required); - } + //@} + + /// @name Register functions with arbitrary param lists + //@{ + + /** + * Register a free function with arbitrary parameters. (This also works + * for static class methods.) + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * When calling this name, pass an LLSD::Array. Each entry in turn will be + * converted to the corresponding parameter type using LLSDParam. + */ + template<typename Function> + typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> + >::type add(const std::string& name, + const std::string& desc, + Function f); + + /** + * Register a nonstatic class method with arbitrary parameters. + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * To cover cases such as a method on an LLSingleton we don't yet want to + * instantiate, instead of directly storing an instance pointer, accept a + * nullary callable returning a pointer/reference to the desired class + * instance. If you already have an instance in hand, + * boost::lambda::var(instance) or boost::lambda::constant(instance_ptr) + * produce suitable callables. + * + * When calling this name, pass an LLSD::Array. Each entry in turn will be + * converted to the corresponding parameter type using LLSDParam. + */ + template<typename Method, typename InstanceGetter> + typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> + >::type add(const std::string& name, + const std::string& desc, + Method f, + const InstanceGetter& getter); + + /** + * Register a free function with arbitrary parameters. (This also works + * for static class methods.) + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * Pass an LLSD::Array of parameter names, and optionally another + * LLSD::Array of default parameter values, a la LLSDArgsMapper. + * + * When calling this name, pass an LLSD::Map. We will internally generate + * an LLSD::Array using LLSDArgsMapper and then convert each entry in turn + * to the corresponding parameter type using LLSDParam. + */ + template<typename Function> + typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> + >::type add(const std::string& name, + const std::string& desc, + Function f, + const LLSD& params, + const LLSD& defaults=LLSD()); + + /** + * Register a nonstatic class method with arbitrary parameters. + * + * @note This supports functions with up to about 6 parameters -- after + * that you start getting dismaying compile errors in which + * boost::fusion::joint_view is mentioned a surprising number of times. + * + * To cover cases such as a method on an LLSingleton we don't yet want to + * instantiate, instead of directly storing an instance pointer, accept a + * nullary callable returning a pointer/reference to the desired class + * instance. If you already have an instance in hand, + * boost::lambda::var(instance) or boost::lambda::constant(instance_ptr) + * produce suitable callables. + * + * Pass an LLSD::Array of parameter names, and optionally another + * LLSD::Array of default parameter values, a la LLSDArgsMapper. + * + * When calling this name, pass an LLSD::Map. We will internally generate + * an LLSD::Array using LLSDArgsMapper and then convert each entry in turn + * to the corresponding parameter type using LLSDParam. + */ + template<typename Method, typename InstanceGetter> + typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> + >::type add(const std::string& name, + const std::string& desc, + Method f, + const InstanceGetter& getter, + const LLSD& params, + const LLSD& defaults=LLSD()); + + //@} /// Unregister a callable bool remove(const std::string& name); @@ -109,12 +259,25 @@ public: /// the @a required prototype specified at add() time, die with LL_ERRS. void operator()(const std::string& name, const LLSD& event) const; + /// Call a registered callable with an explicitly-specified name and + /// return <tt>true</tt>. If no such callable exists, return + /// <tt>false</tt>. If the @a event fails to match the @a required + /// prototype specified at add() time, die with LL_ERRS. + bool try_call(const std::string& name, const LLSD& event) const; + /// Extract the @a key value from the incoming @a event, and call the /// callable whose name is specified by that map @a key. If no such /// callable exists, die with LL_ERRS. If the @a event fails to match the /// @a required prototype specified at add() time, die with LL_ERRS. void operator()(const LLSD& event) const; + /// Extract the @a key value from the incoming @a event, call the callable + /// whose name is specified by that map @a key and return <tt>true</tt>. + /// If no such callable exists, return <tt>false</tt>. If the @a event + /// fails to match the @a required prototype specified at add() time, die + /// with LL_ERRS. + bool try_call(const LLSD& event) const; + /// @name Iterate over defined names //@{ typedef std::pair<std::string, std::string> NameDesc; @@ -122,16 +285,22 @@ public: private: struct DispatchEntry { - DispatchEntry(const Callable& func, const std::string& desc, const LLSD& required): - mFunc(func), - mDesc(desc), - mRequired(required) - {} - Callable mFunc; + DispatchEntry(const std::string& desc); + virtual ~DispatchEntry() {} // suppress MSVC warning, sigh + std::string mDesc; - LLSD mRequired; + + virtual void call(const std::string& desc, const LLSD& event) const = 0; + virtual LLSD addMetadata(LLSD) const = 0; }; - typedef std::map<std::string, DispatchEntry> DispatchMap; + // Tried using boost::ptr_map<std::string, DispatchEntry>, but ptr_map<> + // wants its value type to be "clonable," even just to dereference an + // iterator. I don't want to clone entries -- if I have to copy an entry + // around, I want it to continue pointing to the same DispatchEntry + // subclass object. However, I definitely want DispatchMap to destroy + // DispatchEntry if no references are outstanding at the time an entry is + // removed. This looks like a job for boost::shared_ptr. + typedef std::map<std::string, boost::shared_ptr<DispatchEntry> > DispatchMap; public: /// We want the flexibility to redefine what data we store per name, @@ -149,10 +318,6 @@ public: } //@} - /// Fetch the Callable for the specified name. If no such name was - /// registered, return an empty() Callable. - Callable get(const std::string& name) const; - /// Get information about a specific Callable LLSD getMetadata(const std::string& name) const; @@ -175,18 +340,184 @@ private: } } void addFail(const std::string& name, const std::string& classname) const; - /// try to dispatch, return @c true if success - bool attemptCall(const std::string& name, const LLSD& event) const; std::string mDesc, mKey; DispatchMap mDispatch; static NameDesc makeNameDesc(const DispatchMap::value_type& item) { - return NameDesc(item.first, item.second.mDesc); + return NameDesc(item.first, item.second->mDesc); + } + + struct LLSDDispatchEntry; + struct ParamsDispatchEntry; + struct ArrayParamsDispatchEntry; + struct MapParamsDispatchEntry; + + // Step 2 of parameter analysis. Instantiating invoker<some_function_type> + // implicitly sets its From and To parameters to the (compile time) begin + // and end iterators over that function's parameter types. + template< typename Function + , class From = typename boost::mpl::begin< boost::function_types::parameter_types<Function> >::type + , class To = typename boost::mpl::end< boost::function_types::parameter_types<Function> >::type + > + struct invoker; + + // deliver LLSD arguments one at a time + typedef boost::function<LLSD()> args_source; + // obtain args from an args_source to build param list and call target + // function + typedef boost::function<void(const args_source&)> invoker_function; + + template <typename Function> + invoker_function make_invoker(Function f); + template <typename Method, typename InstanceGetter> + invoker_function make_invoker(Method f, const InstanceGetter& getter); + void addArrayParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + LLSD::Integer arity); + void addMapParamsDispatchEntry(const std::string& name, + const std::string& desc, + const invoker_function& invoker, + const LLSD& params, + const LLSD& defaults); +}; + +/***************************************************************************** +* LLEventDispatcher template implementation details +*****************************************************************************/ +// Step 3 of parameter analysis, the recursive case. +template<typename Function, class From, class To> +struct LLEventDispatcher::invoker +{ + template<typename T> + struct remove_cv_ref + : boost::remove_cv< typename boost::remove_reference<T>::type > + { }; + + // apply() accepts an arbitrary boost::fusion sequence as args. It + // examines the next parameter type in the parameter-types sequence + // bounded by From and To, obtains the next LLSD object from the passed + // args_source and constructs an LLSDParam of appropriate type to try + // to convert the value. It then recurs with the next parameter-types + // iterator, passing the args sequence thus far. + template<typename Args> + static inline + void apply(Function func, const args_source& argsrc, Args const & args) + { + typedef typename boost::mpl::deref<From>::type arg_type; + typedef typename boost::mpl::next<From>::type next_iter_type; + typedef typename remove_cv_ref<arg_type>::type plain_arg_type; + + invoker<Function, next_iter_type, To>::apply + ( func, argsrc, boost::fusion::push_back(args, LLSDParam<plain_arg_type>(argsrc()))); + } + + // Special treatment for instance (first) parameter of a non-static member + // function. Accept the instance-getter callable, calling that to produce + // the first args value. Since we know we're at the top of the recursion + // chain, we need not also require a partial args sequence from our caller. + template <typename InstanceGetter> + static inline + void method_apply(Function func, const args_source& argsrc, const InstanceGetter& getter) + { + typedef typename boost::mpl::next<From>::type next_iter_type; + + // Instead of grabbing the first item from argsrc and making an + // LLSDParam of it, call getter() and pass that as the instance param. + invoker<Function, next_iter_type, To>::apply + ( func, argsrc, boost::fusion::push_back(boost::fusion::nil(), boost::ref(getter()))); + } +}; + +// Step 4 of parameter analysis, the leaf case. When the general +// invoker<Function, From, To> logic has advanced From until it matches To, +// the compiler will pick this template specialization. +template<typename Function, class To> +struct LLEventDispatcher::invoker<Function,To,To> +{ + // the argument list is complete, now call the function + template<typename Args> + static inline + void apply(Function func, const args_source&, Args const & args) + { + boost::fusion::invoke(func, args); } }; +template<typename Function> +typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Function f) +{ + // Construct an invoker_function, a callable accepting const args_source&. + // Add to DispatchMap an ArrayParamsDispatchEntry that will handle the + // caller's LLSD::Array. + addArrayParamsDispatchEntry(name, desc, make_invoker(f), + boost::function_types::function_arity<Function>::value); +} + +template<typename Method, typename InstanceGetter> +typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Method f, + const InstanceGetter& getter) +{ + // Subtract 1 from the compile-time arity because the getter takes care of + // the first parameter. We only need (arity - 1) additional arguments. + addArrayParamsDispatchEntry(name, desc, make_invoker(f, getter), + boost::function_types::function_arity<Method>::value - 1); +} + +template<typename Function> +typename boost::enable_if< boost::function_types::is_nonmember_callable_builtin<Function> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Function f, + const LLSD& params, const LLSD& defaults) +{ + // See comments for previous is_nonmember_callable_builtin add(). + addMapParamsDispatchEntry(name, desc, make_invoker(f), params, defaults); +} + +template<typename Method, typename InstanceGetter> +typename boost::enable_if< boost::function_types::is_member_function_pointer<Method> >::type +LLEventDispatcher::add(const std::string& name, const std::string& desc, Method f, + const InstanceGetter& getter, + const LLSD& params, const LLSD& defaults) +{ + addMapParamsDispatchEntry(name, desc, make_invoker(f, getter), params, defaults); +} + +template <typename Function> +LLEventDispatcher::invoker_function +LLEventDispatcher::make_invoker(Function f) +{ + // Step 1 of parameter analysis, the top of the recursion. Passing a + // suitable f (see add()'s enable_if condition) to this method causes it + // to infer the function type; specifying that function type to invoker<> + // causes it to fill in the begin/end MPL iterators over the function's + // list of parameter types. + // While normally invoker::apply() could infer its template type from the + // boost::fusion::nil parameter value, here we must be explicit since + // we're boost::bind()ing it rather than calling it directly. + return boost::bind(&invoker<Function>::template apply<boost::fusion::nil>, + f, + _1, + boost::fusion::nil()); +} + +template <typename Method, typename InstanceGetter> +LLEventDispatcher::invoker_function +LLEventDispatcher::make_invoker(Method f, const InstanceGetter& getter) +{ + // Use invoker::method_apply() to treat the instance (first) arg specially. + return boost::bind(&invoker<Method>::template method_apply<InstanceGetter>, + f, + _1, + getter); +} + +/***************************************************************************** +* LLDispatchListener +*****************************************************************************/ /** * Bundle an LLEventPump and a listener with an LLEventDispatcher. A class * that contains (or derives from) LLDispatchListener need only specify the diff --git a/indra/llcommon/llevents.cpp b/indra/llcommon/llevents.cpp index 97e2bdeb57..ff03506e84 100644 --- a/indra/llcommon/llevents.cpp +++ b/indra/llcommon/llevents.cpp @@ -588,3 +588,16 @@ void LLReqID::stamp(LLSD& response) const } response["reqid"] = mReqid; } + +bool sendReply(const LLSD& reply, const LLSD& request, const std::string& replyKey) +{ + // Copy 'reply' to modify it. + LLSD newreply(reply); + // Get the ["reqid"] element from request + LLReqID reqID(request); + // and copy it to 'newreply'. + reqID.stamp(newreply); + // Send reply on LLEventPump named in request[replyKey]. Don't forget to + // send the modified 'newreply' instead of the original 'reply'. + return LLEventPumps::instance().obtain(request[replyKey]).post(newreply); +} diff --git a/indra/llcommon/llevents.h b/indra/llcommon/llevents.h index 2491cf1371..65b0fef354 100644 --- a/indra/llcommon/llevents.h +++ b/indra/llcommon/llevents.h @@ -692,6 +692,20 @@ private: }; /** + * Conventionally send a reply to a request event. + * + * @a reply is the LLSD reply event to send + * @a request is the corresponding LLSD request event + * @a replyKey is the key in the @a request event, conventionally ["reply"], + * whose value is the name of the LLEventPump on which to send the reply. + * + * Before sending the reply event, sendReply() copies the ["reqid"] item from + * the request to the reply. + */ +LL_COMMON_API bool sendReply(const LLSD& reply, const LLSD& request, + const std::string& replyKey="reply"); + +/** * Base class for LLListenerWrapper. See visit_and_connect() and llwrap(). We * provide virtual @c accept_xxx() methods, customization points allowing a * subclass access to certain data visible at LLEventPump::listen() time. diff --git a/indra/llcommon/lleventtimer.cpp b/indra/llcommon/lleventtimer.cpp index 7743826c60..0d96e03da4 100644 --- a/indra/llcommon/lleventtimer.cpp +++ b/indra/llcommon/lleventtimer.cpp @@ -58,19 +58,15 @@ LLEventTimer::~LLEventTimer() void LLEventTimer::updateClass() { std::list<LLEventTimer*> completed_timers; - + for (instance_iter iter = beginInstances(); iter != endInstances(); ) { - LLInstanceTrackerScopedGuard guard; - for (instance_iter iter = guard.beginInstances(); iter != guard.endInstances(); ) - { - LLEventTimer& timer = *iter++; - F32 et = timer.mEventTimer.getElapsedTimeF32(); - if (timer.mEventTimer.getStarted() && et > timer.mPeriod) { - timer.mEventTimer.reset(); - if ( timer.tick() ) - { - completed_timers.push_back( &timer ); - } + LLEventTimer& timer = *iter++; + F32 et = timer.mEventTimer.getElapsedTimeF32(); + if (timer.mEventTimer.getStarted() && et > timer.mPeriod) { + timer.mEventTimer.reset(); + if ( timer.tick() ) + { + completed_timers.push_back( &timer ); } } } diff --git a/indra/llcommon/llfasttimer.h b/indra/llcommon/llfasttimer.h index 4ff93a553c..2b25f2fabb 100644 --- a/indra/llcommon/llfasttimer.h +++ b/indra/llcommon/llfasttimer.h @@ -27,140 +27,9 @@ #ifndef LL_FASTTIMER_H #define LL_FASTTIMER_H +// Implementation of getCPUClockCount32() and getCPUClockCount64 are now in llfastertimer_class.cpp. + // pull in the actual class definition #include "llfasttimer_class.h" -// -// Important note: These implementations must be FAST! -// - -#if LL_WINDOWS -// -// Windows implementation of CPU clock -// - -// -// NOTE: put back in when we aren't using platform sdk anymore -// -// because MS has different signatures for these functions in winnt.h -// need to rename them to avoid conflicts -//#define _interlockedbittestandset _renamed_interlockedbittestandset -//#define _interlockedbittestandreset _renamed_interlockedbittestandreset -//#include <intrin.h> -//#undef _interlockedbittestandset -//#undef _interlockedbittestandreset - -//inline U32 LLFastTimer::getCPUClockCount32() -//{ -// U64 time_stamp = __rdtsc(); -// return (U32)(time_stamp >> 8); -//} -// -//// return full timer value, *not* shifted by 8 bits -//inline U64 LLFastTimer::getCPUClockCount64() -//{ -// return __rdtsc(); -//} - -// shift off lower 8 bits for lower resolution but longer term timing -// on 1Ghz machine, a 32-bit word will hold ~1000 seconds of timing -inline U32 LLFastTimer::getCPUClockCount32() -{ - U32 ret_val; - __asm - { - _emit 0x0f - _emit 0x31 - shr eax,8 - shl edx,24 - or eax, edx - mov dword ptr [ret_val], eax - } - return ret_val; -} - -// return full timer value, *not* shifted by 8 bits -inline U64 LLFastTimer::getCPUClockCount64() -{ - U64 ret_val; - __asm - { - _emit 0x0f - _emit 0x31 - mov eax,eax - mov edx,edx - mov dword ptr [ret_val+4], edx - mov dword ptr [ret_val], eax - } - return ret_val; -} -#endif - - -#if (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) -// -// Linux and Solaris implementation of CPU clock - non-x86. -// This is accurate but SLOW! Only use out of desperation. -// -// Try to use the MONOTONIC clock if available, this is a constant time counter -// with nanosecond resolution (but not necessarily accuracy) and attempts are -// made to synchronize this value between cores at kernel start. It should not -// be affected by CPU frequency. If not available use the REALTIME clock, but -// this may be affected by NTP adjustments or other user activity affecting -// the system time. -inline U64 LLFastTimer::getCPUClockCount64() -{ - struct timespec tp; - -#ifdef CLOCK_MONOTONIC // MONOTONIC supported at build-time? - if (-1 == clock_gettime(CLOCK_MONOTONIC,&tp)) // if MONOTONIC isn't supported at runtime then ouch, try REALTIME -#endif - clock_gettime(CLOCK_REALTIME,&tp); - - return (tp.tv_sec*LLFastTimer::sClockResolution)+tp.tv_nsec; -} - -inline U32 LLFastTimer::getCPUClockCount32() -{ - return (U32)(LLFastTimer::getCPUClockCount64() >> 8); -} -#endif // (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) - - -#if (LL_LINUX || LL_SOLARIS || LL_DARWIN) && (defined(__i386__) || defined(__amd64__)) -// -// Mac+Linux+Solaris FAST x86 implementation of CPU clock -inline U32 LLFastTimer::getCPUClockCount32() -{ - U64 x; - __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); - return (U32)(x >> 8); -} - -inline U64 LLFastTimer::getCPUClockCount64() -{ - U64 x; - __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); - return x; -} -#endif - - -#if ( LL_DARWIN && !(defined(__i386__) || defined(__amd64__))) -// -// Mac PPC (deprecated) implementation of CPU clock -// -// Just use gettimeofday implementation for now - -inline U32 LLFastTimer::getCPUClockCount32() -{ - return (U32)(get_clock_count()>>8); -} - -inline U64 LLFastTimer::getCPUClockCount64() -{ - return get_clock_count(); -} -#endif - #endif // LL_LLFASTTIMER_H diff --git a/indra/llcommon/llfasttimer_class.cpp b/indra/llcommon/llfasttimer_class.cpp index bce87ada96..463f558c2c 100644 --- a/indra/llcommon/llfasttimer_class.cpp +++ b/indra/llcommon/llfasttimer_class.cpp @@ -35,10 +35,13 @@ #include <boost/bind.hpp> + #if LL_WINDOWS +#include "lltimer.h" #elif LL_LINUX || LL_SOLARIS #include <sys/time.h> #include <sched.h> +#include "lltimer.h" #elif LL_DARWIN #include <sys/time.h> #include "lltimer.h" // get_clock_count() @@ -61,6 +64,8 @@ BOOL LLFastTimer::sMetricLog = FALSE; LLMutex* LLFastTimer::sLogLock = NULL; std::queue<LLSD> LLFastTimer::sLogQueue; +#define USE_RDTSC 0 + #if LL_LINUX || LL_SOLARIS U64 LLFastTimer::sClockResolution = 1000000000; // Nanosecond resolution #else @@ -214,15 +219,20 @@ LLFastTimer::DeclareTimer::DeclareTimer(const std::string& name) // static void LLFastTimer::DeclareTimer::updateCachedPointers() { - DeclareTimer::LLInstanceTrackerScopedGuard guard; // propagate frame state pointers to timer declarations - for (DeclareTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { // update cached pointer it->mFrameState = &it->mTimer.getFrameState(); } + + // also update frame states of timers on stack + LLFastTimer* cur_timerp = LLFastTimer::sCurTimerData.mCurTimer; + while(cur_timerp->mLastTimerData.mCurTimer != cur_timerp) + { + cur_timerp->mFrameState = &cur_timerp->mFrameState->mTimer->getFrameState(); + cur_timerp = cur_timerp->mLastTimerData.mCurTimer; + } } //static @@ -234,10 +244,23 @@ U64 LLFastTimer::countsPerSecond() // counts per second for the *32-bit* timer #else // windows or x86-mac or x86-linux or x86-solaris U64 LLFastTimer::countsPerSecond() // counts per second for the *32-bit* timer { +#if USE_RDTSC || !LL_WINDOWS //getCPUFrequency returns MHz and sCPUClockFrequency wants to be in Hz static U64 sCPUClockFrequency = U64(LLProcessorInfo().getCPUFrequency()*1000000.0); // we drop the low-order byte in our timers, so report a lower frequency +#else + // If we're not using RDTSC, each fasttimer tick is just a performance counter tick. + // Not redefining the clock frequency itself (in llprocessor.cpp/calculate_cpu_frequency()) + // since that would change displayed MHz stats for CPUs + static bool firstcall = true; + static U64 sCPUClockFrequency; + if (firstcall) + { + QueryPerformanceFrequency((LARGE_INTEGER*)&sCPUClockFrequency); + firstcall = false; + } +#endif return sCPUClockFrequency >> 8; } #endif @@ -280,14 +303,15 @@ LLFastTimer::NamedTimer::~NamedTimer() std::string LLFastTimer::NamedTimer::getToolTip(S32 history_idx) { + F64 ms_multiplier = 1000.0 / (F64)LLFastTimer::countsPerSecond(); if (history_idx < 0) { - // by default, show average number of calls - return llformat("%s (%d calls)", getName().c_str(), (S32)getCallAverage()); + // by default, show average number of call + return llformat("%s (%d ms, %d calls)", getName().c_str(), (S32)(getCountAverage() * ms_multiplier), (S32)getCallAverage()); } else { - return llformat("%s (%d calls)", getName().c_str(), (S32)getHistoricalCalls(history_idx)); + return llformat("%s (%d ms, %d calls)", getName().c_str(), (S32)(getHistoricalCount(history_idx) * ms_multiplier), (S32)getHistoricalCalls(history_idx)); } } @@ -370,10 +394,7 @@ void LLFastTimer::NamedTimer::buildHierarchy() // set up initial tree { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; if (&timer == NamedTimerFactory::instance().getRootTimer()) continue; @@ -482,16 +503,26 @@ void LLFastTimer::NamedTimer::resetFrame() { if (sLog) { //output current frame counts to performance log + + static S32 call_count = 0; + if (call_count % 100 == 0) + { + llinfos << "countsPerSecond (32 bit): " << countsPerSecond() << llendl; + llinfos << "get_clock_count (64 bit): " << get_clock_count() << llendl; + llinfos << "LLProcessorInfo().getCPUFrequency() " << LLProcessorInfo().getCPUFrequency() << llendl; + llinfos << "getCPUClockCount32() " << getCPUClockCount32() << llendl; + llinfos << "getCPUClockCount64() " << getCPUClockCount64() << llendl; + llinfos << "elapsed sec " << ((F64)getCPUClockCount64())/((F64)LLProcessorInfo().getCPUFrequency()*1000000.0) << llendl; + } + call_count++; + F64 iclock_freq = 1000.0 / countsPerSecond(); // good place to calculate clock frequency F64 total_time = 0; LLSD sd; { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (NamedTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; FrameState& info = timer.getFrameState(); @@ -528,7 +559,7 @@ void LLFastTimer::NamedTimer::resetFrame() llassert_always(timerp->mFrameStateIndex < (S32)getFrameStateList().size()); } - // sort timers by dfs traversal order to improve cache coherency + // sort timers by DFS traversal order to improve cache coherency std::sort(getFrameStateList().begin(), getFrameStateList().end(), SortTimersDFS()); // update pointers into framestatelist now that we've sorted it @@ -536,10 +567,7 @@ void LLFastTimer::NamedTimer::resetFrame() // reset for next frame { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (NamedTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; @@ -583,10 +611,7 @@ void LLFastTimer::NamedTimer::reset() // reset all history { - NamedTimer::LLInstanceTrackerScopedGuard guard; - for (NamedTimer::instance_iter it = guard.beginInstances(); - it != guard.endInstances(); - ++it) + for (instance_iter it = beginInstances(); it != endInstances(); ++it) { NamedTimer& timer = *it; if (&timer != NamedTimerFactory::instance().getRootTimer()) @@ -669,17 +694,7 @@ void LLFastTimer::nextFrame() llinfos << "Slow frame, fast timers inaccurate" << llendl; } - if (sPauseHistory) - { - sResetHistory = true; - } - else if (sResetHistory) - { - sLastFrameIndex = 0; - sCurFrameIndex = 0; - sResetHistory = false; - } - else // not paused + if (!sPauseHistory) { NamedTimer::processTimes(); sLastFrameIndex = sCurFrameIndex++; @@ -763,3 +778,144 @@ LLFastTimer::LLFastTimer(LLFastTimer::FrameState* state) ////////////////////////////////////////////////////////////////////////////// +// +// Important note: These implementations must be FAST! +// + + +#if LL_WINDOWS +// +// Windows implementation of CPU clock +// + +// +// NOTE: put back in when we aren't using platform sdk anymore +// +// because MS has different signatures for these functions in winnt.h +// need to rename them to avoid conflicts +//#define _interlockedbittestandset _renamed_interlockedbittestandset +//#define _interlockedbittestandreset _renamed_interlockedbittestandreset +//#include <intrin.h> +//#undef _interlockedbittestandset +//#undef _interlockedbittestandreset + +//inline U32 LLFastTimer::getCPUClockCount32() +//{ +// U64 time_stamp = __rdtsc(); +// return (U32)(time_stamp >> 8); +//} +// +//// return full timer value, *not* shifted by 8 bits +//inline U64 LLFastTimer::getCPUClockCount64() +//{ +// return __rdtsc(); +//} + +// shift off lower 8 bits for lower resolution but longer term timing +// on 1Ghz machine, a 32-bit word will hold ~1000 seconds of timing +#if USE_RDTSC +U32 LLFastTimer::getCPUClockCount32() +{ + U32 ret_val; + __asm + { + _emit 0x0f + _emit 0x31 + shr eax,8 + shl edx,24 + or eax, edx + mov dword ptr [ret_val], eax + } + return ret_val; +} + +// return full timer value, *not* shifted by 8 bits +U64 LLFastTimer::getCPUClockCount64() +{ + U64 ret_val; + __asm + { + _emit 0x0f + _emit 0x31 + mov eax,eax + mov edx,edx + mov dword ptr [ret_val+4], edx + mov dword ptr [ret_val], eax + } + return ret_val; +} + +std::string LLFastTimer::sClockType = "rdtsc"; + +#else +//LL_COMMON_API U64 get_clock_count(); // in lltimer.cpp +// These use QueryPerformanceCounter, which is arguably fine and also works on AMD architectures. +U32 LLFastTimer::getCPUClockCount32() +{ + return (U32)(get_clock_count()>>8); +} + +U64 LLFastTimer::getCPUClockCount64() +{ + return get_clock_count(); +} + +std::string LLFastTimer::sClockType = "QueryPerformanceCounter"; +#endif + +#endif + + +#if (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) +// +// Linux and Solaris implementation of CPU clock - non-x86. +// This is accurate but SLOW! Only use out of desperation. +// +// Try to use the MONOTONIC clock if available, this is a constant time counter +// with nanosecond resolution (but not necessarily accuracy) and attempts are +// made to synchronize this value between cores at kernel start. It should not +// be affected by CPU frequency. If not available use the REALTIME clock, but +// this may be affected by NTP adjustments or other user activity affecting +// the system time. +U64 LLFastTimer::getCPUClockCount64() +{ + struct timespec tp; + +#ifdef CLOCK_MONOTONIC // MONOTONIC supported at build-time? + if (-1 == clock_gettime(CLOCK_MONOTONIC,&tp)) // if MONOTONIC isn't supported at runtime then ouch, try REALTIME +#endif + clock_gettime(CLOCK_REALTIME,&tp); + + return (tp.tv_sec*LLFastTimer::sClockResolution)+tp.tv_nsec; +} + +U32 LLFastTimer::getCPUClockCount32() +{ + return (U32)(LLFastTimer::getCPUClockCount64() >> 8); +} + +std::string LLFastTimer::sClockType = "clock_gettime"; + +#endif // (LL_LINUX || LL_SOLARIS) && !(defined(__i386__) || defined(__amd64__)) + + +#if (LL_LINUX || LL_SOLARIS || LL_DARWIN) && (defined(__i386__) || defined(__amd64__)) +// +// Mac+Linux+Solaris FAST x86 implementation of CPU clock +U32 LLFastTimer::getCPUClockCount32() +{ + U64 x; + __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); + return (U32)(x >> 8); +} + +U64 LLFastTimer::getCPUClockCount64() +{ + U64 x; + __asm__ volatile (".byte 0x0f, 0x31": "=A"(x)); + return x; +} + +std::string LLFastTimer::sClockType = "rdtsc"; +#endif + diff --git a/indra/llcommon/llfasttimer_class.h b/indra/llcommon/llfasttimer_class.h index eb9789682b..f481e968a6 100644 --- a/indra/llcommon/llfasttimer_class.h +++ b/indra/llcommon/llfasttimer_class.h @@ -31,12 +31,15 @@ #define FAST_TIMER_ON 1 #define TIME_FAST_TIMERS 0 +#define DEBUG_FAST_TIMER_THREADS 1 class LLMutex; #include <queue> #include "llsd.h" +LL_COMMON_API void assert_main_thread(); + class LL_COMMON_API LLFastTimer { public: @@ -63,7 +66,7 @@ public: public: ~NamedTimer(); - enum { HISTORY_NUM = 60 }; + enum { HISTORY_NUM = 300 }; const std::string& getName() const { return mName; } NamedTimer* getParent() const { return mParent; } @@ -176,6 +179,11 @@ public: U64 timer_end = getCPUClockCount64(); sTimerCycles += timer_end - timer_start; #endif +#if DEBUG_FAST_TIMER_THREADS +#if !LL_RELEASE + assert_main_thread(); +#endif +#endif } LL_FORCE_INLINE ~LLFastTimer() @@ -245,6 +253,7 @@ public: U32 mChildTime; }; static CurTimerData sCurTimerData; + static std::string sClockType; private: static U32 getCPUClockCount32(); diff --git a/indra/llcommon/llfoldertype.cpp b/indra/llcommon/llfoldertype.cpp index ebc79af412..f6d0f5bce8 100644 --- a/indra/llcommon/llfoldertype.cpp +++ b/indra/llcommon/llfoldertype.cpp @@ -89,7 +89,12 @@ LLFolderDictionary::LLFolderDictionary() addEntry(LLFolderType::FT_CURRENT_OUTFIT, new FolderEntry("current", TRUE)); addEntry(LLFolderType::FT_OUTFIT, new FolderEntry("outfit", FALSE)); addEntry(LLFolderType::FT_MY_OUTFITS, new FolderEntry("my_otfts", TRUE)); + + addEntry(LLFolderType::FT_MESH, new FolderEntry("mesh", TRUE)); + addEntry(LLFolderType::FT_INBOX, new FolderEntry("inbox", TRUE)); + addEntry(LLFolderType::FT_OUTBOX, new FolderEntry("outbox", TRUE)); + addEntry(LLFolderType::FT_BASIC_ROOT, new FolderEntry("basic_rt", TRUE)); addEntry(LLFolderType::FT_NONE, new FolderEntry("-1", FALSE)); }; diff --git a/indra/llcommon/llfoldertype.h b/indra/llcommon/llfoldertype.h index 936fbed17d..a0c847914f 100644 --- a/indra/llcommon/llfoldertype.h +++ b/indra/llcommon/llfoldertype.h @@ -80,9 +80,14 @@ public: FT_OUTFIT = 47, FT_MY_OUTFITS = 48, - FT_INBOX = 49, + FT_MESH = 49, - FT_COUNT = 50, + FT_INBOX = 50, + FT_OUTBOX = 51, + + FT_BASIC_ROOT = 52, + + FT_COUNT, FT_NONE = -1 }; diff --git a/indra/llcommon/llinstancetracker.cpp b/indra/llcommon/llinstancetracker.cpp index 89bc6cca39..5dc3ea5d7b 100644 --- a/indra/llcommon/llinstancetracker.cpp +++ b/indra/llcommon/llinstancetracker.cpp @@ -32,6 +32,18 @@ // external library headers // other Linden headers -// llinstancetracker.h is presently header-only. This file exists only because our CMake -// test macro ADD_BUILD_TEST requires it. -int dummy = 0; +//static +void * & LLInstanceTrackerBase::getInstances(std::type_info const & info) +{ + typedef std::map<std::string, void *> InstancesMap; + static InstancesMap instances; + + // std::map::insert() is just what we want here. You attempt to insert a + // (key, value) pair. If the specified key doesn't yet exist, it inserts + // the pair and returns a std::pair of (iterator, true). If the specified + // key DOES exist, insert() simply returns (iterator, false). One lookup + // handles both cases. + return instances.insert(InstancesMap::value_type(info.name(), + InstancesMap::mapped_type())) + .first->second; +} diff --git a/indra/llcommon/llinstancetracker.h b/indra/llcommon/llinstancetracker.h index 4945461d62..5a3990a8df 100644 --- a/indra/llcommon/llinstancetracker.h +++ b/indra/llcommon/llinstancetracker.h @@ -29,6 +29,7 @@ #define LL_LLINSTANCETRACKER_H #include <map> +#include <typeinfo> #include "string_table.h" #include <boost/utility.hpp> @@ -37,21 +38,132 @@ #include <boost/iterator/transform_iterator.hpp> #include <boost/iterator/indirect_iterator.hpp> +/** + * Base class manages "class-static" data that must actually have singleton + * semantics: one instance per process, rather than one instance per module as + * sometimes happens with data simply declared static. + */ +class LL_COMMON_API LLInstanceTrackerBase : public boost::noncopyable +{ +protected: + /// Get a process-unique void* pointer slot for the specified type_info + static void * & getInstances(std::type_info const & info); + + /// Find or create a STATICDATA instance for the specified TRACKED class. + /// STATICDATA must be default-constructible. + template<typename STATICDATA, class TRACKED> + static STATICDATA& getStatic() + { + void *& instances = getInstances(typeid(TRACKED)); + if (! instances) + { + instances = new STATICDATA; + } + return *static_cast<STATICDATA*>(instances); + } + + /// It's not essential to derive your STATICDATA (for use with + /// getStatic()) from StaticBase; it's just that both known + /// implementations do. + struct StaticBase + { + StaticBase(): + sIterationNestDepth(0) + {} + S32 sIterationNestDepth; + }; +}; + /// This mix-in class adds support for tracking all instances of the specified class parameter T /// The (optional) key associates a value of type KEY with a given instance of T, for quick lookup /// If KEY is not provided, then instances are stored in a simple set /// @NOTE: see explicit specialization below for default KEY==T* case template<typename T, typename KEY = T*> -class LLInstanceTracker : boost::noncopyable +class LLInstanceTracker : public LLInstanceTrackerBase { + typedef LLInstanceTracker<T, KEY> MyT; typedef typename std::map<KEY, T*> InstanceMap; - typedef boost::function<const KEY&(typename InstanceMap::value_type&)> KeyGetter; - typedef boost::function<T*(typename InstanceMap::value_type&)> InstancePtrGetter; + struct StaticData: public StaticBase + { + InstanceMap sMap; + }; + static StaticData& getStatic() { return LLInstanceTrackerBase::getStatic<StaticData, MyT>(); } + static InstanceMap& getMap_() { return getStatic().sMap; } + public: - /// Dereferencing key_iter gives you a const KEY& - typedef boost::transform_iterator<KeyGetter, typename InstanceMap::iterator> key_iter; - /// Dereferencing instance_iter gives you a T& - typedef boost::indirect_iterator< boost::transform_iterator<InstancePtrGetter, typename InstanceMap::iterator> > instance_iter; + class instance_iter : public boost::iterator_facade<instance_iter, T, boost::forward_traversal_tag> + { + public: + typedef boost::iterator_facade<instance_iter, T, boost::forward_traversal_tag> super_t; + + instance_iter(const typename InstanceMap::iterator& it) + : mIterator(it) + { + ++getStatic().sIterationNestDepth; + } + + ~instance_iter() + { + --getStatic().sIterationNestDepth; + } + + + private: + friend class boost::iterator_core_access; + + void increment() { mIterator++; } + bool equal(instance_iter const& other) const + { + return mIterator == other.mIterator; + } + + T& dereference() const + { + return *(mIterator->second); + } + + typename InstanceMap::iterator mIterator; + }; + + class key_iter : public boost::iterator_facade<key_iter, KEY, boost::forward_traversal_tag> + { + public: + typedef boost::iterator_facade<key_iter, KEY, boost::forward_traversal_tag> super_t; + + key_iter(typename InstanceMap::iterator it) + : mIterator(it) + { + ++getStatic().sIterationNestDepth; + } + + key_iter(const key_iter& other) + : mIterator(other.mIterator) + { + ++getStatic().sIterationNestDepth; + } + + ~key_iter() + { + --getStatic().sIterationNestDepth; + } + + + private: + friend class boost::iterator_core_access; + + void increment() { mIterator++; } + bool equal(key_iter const& other) const + { + return mIterator == other.mIterator; + } + + KEY& dereference() const + { + return const_cast<KEY&>(mIterator->first); + } + + typename InstanceMap::iterator mIterator; + }; static T* getInstance(const KEY& k) { @@ -59,132 +171,130 @@ public: return (found == getMap_().end()) ? NULL : found->second; } - static key_iter beginKeys() - { - return boost::make_transform_iterator(getMap_().begin(), - boost::bind(&InstanceMap::value_type::first, _1)); + static instance_iter beginInstances() + { + return instance_iter(getMap_().begin()); } - static key_iter endKeys() + + static instance_iter endInstances() { - return boost::make_transform_iterator(getMap_().end(), - boost::bind(&InstanceMap::value_type::first, _1)); + return instance_iter(getMap_().end()); } - static instance_iter beginInstances() + + static S32 instanceCount() { return getMap_().size(); } + + static key_iter beginKeys() { - return instance_iter(boost::make_transform_iterator(getMap_().begin(), - boost::bind(&InstanceMap::value_type::second, _1))); + return key_iter(getMap_().begin()); } - static instance_iter endInstances() + static key_iter endKeys() { - return instance_iter(boost::make_transform_iterator(getMap_().end(), - boost::bind(&InstanceMap::value_type::second, _1))); + return key_iter(getMap_().end()); } - static S32 instanceCount() { return getMap_().size(); } + protected: LLInstanceTracker(KEY key) { add_(key); } - virtual ~LLInstanceTracker() { remove_(); } + virtual ~LLInstanceTracker() + { + // it's unsafe to delete instances of this type while all instances are being iterated over. + llassert_always(getStatic().sIterationNestDepth == 0); + remove_(); + } virtual void setKey(KEY key) { remove_(); add_(key); } - virtual const KEY& getKey() const { return mKey; } + virtual const KEY& getKey() const { return mInstanceKey; } private: void add_(KEY key) { - mKey = key; + mInstanceKey = key; getMap_()[key] = static_cast<T*>(this); } void remove_() { - getMap_().erase(mKey); + getMap_().erase(mInstanceKey); } - static InstanceMap& getMap_() - { - if (! sInstances) - { - sInstances = new InstanceMap; - } - return *sInstances; - } - private: - - KEY mKey; - static InstanceMap* sInstances; + KEY mInstanceKey; }; /// explicit specialization for default case where KEY is T* /// use a simple std::set<T*> template<typename T> -class LLInstanceTracker<T, T*> +class LLInstanceTracker<T, T*> : public LLInstanceTrackerBase { + typedef LLInstanceTracker<T, T*> MyT; typedef typename std::set<T*> InstanceSet; + struct StaticData: public StaticBase + { + InstanceSet sSet; + }; + static StaticData& getStatic() { return LLInstanceTrackerBase::getStatic<StaticData, MyT>(); } + static InstanceSet& getSet_() { return getStatic().sSet; } + public: - /// Dereferencing key_iter gives you a T* (since T* is the key) - typedef typename InstanceSet::iterator key_iter; - /// Dereferencing instance_iter gives you a T& - typedef boost::indirect_iterator<key_iter> instance_iter; /// for completeness of analogy with the generic implementation static T* getInstance(T* k) { return k; } static S32 instanceCount() { return getSet_().size(); } - // Instantiate this to get access to iterators for this type. It's a 'guard' in the sense - // that it treats deletes of this type as errors as long as there is an instance of - // this class alive in scope somewhere (i.e. deleting while iterating is bad). - class LLInstanceTrackerScopedGuard + class instance_iter : public boost::iterator_facade<instance_iter, T, boost::forward_traversal_tag> { public: - LLInstanceTrackerScopedGuard() + instance_iter(const typename InstanceSet::iterator& it) + : mIterator(it) + { + ++getStatic().sIterationNestDepth; + } + + instance_iter(const instance_iter& other) + : mIterator(other.mIterator) + { + ++getStatic().sIterationNestDepth; + } + + ~instance_iter() { - ++sIterationNestDepth; + --getStatic().sIterationNestDepth; } - ~LLInstanceTrackerScopedGuard() + private: + friend class boost::iterator_core_access; + + void increment() { mIterator++; } + bool equal(instance_iter const& other) const { - --sIterationNestDepth; + return mIterator == other.mIterator; } - static instance_iter beginInstances() { return instance_iter(getSet_().begin()); } - static instance_iter endInstances() { return instance_iter(getSet_().end()); } - static key_iter beginKeys() { return getSet_().begin(); } - static key_iter endKeys() { return getSet_().end(); } + T& dereference() const + { + return **mIterator; + } + + typename InstanceSet::iterator mIterator; }; + static instance_iter beginInstances() { return instance_iter(getSet_().begin()); } + static instance_iter endInstances() { return instance_iter(getSet_().end()); } + protected: LLInstanceTracker() { - // it's safe but unpredictable to create instances of this type while all instances are being iterated over. I hate unpredictable. This assert will probably be turned on early in the next development cycle. - //llassert(sIterationNestDepth == 0); + // it's safe but unpredictable to create instances of this type while all instances are being iterated over. I hate unpredictable. This assert will probably be turned on early in the next development cycle. getSet_().insert(static_cast<T*>(this)); } virtual ~LLInstanceTracker() { // it's unsafe to delete instances of this type while all instances are being iterated over. - llassert(sIterationNestDepth == 0); + llassert_always(getStatic().sIterationNestDepth == 0); getSet_().erase(static_cast<T*>(this)); } LLInstanceTracker(const LLInstanceTracker& other) { - //llassert(sIterationNestDepth == 0); getSet_().insert(static_cast<T*>(this)); } - - static InstanceSet& getSet_() - { - if (! sInstances) - { - sInstances = new InstanceSet; - } - return *sInstances; - } - - static InstanceSet* sInstances; - static S32 sIterationNestDepth; }; -template <typename T, typename KEY> typename LLInstanceTracker<T, KEY>::InstanceMap* LLInstanceTracker<T, KEY>::sInstances = NULL; -template <typename T> typename LLInstanceTracker<T, T*>::InstanceSet* LLInstanceTracker<T, T*>::sInstances = NULL; -template <typename T> S32 LLInstanceTracker<T, T*>::sIterationNestDepth = 0; - #endif diff --git a/indra/llcommon/lllslconstants.h b/indra/llcommon/lllslconstants.h index 539c807020..9f32598e61 100644 --- a/indra/llcommon/lllslconstants.h +++ b/indra/llcommon/lllslconstants.h @@ -181,7 +181,7 @@ const S32 OBJECT_GROUP = 7; const S32 OBJECT_CREATOR = 8; // llTextBox() magic token string - yes this is a hack. sue me. -const std::string TEXTBOX_MAGIC_TOKEN = "!!llTextBox!!"; +char const* const TEXTBOX_MAGIC_TOKEN = "!!llTextBox!!"; // changed() event flags const U32 CHANGED_NONE = 0x0; diff --git a/indra/llcommon/llmd5.h b/indra/llcommon/llmd5.h index c8acbbe591..1526e6ac3c 100644 --- a/indra/llcommon/llmd5.h +++ b/indra/llcommon/llmd5.h @@ -103,7 +103,7 @@ public: void raw_digest(unsigned char *array) const; // provide 16-byte array for binary data void hex_digest(char *string) const; // provide 33-byte array for ascii-hex string - friend std::ostream& operator<< (std::ostream&, LLMD5 context); + friend LL_COMMON_API std::ostream& operator<< (std::ostream&, LLMD5 context); private: diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 51fcd5b717..3b27a1639a 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -26,14 +26,13 @@ #include "linden_common.h" -#include "llmemory.h" -#if MEM_TRACK_MEM +//#if MEM_TRACK_MEM #include "llthread.h" -#endif +//#endif #if defined(LL_WINDOWS) -# include <windows.h> +//# include <windows.h> # include <psapi.h> #elif defined(LL_DARWIN) # include <sys/types.h> @@ -43,10 +42,24 @@ # include <unistd.h> #endif +#include "llmemory.h" + +#include "llsys.h" +#include "llframetimer.h" //---------------------------------------------------------------------------- //static char* LLMemory::reserveMem = 0; +U32 LLMemory::sAvailPhysicalMemInKB = U32_MAX ; +U32 LLMemory::sMaxPhysicalMemInKB = 0; +U32 LLMemory::sAllocatedMemInKB = 0; +U32 LLMemory::sAllocatedPageSizeInKB = 0 ; +U32 LLMemory::sMaxHeapSizeInKB = U32_MAX ; +BOOL LLMemory::sEnableMemoryFailurePrevention = FALSE; + +#if __DEBUG_PRIVATE_MEM__ +LLPrivateMemoryPoolManager::mem_allocation_info_t LLPrivateMemoryPoolManager::sMemAllocationTracker; +#endif //static void LLMemory::initClass() @@ -71,6 +84,161 @@ void LLMemory::freeReserve() reserveMem = NULL; } +//static +void LLMemory::initMaxHeapSizeGB(F32 max_heap_size_gb, BOOL prevent_heap_failure) +{ + sMaxHeapSizeInKB = (U32)(max_heap_size_gb * 1024 * 1024) ; + sEnableMemoryFailurePrevention = prevent_heap_failure ; +} + +//static +void LLMemory::updateMemoryInfo() +{ +#if LL_WINDOWS + HANDLE self = GetCurrentProcess(); + PROCESS_MEMORY_COUNTERS counters; + + if (!GetProcessMemoryInfo(self, &counters, sizeof(counters))) + { + llwarns << "GetProcessMemoryInfo failed" << llendl; + return ; + } + + sAllocatedMemInKB = (U32)(counters.WorkingSetSize / 1024) ; + sAllocatedPageSizeInKB = (U32)(counters.PagefileUsage / 1024) ; + + U32 avail_phys, avail_virtual; + LLMemoryInfo::getAvailableMemoryKB(avail_phys, avail_virtual) ; + sMaxPhysicalMemInKB = llmin(avail_phys + sAllocatedMemInKB, sMaxHeapSizeInKB); + + if(sMaxPhysicalMemInKB > sAllocatedMemInKB) + { + sAvailPhysicalMemInKB = sMaxPhysicalMemInKB - sAllocatedMemInKB ; + } + else + { + sAvailPhysicalMemInKB = 0 ; + } +#else + //not valid for other systems for now. + sAllocatedMemInKB = (U32)(LLMemory::getCurrentRSS() / 1024) ; + sMaxPhysicalMemInKB = U32_MAX ; + sAvailPhysicalMemInKB = U32_MAX ; +#endif + + return ; +} + +// +//this function is to test if there is enough space with the size in the virtual address space. +//it does not do any real allocation +//if success, it returns the address where the memory chunk can fit in; +//otherwise it returns NULL. +// +//static +void* LLMemory::tryToAlloc(void* address, U32 size) +{ +#if LL_WINDOWS + address = VirtualAlloc(address, size, MEM_RESERVE | MEM_TOP_DOWN, PAGE_NOACCESS) ; + if(address) + { + if(!VirtualFree(address, 0, MEM_RELEASE)) + { + llerrs << "error happens when free some memory reservation." << llendl ; + } + } + return address ; +#else + return (void*)0x01 ; //skip checking +#endif +} + +//static +void LLMemory::logMemoryInfo(BOOL update) +{ + if(update) + { + updateMemoryInfo() ; + } + + llinfos << "Current allocated physical memory(KB): " << sAllocatedMemInKB << llendl ; + llinfos << "Current allocated page size (KB): " << sAllocatedPageSizeInKB << llendl ; + llinfos << "Current availabe physical memory(KB): " << sAvailPhysicalMemInKB << llendl ; + llinfos << "Current max usable memory(KB): " << sMaxPhysicalMemInKB << llendl ; + + llinfos << "--- private pool information -- " << llendl ; + llinfos << "Total reserved (KB): " << LLPrivateMemoryPoolManager::getInstance()->mTotalReservedSize / 1024 << llendl ; + llinfos << "Total allocated (KB): " << LLPrivateMemoryPoolManager::getInstance()->mTotalAllocatedSize / 1024 << llendl ; +} + +//return 0: everything is normal; +//return 1: the memory pool is low, but not in danger; +//return -1: the memory pool is in danger, is about to crash. +//static +bool LLMemory::isMemoryPoolLow() +{ + static const U32 LOW_MEMEOY_POOL_THRESHOLD_KB = 64 * 1024 ; //64 MB for emergency use + const static U32 MAX_SIZE_CHECKED_MEMORY_BLOCK = 64 * 1024 * 1024 ; //64 MB + static void* last_reserved_address = NULL ; + + if(!sEnableMemoryFailurePrevention) + { + return false ; //no memory failure prevention. + } + + if(sAvailPhysicalMemInKB < (LOW_MEMEOY_POOL_THRESHOLD_KB >> 2)) //out of physical memory + { + return true ; + } + + if(sAllocatedPageSizeInKB + (LOW_MEMEOY_POOL_THRESHOLD_KB >> 2) > sMaxHeapSizeInKB) //out of virtual address space. + { + return true ; + } + + bool is_low = (S32)(sAvailPhysicalMemInKB < LOW_MEMEOY_POOL_THRESHOLD_KB || + sAllocatedPageSizeInKB + LOW_MEMEOY_POOL_THRESHOLD_KB > sMaxHeapSizeInKB) ; + + //check the virtual address space fragmentation + if(!is_low) + { + if(!last_reserved_address) + { + last_reserved_address = LLMemory::tryToAlloc(last_reserved_address, MAX_SIZE_CHECKED_MEMORY_BLOCK) ; + } + else + { + last_reserved_address = LLMemory::tryToAlloc(last_reserved_address, MAX_SIZE_CHECKED_MEMORY_BLOCK) ; + if(!last_reserved_address) //failed, try once more + { + last_reserved_address = LLMemory::tryToAlloc(last_reserved_address, MAX_SIZE_CHECKED_MEMORY_BLOCK) ; + } + } + + is_low = !last_reserved_address ; //allocation failed + } + + return is_low ; +} + +//static +U32 LLMemory::getAvailableMemKB() +{ + return sAvailPhysicalMemInKB ; +} + +//static +U32 LLMemory::getMaxMemKB() +{ + return sMaxPhysicalMemInKB ; +} + +//static +U32 LLMemory::getAllocatedMemKB() +{ + return sAllocatedMemInKB ; +} + void* ll_allocate (size_t size) { if (size == 0) @@ -86,11 +254,6 @@ void* ll_allocate (size_t size) return p; } -void ll_release (void *p) -{ - free(p); -} - //---------------------------------------------------------------------------- #if defined(LL_WINDOWS) @@ -256,7 +419,7 @@ U64 LLMemory::getCurrentRSS() U32 LLMemory::getWorkingSetSize() { - return 0 ; + return 0; } #endif @@ -277,7 +440,7 @@ LLMemTracker::LLMemTracker() mDrawnIndex = 0 ; mPaused = FALSE ; - mMutexp = new LLMutex(NULL) ; + mMutexp = new LLMutex() ; mStringBuffer = new char*[128] ; mStringBuffer[0] = new char[mCapacity * 128] ; for(S32 i = 1 ; i < mCapacity ; i++) @@ -395,3 +558,1685 @@ const char* LLMemTracker::getNextLine() #endif //MEM_TRACK_MEM //-------------------------------------------------------------------------------------------------- + +//-------------------------------------------------------------------------------------------------- +//-------------------------------------------------------------------------------------------------- +//minimum slot size and minimal slot size interval +const U32 ATOMIC_MEM_SLOT = 16 ; //bytes + +//minimum block sizes (page size) for small allocation, medium allocation, large allocation +const U32 MIN_BLOCK_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {2 << 10, 4 << 10, 16 << 10} ; // + +//maximum block sizes for small allocation, medium allocation, large allocation +const U32 MAX_BLOCK_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {64 << 10, 1 << 20, 4 << 20} ; + +//minimum slot sizes for small allocation, medium allocation, large allocation +const U32 MIN_SLOT_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {ATOMIC_MEM_SLOT, 2 << 10, 512 << 10}; + +//maximum slot sizes for small allocation, medium allocation, large allocation +const U32 MAX_SLOT_SIZES[LLPrivateMemoryPool::SUPER_ALLOCATION] = {(2 << 10) - ATOMIC_MEM_SLOT, (512 - 2) << 10, 4 << 20}; + +//size of a block with multiple slots can not exceed CUT_OFF_SIZE +const U32 CUT_OFF_SIZE = (64 << 10) ; //64 KB + +//max number of slots in a block +const U32 MAX_NUM_SLOTS_IN_A_BLOCK = llmin(MIN_BLOCK_SIZES[0] / ATOMIC_MEM_SLOT, ATOMIC_MEM_SLOT * 8) ; + +//------------------------------------------------------------- +//align val to be integer times of ATOMIC_MEM_SLOT +U32 align(U32 val) +{ + U32 aligned = (val / ATOMIC_MEM_SLOT) * ATOMIC_MEM_SLOT ; + if(aligned < val) + { + aligned += ATOMIC_MEM_SLOT ; + } + + return aligned ; +} + +//------------------------------------------------------------- +//class LLPrivateMemoryPool::LLMemoryBlock +//------------------------------------------------------------- +// +//each memory block could fit for two page sizes: 0.75 * mSlotSize, which starts from the beginning of the memory chunk and grow towards the end of the +//the block; another is mSlotSize, which starts from the end of the block and grows towards the beginning of the block. +// +LLPrivateMemoryPool::LLMemoryBlock::LLMemoryBlock() +{ + //empty +} + +LLPrivateMemoryPool::LLMemoryBlock::~LLMemoryBlock() +{ + //empty +} + +//create and initialize a memory block +void LLPrivateMemoryPool::LLMemoryBlock::init(char* buffer, U32 buffer_size, U32 slot_size) +{ + mBuffer = buffer ; + mBufferSize = buffer_size ; + mSlotSize = slot_size ; + mTotalSlots = buffer_size / mSlotSize ; + + llassert_always(buffer_size / mSlotSize <= MAX_NUM_SLOTS_IN_A_BLOCK) ; //max number is 128 + + mAllocatedSlots = 0 ; + mDummySize = 0 ; + + //init the bit map. + //mark free bits + if(mTotalSlots > 32) //reserve extra space from mBuffer to store bitmap if needed. + { + mDummySize = ATOMIC_MEM_SLOT ; + mTotalSlots -= (mDummySize + mSlotSize - 1) / mSlotSize ; + mUsageBits = 0 ; + + S32 usage_bit_len = (mTotalSlots + 31) / 32 ; + + for(S32 i = 0 ; i < usage_bit_len - 1 ; i++) + { + *((U32*)mBuffer + i) = 0 ; + } + for(S32 i = usage_bit_len - 1 ; i < mDummySize / sizeof(U32) ; i++) + { + *((U32*)mBuffer + i) = 0xffffffff ; + } + + if(mTotalSlots & 31) + { + *((U32*)mBuffer + usage_bit_len - 2) = (0xffffffff << (mTotalSlots & 31)) ; + } + } + else//no extra bitmap space reserved + { + mUsageBits = 0 ; + if(mTotalSlots & 31) + { + mUsageBits = (0xffffffff << (mTotalSlots & 31)) ; + } + } + + mSelf = this ; + mNext = NULL ; + mPrev = NULL ; + + llassert_always(mTotalSlots > 0) ; +} + +//mark this block to be free with the memory [mBuffer, mBuffer + mBufferSize). +void LLPrivateMemoryPool::LLMemoryBlock::setBuffer(char* buffer, U32 buffer_size) +{ + mBuffer = buffer ; + mBufferSize = buffer_size ; + mSelf = NULL ; + mTotalSlots = 0 ; //set the block is free. +} + +//reserve a slot +char* LLPrivateMemoryPool::LLMemoryBlock::allocate() +{ + llassert_always(mAllocatedSlots < mTotalSlots) ; + + //find a free slot + U32* bits = NULL ; + U32 k = 0 ; + if(mUsageBits != 0xffffffff) + { + bits = &mUsageBits ; + } + else if(mDummySize > 0)//go to extra space + { + for(S32 i = 0 ; i < mDummySize / sizeof(U32); i++) + { + if(*((U32*)mBuffer + i) != 0xffffffff) + { + bits = (U32*)mBuffer + i ; + k = i + 1 ; + break ; + } + } + } + S32 idx = 0 ; + U32 tmp = *bits ; + for(; tmp & 1 ; tmp >>= 1, idx++) ; + + //set the slot reserved + if(!idx) + { + *bits |= 1 ; + } + else + { + *bits |= (1 << idx) ; + } + + mAllocatedSlots++ ; + + return mBuffer + mDummySize + (k * 32 + idx) * mSlotSize ; +} + +//free a slot +void LLPrivateMemoryPool::LLMemoryBlock::freeMem(void* addr) +{ + //bit index + U32 idx = ((U32)addr - (U32)mBuffer - mDummySize) / mSlotSize ; + + U32* bits = &mUsageBits ; + if(idx >= 32) + { + bits = (U32*)mBuffer + (idx - 32) / 32 ; + } + + //reset the bit + if(idx & 31) + { + *bits &= ~(1 << (idx & 31)) ; + } + else + { + *bits &= ~1 ; + } + + mAllocatedSlots-- ; +} + +//for debug use: reset the entire bitmap. +void LLPrivateMemoryPool::LLMemoryBlock::resetBitMap() +{ + for(S32 i = 0 ; i < mDummySize / sizeof(U32) ; i++) + { + *((U32*)mBuffer + i) = 0 ; + } + mUsageBits = 0 ; +} +//------------------------------------------------------------------- +//class LLMemoryChunk +//-------------------------------------------------------------------- +LLPrivateMemoryPool::LLMemoryChunk::LLMemoryChunk() +{ + //empty +} + +LLPrivateMemoryPool::LLMemoryChunk::~LLMemoryChunk() +{ + //empty +} + +//create and init a memory chunk +void LLPrivateMemoryPool::LLMemoryChunk::init(char* buffer, U32 buffer_size, U32 min_slot_size, U32 max_slot_size, U32 min_block_size, U32 max_block_size) +{ + mBuffer = buffer ; + mBufferSize = buffer_size ; + mAlloatedSize = 0 ; + + mMetaBuffer = mBuffer + sizeof(LLMemoryChunk) ; + + mMinBlockSize = min_block_size; //page size + mMinSlotSize = min_slot_size; + mMaxSlotSize = max_slot_size ; + mBlockLevels = mMaxSlotSize / mMinSlotSize ; + mPartitionLevels = max_block_size / mMinBlockSize + 1 ; + + S32 max_num_blocks = (buffer_size - sizeof(LLMemoryChunk) - mBlockLevels * sizeof(LLMemoryBlock*) - mPartitionLevels * sizeof(LLMemoryBlock*)) / + (mMinBlockSize + sizeof(LLMemoryBlock)) ; + //meta data space + mBlocks = (LLMemoryBlock*)mMetaBuffer ; //space reserved for all memory blocks. + mAvailBlockList = (LLMemoryBlock**)((char*)mBlocks + sizeof(LLMemoryBlock) * max_num_blocks) ; + mFreeSpaceList = (LLMemoryBlock**)((char*)mAvailBlockList + sizeof(LLMemoryBlock*) * mBlockLevels) ; + + //data buffer, which can be used for allocation + mDataBuffer = (char*)mFreeSpaceList + sizeof(LLMemoryBlock*) * mPartitionLevels ; + + //alignmnet + mDataBuffer = mBuffer + align(mDataBuffer - mBuffer) ; + + //init + for(U32 i = 0 ; i < mBlockLevels; i++) + { + mAvailBlockList[i] = NULL ; + } + for(U32 i = 0 ; i < mPartitionLevels ; i++) + { + mFreeSpaceList[i] = NULL ; + } + + //assign the entire chunk to the first block + mBlocks[0].mPrev = NULL ; + mBlocks[0].mNext = NULL ; + mBlocks[0].setBuffer(mDataBuffer, buffer_size - (mDataBuffer - mBuffer)) ; + addToFreeSpace(&mBlocks[0]) ; + + mNext = NULL ; + mPrev = NULL ; +} + +//static +U32 LLPrivateMemoryPool::LLMemoryChunk::getMaxOverhead(U32 data_buffer_size, U32 min_slot_size, + U32 max_slot_size, U32 min_block_size, U32 max_block_size) +{ + //for large allocations, reserve some extra memory for meta data to avoid wasting much + if(data_buffer_size / min_slot_size < 64) //large allocations + { + U32 overhead = sizeof(LLMemoryChunk) + (data_buffer_size / min_block_size) * sizeof(LLMemoryBlock) + + sizeof(LLMemoryBlock*) * (max_slot_size / min_slot_size) + sizeof(LLMemoryBlock*) * (max_block_size / min_block_size + 1) ; + + //round to integer times of min_block_size + overhead = ((overhead + min_block_size - 1) / min_block_size) * min_block_size ; + return overhead ; + } + else + { + return 0 ; //do not reserve extra overhead if for small allocations + } +} + +char* LLPrivateMemoryPool::LLMemoryChunk::allocate(U32 size) +{ + if(mMinSlotSize > size) + { + size = mMinSlotSize ; + } + if(mAlloatedSize + size > mBufferSize - (mDataBuffer - mBuffer)) + { + return NULL ; //no enough space in this chunk. + } + + char* p = NULL ; + U32 blk_idx = getBlockLevel(size); + + LLMemoryBlock* blk = NULL ; + + //check if there is free block available + if(mAvailBlockList[blk_idx]) + { + blk = mAvailBlockList[blk_idx] ; + p = blk->allocate() ; + + if(blk->isFull()) + { + popAvailBlockList(blk_idx) ; + } + } + + //ask for a new block + if(!p) + { + blk = addBlock(blk_idx) ; + if(blk) + { + p = blk->allocate() ; + + if(blk->isFull()) + { + popAvailBlockList(blk_idx) ; + } + } + } + + //ask for space from larger blocks + if(!p) + { + for(S32 i = blk_idx + 1 ; i < mBlockLevels; i++) + { + if(mAvailBlockList[i]) + { + blk = mAvailBlockList[i] ; + p = blk->allocate() ; + + if(blk->isFull()) + { + popAvailBlockList(i) ; + } + break ; + } + } + } + + if(p && blk) + { + mAlloatedSize += blk->getSlotSize() ; + } + return p ; +} + +void LLPrivateMemoryPool::LLMemoryChunk::freeMem(void* addr) +{ + U32 blk_idx = getPageIndex((U32)addr) ; + LLMemoryBlock* blk = (LLMemoryBlock*)(mMetaBuffer + blk_idx * sizeof(LLMemoryBlock)) ; + blk = blk->mSelf ; + + bool was_full = blk->isFull() ; + blk->freeMem(addr) ; + mAlloatedSize -= blk->getSlotSize() ; + + if(blk->empty()) + { + removeBlock(blk) ; + } + else if(was_full) + { + addToAvailBlockList(blk) ; + } +} + +bool LLPrivateMemoryPool::LLMemoryChunk::empty() +{ + return !mAlloatedSize ; +} + +bool LLPrivateMemoryPool::LLMemoryChunk::containsAddress(const char* addr) const +{ + return (U32)mBuffer <= (U32)addr && (U32)mBuffer + mBufferSize > (U32)addr ; +} + +//debug use +void LLPrivateMemoryPool::LLMemoryChunk::dump() +{ +#if 0 + //sanity check + //for(S32 i = 0 ; i < mBlockLevels ; i++) + //{ + // LLMemoryBlock* blk = mAvailBlockList[i] ; + // while(blk) + // { + // blk_list.push_back(blk) ; + // blk = blk->mNext ; + // } + //} + for(S32 i = 0 ; i < mPartitionLevels ; i++) + { + LLMemoryBlock* blk = mFreeSpaceList[i] ; + while(blk) + { + blk_list.push_back(blk) ; + blk = blk->mNext ; + } + } + + std::sort(blk_list.begin(), blk_list.end(), LLMemoryBlock::CompareAddress()); + + U32 total_size = blk_list[0]->getBufferSize() ; + for(U32 i = 1 ; i < blk_list.size(); i++) + { + total_size += blk_list[i]->getBufferSize() ; + if((U32)blk_list[i]->getBuffer() < (U32)blk_list[i-1]->getBuffer() + blk_list[i-1]->getBufferSize()) + { + llerrs << "buffer corrupted." << llendl ; + } + } + + llassert_always(total_size + mMinBlockSize >= mBufferSize - ((U32)mDataBuffer - (U32)mBuffer)) ; + + U32 blk_num = (mBufferSize - (mDataBuffer - mBuffer)) / mMinBlockSize ; + for(U32 i = 0 ; i < blk_num ; ) + { + LLMemoryBlock* blk = &mBlocks[i] ; + if(blk->mSelf) + { + U32 end = blk->getBufferSize() / mMinBlockSize ; + for(U32 j = 0 ; j < end ; j++) + { + llassert_always(blk->mSelf == blk || !blk->mSelf) ; + } + i += end ; + } + else + { + llerrs << "gap happens" << llendl ; + } + } +#endif +#if 0 + llinfos << "---------------------------" << llendl ; + llinfos << "Chunk buffer: " << (U32)getBuffer() << " size: " << getBufferSize() << llendl ; + + llinfos << "available blocks ... " << llendl ; + for(S32 i = 0 ; i < mBlockLevels ; i++) + { + LLMemoryBlock* blk = mAvailBlockList[i] ; + while(blk) + { + llinfos << "blk buffer " << (U32)blk->getBuffer() << " size: " << blk->getBufferSize() << llendl ; + blk = blk->mNext ; + } + } + + llinfos << "free blocks ... " << llendl ; + for(S32 i = 0 ; i < mPartitionLevels ; i++) + { + LLMemoryBlock* blk = mFreeSpaceList[i] ; + while(blk) + { + llinfos << "blk buffer " << (U32)blk->getBuffer() << " size: " << blk->getBufferSize() << llendl ; + blk = blk->mNext ; + } + } +#endif +} + +//compute the size for a block, the size is round to integer times of mMinBlockSize. +U32 LLPrivateMemoryPool::LLMemoryChunk::calcBlockSize(U32 slot_size) +{ + // + //Note: we try to make a block to have 32 slots if the size is not over 32 pages + //32 is the number of bits of an integer in a 32-bit system + // + + U32 block_size; + U32 cut_off_size = llmin(CUT_OFF_SIZE, (U32)(mMinBlockSize << 5)) ; + + if((slot_size << 5) <= mMinBlockSize)//for small allocations, return one page + { + block_size = mMinBlockSize ; + } + else if(slot_size >= cut_off_size)//for large allocations, return one-slot block + { + block_size = (slot_size / mMinBlockSize) * mMinBlockSize ; + if(block_size < slot_size) + { + block_size += mMinBlockSize ; + } + } + else //medium allocations + { + if((slot_size << 5) >= cut_off_size) + { + block_size = cut_off_size ; + } + else + { + block_size = ((slot_size << 5) / mMinBlockSize) * mMinBlockSize ; + } + } + + llassert_always(block_size >= slot_size) ; + + return block_size ; +} + +//create a new block in the chunk +LLPrivateMemoryPool::LLMemoryBlock* LLPrivateMemoryPool::LLMemoryChunk::addBlock(U32 blk_idx) +{ + U32 slot_size = mMinSlotSize * (blk_idx + 1) ; + U32 preferred_block_size = calcBlockSize(slot_size) ; + U16 idx = getPageLevel(preferred_block_size); + LLMemoryBlock* blk = NULL ; + + if(mFreeSpaceList[idx])//if there is free slot for blk_idx + { + blk = createNewBlock(mFreeSpaceList[idx], preferred_block_size, slot_size, blk_idx) ; + } + else if(mFreeSpaceList[mPartitionLevels - 1]) //search free pool + { + blk = createNewBlock(mFreeSpaceList[mPartitionLevels - 1], preferred_block_size, slot_size, blk_idx) ; + } + else //search for other non-preferred but enough space slot. + { + S32 min_idx = 0 ; + if(slot_size > mMinBlockSize) + { + min_idx = getPageLevel(slot_size) ; + } + for(S32 i = (S32)idx - 1 ; i >= min_idx ; i--) //search the small slots first + { + if(mFreeSpaceList[i]) + { + U32 new_preferred_block_size = mFreeSpaceList[i]->getBufferSize(); + new_preferred_block_size = (new_preferred_block_size / mMinBlockSize) * mMinBlockSize ; //round to integer times of mMinBlockSize. + + //create a NEW BLOCK THERE. + if(new_preferred_block_size >= slot_size) //at least there is space for one slot. + { + + blk = createNewBlock(mFreeSpaceList[i], new_preferred_block_size, slot_size, blk_idx) ; + } + break ; + } + } + + if(!blk) + { + for(U16 i = idx + 1 ; i < mPartitionLevels - 1; i++) //search the large slots + { + if(mFreeSpaceList[i]) + { + //create a NEW BLOCK THERE. + blk = createNewBlock(mFreeSpaceList[i], preferred_block_size, slot_size, blk_idx) ; + break ; + } + } + } + } + + return blk ; +} + +//create a new block at the designed location +LLPrivateMemoryPool::LLMemoryBlock* LLPrivateMemoryPool::LLMemoryChunk::createNewBlock(LLMemoryBlock* blk, U32 buffer_size, U32 slot_size, U32 blk_idx) +{ + //unlink from the free space + removeFromFreeSpace(blk) ; + + //check the rest space + U32 new_free_blk_size = blk->getBufferSize() - buffer_size ; + if(new_free_blk_size < mMinBlockSize) //can not partition the memory into size smaller than mMinBlockSize + { + new_free_blk_size = 0 ; //discard the last small extra space. + } + + //add the rest space back to the free list + if(new_free_blk_size > 0) //blk still has free space + { + LLMemoryBlock* next_blk = blk + (buffer_size / mMinBlockSize) ; + next_blk->mPrev = NULL ; + next_blk->mNext = NULL ; + next_blk->setBuffer(blk->getBuffer() + buffer_size, new_free_blk_size) ; + addToFreeSpace(next_blk) ; + } + + blk->init(blk->getBuffer(), buffer_size, slot_size) ; + //insert to the available block list... + mAvailBlockList[blk_idx] = blk ; + + //mark the address map: all blocks covered by this block space pointing back to this block. + U32 end = (buffer_size / mMinBlockSize) ; + for(U32 i = 1 ; i < end ; i++) + { + (blk + i)->mSelf = blk ; + } + + return blk ; +} + +//delete a block, release the block to the free pool. +void LLPrivateMemoryPool::LLMemoryChunk::removeBlock(LLMemoryBlock* blk) +{ + //remove from the available block list + if(blk->mPrev) + { + blk->mPrev->mNext = blk->mNext ; + } + if(blk->mNext) + { + blk->mNext->mPrev = blk->mPrev ; + } + U32 blk_idx = getBlockLevel(blk->getSlotSize()); + if(mAvailBlockList[blk_idx] == blk) + { + mAvailBlockList[blk_idx] = blk->mNext ; + } + + blk->mNext = NULL ; + blk->mPrev = NULL ; + + //mark it free + blk->setBuffer(blk->getBuffer(), blk->getBufferSize()) ; + +#if 1 + //merge blk with neighbors if possible + if(blk->getBuffer() > mDataBuffer) //has the left neighbor + { + if((blk - 1)->mSelf->isFree()) + { + LLMemoryBlock* left_blk = (blk - 1)->mSelf ; + removeFromFreeSpace((blk - 1)->mSelf); + left_blk->setBuffer(left_blk->getBuffer(), left_blk->getBufferSize() + blk->getBufferSize()) ; + blk = left_blk ; + } + } + if(blk->getBuffer() + blk->getBufferSize() <= mBuffer + mBufferSize - mMinBlockSize) //has the right neighbor + { + U32 d = blk->getBufferSize() / mMinBlockSize ; + if((blk + d)->isFree()) + { + LLMemoryBlock* right_blk = blk + d ; + removeFromFreeSpace(blk + d) ; + blk->setBuffer(blk->getBuffer(), blk->getBufferSize() + right_blk->getBufferSize()) ; + } + } +#endif + + addToFreeSpace(blk) ; + + return ; +} + +//the top block in the list is full, pop it out of the list +void LLPrivateMemoryPool::LLMemoryChunk::popAvailBlockList(U32 blk_idx) +{ + if(mAvailBlockList[blk_idx]) + { + LLMemoryBlock* next = mAvailBlockList[blk_idx]->mNext ; + if(next) + { + next->mPrev = NULL ; + } + mAvailBlockList[blk_idx]->mPrev = NULL ; + mAvailBlockList[blk_idx]->mNext = NULL ; + mAvailBlockList[blk_idx] = next ; + } +} + +//add the block back to the free pool +void LLPrivateMemoryPool::LLMemoryChunk::addToFreeSpace(LLMemoryBlock* blk) +{ + llassert_always(!blk->mPrev) ; + llassert_always(!blk->mNext) ; + + U16 free_idx = blk->getBufferSize() / mMinBlockSize - 1; + + (blk + free_idx)->mSelf = blk ; //mark the end pointing back to the head. + free_idx = llmin(free_idx, (U16)(mPartitionLevels - 1)) ; + + blk->mNext = mFreeSpaceList[free_idx] ; + if(mFreeSpaceList[free_idx]) + { + mFreeSpaceList[free_idx]->mPrev = blk ; + } + mFreeSpaceList[free_idx] = blk ; + blk->mPrev = NULL ; + blk->mSelf = blk ; + + return ; +} + +//remove the space from the free pool +void LLPrivateMemoryPool::LLMemoryChunk::removeFromFreeSpace(LLMemoryBlock* blk) +{ + U16 free_idx = blk->getBufferSize() / mMinBlockSize - 1; + free_idx = llmin(free_idx, (U16)(mPartitionLevels - 1)) ; + + if(mFreeSpaceList[free_idx] == blk) + { + mFreeSpaceList[free_idx] = blk->mNext ; + } + if(blk->mPrev) + { + blk->mPrev->mNext = blk->mNext ; + } + if(blk->mNext) + { + blk->mNext->mPrev = blk->mPrev ; + } + blk->mNext = NULL ; + blk->mPrev = NULL ; + blk->mSelf = NULL ; + + return ; +} + +void LLPrivateMemoryPool::LLMemoryChunk::addToAvailBlockList(LLMemoryBlock* blk) +{ + llassert_always(!blk->mPrev) ; + llassert_always(!blk->mNext) ; + + U32 blk_idx = getBlockLevel(blk->getSlotSize()); + + blk->mNext = mAvailBlockList[blk_idx] ; + if(blk->mNext) + { + blk->mNext->mPrev = blk ; + } + blk->mPrev = NULL ; + mAvailBlockList[blk_idx] = blk ; + + return ; +} + +U32 LLPrivateMemoryPool::LLMemoryChunk::getPageIndex(U32 addr) +{ + return (addr - (U32)mDataBuffer) / mMinBlockSize ; +} + +//for mAvailBlockList +U32 LLPrivateMemoryPool::LLMemoryChunk::getBlockLevel(U32 size) +{ + llassert(size >= mMinSlotSize && size <= mMaxSlotSize) ; + + //start from 0 + return (size + mMinSlotSize - 1) / mMinSlotSize - 1 ; +} + +//for mFreeSpaceList +U16 LLPrivateMemoryPool::LLMemoryChunk::getPageLevel(U32 size) +{ + //start from 0 + U16 level = size / mMinBlockSize - 1 ; + if(level >= mPartitionLevels) + { + level = mPartitionLevels - 1 ; + } + return level ; +} + +//------------------------------------------------------------------- +//class LLPrivateMemoryPool +//-------------------------------------------------------------------- +const U32 CHUNK_SIZE = 4 << 20 ; //4 MB +const U32 LARGE_CHUNK_SIZE = 4 * CHUNK_SIZE ; //16 MB +LLPrivateMemoryPool::LLPrivateMemoryPool(S32 type, U32 max_pool_size) : + mMutexp(NULL), + mReservedPoolSize(0), + mHashFactor(1), + mType(type), + mMaxPoolSize(max_pool_size) +{ + if(type == STATIC_THREADED || type == VOLATILE_THREADED) + { + mMutexp = new LLMutex ; + } + + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + mChunkList[i] = NULL ; + } + + mNumOfChunks = 0 ; +} + +LLPrivateMemoryPool::~LLPrivateMemoryPool() +{ + destroyPool(); + delete mMutexp ; +} + +char* LLPrivateMemoryPool::allocate(U32 size) +{ + if(!size) + { + return NULL ; + } + + //if the asked size larger than MAX_BLOCK_SIZE, fetch from heap directly, the pool does not manage it + if(size >= CHUNK_SIZE) + { + return (char*)malloc(size) ; + } + + char* p = NULL ; + + //find the appropriate chunk + S32 chunk_idx = getChunkIndex(size) ; + + lock() ; + + LLMemoryChunk* chunk = mChunkList[chunk_idx]; + while(chunk) + { + if((p = chunk->allocate(size))) + { + break ; + } + chunk = chunk->mNext ; + } + + //fetch new memory chunk + if(!p) + { + if(mReservedPoolSize + CHUNK_SIZE > mMaxPoolSize) + { + chunk = mChunkList[chunk_idx]; + while(chunk) + { + if((p = chunk->allocate(size))) + { + break ; + } + chunk = chunk->mNext ; + } + } + else + { + chunk = addChunk(chunk_idx) ; + if(chunk) + { + p = chunk->allocate(size) ; + } + } + } + + unlock() ; + + if(!p) //to get memory from the private pool failed, try the heap directly + { + static bool to_log = true ; + + if(to_log) + { + llwarns << "The memory pool overflows, now using heap directly!" << llendl ; + to_log = false ; + } + + return (char*)malloc(size) ; + } + + return p ; +} + +void LLPrivateMemoryPool::freeMem(void* addr) +{ + if(!addr) + { + return ; + } + + lock() ; + + LLMemoryChunk* chunk = findChunk((char*)addr) ; + + if(!chunk) + { + free(addr) ; //release from heap + } + else + { + chunk->freeMem(addr) ; + + if(chunk->empty()) + { + removeChunk(chunk) ; + } + } + + unlock() ; +} + +void LLPrivateMemoryPool::dump() +{ +} + +U32 LLPrivateMemoryPool::getTotalAllocatedSize() +{ + U32 total_allocated = 0 ; + + LLMemoryChunk* chunk ; + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + chunk = mChunkList[i]; + while(chunk) + { + total_allocated += chunk->getAllocatedSize() ; + chunk = chunk->mNext ; + } + } + + return total_allocated ; +} + +void LLPrivateMemoryPool::lock() +{ + if(mMutexp) + { + mMutexp->lock() ; + } +} + +void LLPrivateMemoryPool::unlock() +{ + if(mMutexp) + { + mMutexp->unlock() ; + } +} + +S32 LLPrivateMemoryPool::getChunkIndex(U32 size) +{ + S32 i ; + for(i = 0 ; size > MAX_SLOT_SIZES[i]; i++); + + llassert_always(i < SUPER_ALLOCATION); + + return i ; +} + +//destroy the entire pool +void LLPrivateMemoryPool::destroyPool() +{ + lock() ; + + if(mNumOfChunks > 0) + { + llwarns << "There is some memory not freed when destroy the memory pool!" << llendl ; + } + + mNumOfChunks = 0 ; + mChunkHashList.clear() ; + mHashFactor = 1 ; + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + mChunkList[i] = NULL ; + } + + unlock() ; +} + +bool LLPrivateMemoryPool::checkSize(U32 asked_size) +{ + if(mReservedPoolSize + asked_size > mMaxPoolSize) + { + llinfos << "Max pool size: " << mMaxPoolSize << llendl ; + llinfos << "Total reserved size: " << mReservedPoolSize + asked_size << llendl ; + llinfos << "Total_allocated Size: " << getTotalAllocatedSize() << llendl ; + + //llerrs << "The pool is overflowing..." << llendl ; + + return false ; + } + + return true ; +} + +LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::addChunk(S32 chunk_index) +{ + U32 preferred_size ; + U32 overhead ; + if(chunk_index < LARGE_ALLOCATION) + { + preferred_size = CHUNK_SIZE ; //4MB + overhead = LLMemoryChunk::getMaxOverhead(preferred_size, MIN_SLOT_SIZES[chunk_index], + MAX_SLOT_SIZES[chunk_index], MIN_BLOCK_SIZES[chunk_index], MAX_BLOCK_SIZES[chunk_index]) ; + } + else + { + preferred_size = LARGE_CHUNK_SIZE ; //16MB + overhead = LLMemoryChunk::getMaxOverhead(preferred_size, MIN_SLOT_SIZES[chunk_index], + MAX_SLOT_SIZES[chunk_index], MIN_BLOCK_SIZES[chunk_index], MAX_BLOCK_SIZES[chunk_index]) ; + } + + if(!checkSize(preferred_size + overhead)) + { + return NULL ; + } + + mReservedPoolSize += preferred_size + overhead ; + + char* buffer = (char*)malloc(preferred_size + overhead) ; + if(!buffer) + { + return NULL ; + } + + LLMemoryChunk* chunk = new (buffer) LLMemoryChunk() ; + chunk->init(buffer, preferred_size + overhead, MIN_SLOT_SIZES[chunk_index], + MAX_SLOT_SIZES[chunk_index], MIN_BLOCK_SIZES[chunk_index], MAX_BLOCK_SIZES[chunk_index]) ; + + //add to the tail of the linked list + { + if(!mChunkList[chunk_index]) + { + mChunkList[chunk_index] = chunk ; + } + else + { + LLMemoryChunk* cur = mChunkList[chunk_index] ; + while(cur->mNext) + { + cur = cur->mNext ; + } + cur->mNext = chunk ; + chunk->mPrev = cur ; + } + } + + //insert into the hash table + addToHashTable(chunk) ; + + mNumOfChunks++; + + return chunk ; +} + +void LLPrivateMemoryPool::removeChunk(LLMemoryChunk* chunk) +{ + if(!chunk) + { + return ; + } + + //remove from the linked list + for(S32 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + if(mChunkList[i] == chunk) + { + mChunkList[i] = chunk->mNext ; + } + } + + if(chunk->mPrev) + { + chunk->mPrev->mNext = chunk->mNext ; + } + if(chunk->mNext) + { + chunk->mNext->mPrev = chunk->mPrev ; + } + + //remove from the hash table + removeFromHashTable(chunk) ; + + mNumOfChunks--; + mReservedPoolSize -= chunk->getBufferSize() ; + + //release memory + free(chunk->getBuffer()) ; +} + +U16 LLPrivateMemoryPool::findHashKey(const char* addr) +{ + return (((U32)addr) / CHUNK_SIZE) % mHashFactor ; +} + +LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::findChunk(const char* addr) +{ + U16 key = findHashKey(addr) ; + if(mChunkHashList.size() <= key) + { + return NULL ; + } + + return mChunkHashList[key].findChunk(addr) ; +} + +void LLPrivateMemoryPool::addToHashTable(LLMemoryChunk* chunk) +{ + static const U16 HASH_FACTORS[] = {41, 83, 193, 317, 419, 523, 719, 997, 1523, 0xFFFF}; + + U16 i ; + if(mChunkHashList.empty()) + { + mHashFactor = HASH_FACTORS[0] ; + rehash() ; + } + + U16 start_key = findHashKey(chunk->getBuffer()) ; + U16 end_key = findHashKey(chunk->getBuffer() + chunk->getBufferSize() - 1) ; + bool need_rehash = false ; + + if(mChunkHashList[start_key].hasElement(chunk)) + { + return; //already inserted. + } + need_rehash = mChunkHashList[start_key].add(chunk) ; + + if(start_key == end_key && !need_rehash) + { + return ; //done + } + + if(!need_rehash) + { + need_rehash = mChunkHashList[end_key].add(chunk) ; + } + + if(!need_rehash) + { + if(end_key < start_key) + { + need_rehash = fillHashTable(start_key + 1, mHashFactor, chunk) ; + if(!need_rehash) + { + need_rehash = fillHashTable(0, end_key, chunk) ; + } + } + else + { + need_rehash = fillHashTable(start_key + 1, end_key, chunk) ; + } + } + + if(need_rehash) + { + i = 0 ; + while(HASH_FACTORS[i] <= mHashFactor) i++; + + mHashFactor = HASH_FACTORS[i] ; + llassert_always(mHashFactor != 0xFFFF) ;//stop point to prevent endlessly recursive calls + + rehash() ; + } +} + +void LLPrivateMemoryPool::removeFromHashTable(LLMemoryChunk* chunk) +{ + U16 start_key = findHashKey(chunk->getBuffer()) ; + U16 end_key = findHashKey(chunk->getBuffer() + chunk->getBufferSize() - 1) ; + + mChunkHashList[start_key].remove(chunk) ; + if(start_key == end_key) + { + return ; //done + } + + mChunkHashList[end_key].remove(chunk) ; + + if(end_key < start_key) + { + for(U16 i = start_key + 1 ; i < mHashFactor; i++) + { + mChunkHashList[i].remove(chunk) ; + } + for(U16 i = 0 ; i < end_key; i++) + { + mChunkHashList[i].remove(chunk) ; + } + } + else + { + for(U16 i = start_key + 1 ; i < end_key; i++) + { + mChunkHashList[i].remove(chunk) ; + } + } +} + +void LLPrivateMemoryPool::rehash() +{ + llinfos << "new hash factor: " << mHashFactor << llendl ; + + mChunkHashList.clear() ; + mChunkHashList.resize(mHashFactor) ; + + LLMemoryChunk* chunk ; + for(U16 i = 0 ; i < SUPER_ALLOCATION ; i++) + { + chunk = mChunkList[i] ; + while(chunk) + { + addToHashTable(chunk) ; + chunk = chunk->mNext ; + } + } +} + +bool LLPrivateMemoryPool::fillHashTable(U16 start, U16 end, LLMemoryChunk* chunk) +{ + for(U16 i = start; i < end; i++) + { + if(mChunkHashList[i].add(chunk)) + { + return true ; + } + } + + return false ; +} + +//-------------------------------------------------------------------- +// class LLChunkHashElement +//-------------------------------------------------------------------- +LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::LLChunkHashElement::findChunk(const char* addr) +{ + if(mFirst && mFirst->containsAddress(addr)) + { + return mFirst ; + } + else if(mSecond && mSecond->containsAddress(addr)) + { + return mSecond ; + } + + return NULL ; +} + +//return false if successfully inserted to the hash slot. +bool LLPrivateMemoryPool::LLChunkHashElement::add(LLPrivateMemoryPool::LLMemoryChunk* chunk) +{ + llassert_always(!hasElement(chunk)) ; + + if(!mFirst) + { + mFirst = chunk ; + } + else if(!mSecond) + { + mSecond = chunk ; + } + else + { + return true ; //failed + } + + return false ; +} + +void LLPrivateMemoryPool::LLChunkHashElement::remove(LLPrivateMemoryPool::LLMemoryChunk* chunk) +{ + if(mFirst == chunk) + { + mFirst = NULL ; + } + else if(mSecond ==chunk) + { + mSecond = NULL ; + } + else + { + llerrs << "This slot does not contain this chunk!" << llendl ; + } +} + +//-------------------------------------------------------------------- +//class LLPrivateMemoryPoolManager +//-------------------------------------------------------------------- +LLPrivateMemoryPoolManager* LLPrivateMemoryPoolManager::sInstance = NULL ; + +LLPrivateMemoryPoolManager::LLPrivateMemoryPoolManager(BOOL enabled, U32 max_pool_size) +{ + mPoolList.resize(LLPrivateMemoryPool::MAX_TYPES) ; + + for(S32 i = 0 ; i < LLPrivateMemoryPool::MAX_TYPES; i++) + { + mPoolList[i] = NULL ; + } + + mPrivatePoolEnabled = enabled ; + + const U32 MAX_POOL_SIZE = 256 * 1024 * 1024 ; //256 MB + mMaxPrivatePoolSize = llmax(max_pool_size, MAX_POOL_SIZE) ; +} + +LLPrivateMemoryPoolManager::~LLPrivateMemoryPoolManager() +{ + +#if __DEBUG_PRIVATE_MEM__ + if(!sMemAllocationTracker.empty()) + { + llwarns << "there is potential memory leaking here. The list of not freed memory blocks are from: " <<llendl ; + + S32 k = 0 ; + for(mem_allocation_info_t::iterator iter = sMemAllocationTracker.begin() ; iter != sMemAllocationTracker.end() ; ++iter) + { + llinfos << k++ << ", " << iter->second << llendl ; + } + sMemAllocationTracker.clear() ; + } +#endif + +#if 0 + //all private pools should be released by their owners before reaching here. + for(S32 i = 0 ; i < LLPrivateMemoryPool::MAX_TYPES; i++) + { + llassert_always(!mPoolList[i]) ; + } + mPoolList.clear() ; + +#else + //forcefully release all memory + for(S32 i = 0 ; i < LLPrivateMemoryPool::MAX_TYPES; i++) + { + if(mPoolList[i]) + { + delete mPoolList[i] ; + mPoolList[i] = NULL ; + } + } + mPoolList.clear() ; +#endif +} + +//static +void LLPrivateMemoryPoolManager::initClass(BOOL enabled, U32 max_pool_size) +{ + llassert_always(!sInstance) ; + + sInstance = new LLPrivateMemoryPoolManager(enabled, max_pool_size) ; +} + +//static +LLPrivateMemoryPoolManager* LLPrivateMemoryPoolManager::getInstance() +{ + //if(!sInstance) + //{ + // sInstance = new LLPrivateMemoryPoolManager(FALSE) ; + //} + return sInstance ; +} + +//static +void LLPrivateMemoryPoolManager::destroyClass() +{ + if(sInstance) + { + delete sInstance ; + sInstance = NULL ; + } +} + +LLPrivateMemoryPool* LLPrivateMemoryPoolManager::newPool(S32 type) +{ + if(!mPrivatePoolEnabled) + { + return NULL ; + } + + if(!mPoolList[type]) + { + mPoolList[type] = new LLPrivateMemoryPool(type, mMaxPrivatePoolSize) ; + } + + return mPoolList[type] ; +} + +void LLPrivateMemoryPoolManager::deletePool(LLPrivateMemoryPool* pool) +{ + if(pool && pool->isEmpty()) + { + mPoolList[pool->getType()] = NULL ; + delete pool; + } +} + +//debug +void LLPrivateMemoryPoolManager::updateStatistics() +{ + mTotalReservedSize = 0 ; + mTotalAllocatedSize = 0 ; + + for(U32 i = 0; i < mPoolList.size(); i++) + { + if(mPoolList[i]) + { + mTotalReservedSize += mPoolList[i]->getTotalReservedSize() ; + mTotalAllocatedSize += mPoolList[i]->getTotalAllocatedSize() ; + } + } +} + +#if __DEBUG_PRIVATE_MEM__ +//static +char* LLPrivateMemoryPoolManager::allocate(LLPrivateMemoryPool* poolp, U32 size, const char* function, const int line) +{ + char* p ; + + if(!poolp) + { + p = (char*)malloc(size) ; + } + else + { + p = poolp->allocate(size) ; + } + + if(p) + { + char num[16] ; + sprintf(num, " line: %d ", line) ; + std::string str(function) ; + str += num; + + sMemAllocationTracker[p] = str ; + } + + return p ; +} +#else +//static +char* LLPrivateMemoryPoolManager::allocate(LLPrivateMemoryPool* poolp, U32 size) +{ + if(poolp) + { + return poolp->allocate(size) ; + } + else + { + return (char*)malloc(size) ; + } +} +#endif + +//static +void LLPrivateMemoryPoolManager::freeMem(LLPrivateMemoryPool* poolp, void* addr) +{ + if(!addr) + { + return ; + } + +#if __DEBUG_PRIVATE_MEM__ + sMemAllocationTracker.erase((char*)addr) ; +#endif + + if(poolp) + { + poolp->freeMem(addr) ; + } + else + { + free(addr) ; + } +} + +//-------------------------------------------------------------------- +//class LLPrivateMemoryPoolTester +//-------------------------------------------------------------------- +#if 0 +LLPrivateMemoryPoolTester* LLPrivateMemoryPoolTester::sInstance = NULL ; +LLPrivateMemoryPool* LLPrivateMemoryPoolTester::sPool = NULL ; +LLPrivateMemoryPoolTester::LLPrivateMemoryPoolTester() +{ +} + +LLPrivateMemoryPoolTester::~LLPrivateMemoryPoolTester() +{ +} + +//static +LLPrivateMemoryPoolTester* LLPrivateMemoryPoolTester::getInstance() +{ + if(!sInstance) + { + sInstance = ::new LLPrivateMemoryPoolTester() ; + } + return sInstance ; +} + +//static +void LLPrivateMemoryPoolTester::destroy() +{ + if(sInstance) + { + ::delete sInstance ; + sInstance = NULL ; + } + + if(sPool) + { + LLPrivateMemoryPoolManager::getInstance()->deletePool(sPool) ; + sPool = NULL ; + } +} + +void LLPrivateMemoryPoolTester::run(S32 type) +{ + if(sPool) + { + LLPrivateMemoryPoolManager::getInstance()->deletePool(sPool) ; + } + sPool = LLPrivateMemoryPoolManager::getInstance()->newPool(type) ; + + //run the test + correctnessTest() ; + performanceTest() ; + //fragmentationtest() ; + + //release pool. + LLPrivateMemoryPoolManager::getInstance()->deletePool(sPool) ; + sPool = NULL ; +} + +void LLPrivateMemoryPoolTester::test(U32 min_size, U32 max_size, U32 stride, U32 times, + bool random_deletion, bool output_statistics) +{ + U32 levels = (max_size - min_size) / stride + 1 ; + char*** p ; + U32 i, j ; + U32 total_allocated_size = 0 ; + + //allocate space for p ; + if(!(p = ::new char**[times]) || !(*p = ::new char*[times * levels])) + { + llerrs << "memory initialization for p failed" << llendl ; + } + + //init + for(i = 0 ; i < times; i++) + { + p[i] = *p + i * levels ; + for(j = 0 ; j < levels; j++) + { + p[i][j] = NULL ; + } + } + + //allocation + U32 size ; + for(i = 0 ; i < times ; i++) + { + for(j = 0 ; j < levels; j++) + { + size = min_size + j * stride ; + p[i][j] = ALLOCATE_MEM(sPool, size) ; + + total_allocated_size+= size ; + + *(U32*)p[i][j] = i ; + *((U32*)p[i][j] + 1) = j ; + //p[i][j][size - 1] = '\0' ; //access the last element to verify the success of the allocation. + + //randomly release memory + if(random_deletion) + { + S32 k = rand() % levels ; + + if(p[i][k]) + { + llassert_always(*(U32*)p[i][k] == i && *((U32*)p[i][k] + 1) == k) ; + FREE_MEM(sPool, p[i][k]) ; + total_allocated_size -= min_size + k * stride ; + p[i][k] = NULL ; + } + } + } + } + + //output pool allocation statistics + if(output_statistics) + { + } + + //release all memory allocations + for(i = 0 ; i < times; i++) + { + for(j = 0 ; j < levels; j++) + { + if(p[i][j]) + { + llassert_always(*(U32*)p[i][j] == i && *((U32*)p[i][j] + 1) == j) ; + FREE_MEM(sPool, p[i][j]) ; + total_allocated_size -= min_size + j * stride ; + p[i][j] = NULL ; + } + } + } + + ::delete[] *p ; + ::delete[] p ; +} + +void LLPrivateMemoryPoolTester::testAndTime(U32 size, U32 times) +{ + LLTimer timer ; + + llinfos << " -**********************- " << llendl ; + llinfos << "test size: " << size << " test times: " << times << llendl ; + + timer.reset() ; + char** p = new char*[times] ; + + //using the customized memory pool + //allocation + for(U32 i = 0 ; i < times; i++) + { + p[i] = ALLOCATE_MEM(sPool, size) ; + if(!p[i]) + { + llerrs << "allocation failed" << llendl ; + } + } + //de-allocation + for(U32 i = 0 ; i < times; i++) + { + FREE_MEM(sPool, p[i]) ; + p[i] = NULL ; + } + llinfos << "time spent using customized memory pool: " << timer.getElapsedTimeF32() << llendl ; + + timer.reset() ; + + //using the standard allocator/de-allocator: + //allocation + for(U32 i = 0 ; i < times; i++) + { + p[i] = ::new char[size] ; + if(!p[i]) + { + llerrs << "allocation failed" << llendl ; + } + } + //de-allocation + for(U32 i = 0 ; i < times; i++) + { + ::delete[] p[i] ; + p[i] = NULL ; + } + llinfos << "time spent using standard allocator/de-allocator: " << timer.getElapsedTimeF32() << llendl ; + + delete[] p; +} + +void LLPrivateMemoryPoolTester::correctnessTest() +{ + //try many different sized allocation, and all kinds of edge cases, access the allocated memory + //to see if allocation is right. + + //edge case + char* p = ALLOCATE_MEM(sPool, 0) ; + FREE_MEM(sPool, p) ; + + //small sized + // [8 bytes, 2KB), each asks for 256 allocations and deallocations + test(8, 2040, 8, 256, true, true) ; + + //medium sized + //[2KB, 512KB), each asks for 16 allocations and deallocations + test(2048, 512 * 1024 - 2048, 2048, 16, true, true) ; + + //large sized + //[512KB, 4MB], each asks for 8 allocations and deallocations + test(512 * 1024, 4 * 1024 * 1024, 64 * 1024, 6, true, true) ; +} + +void LLPrivateMemoryPoolTester::performanceTest() +{ + U32 test_size[3] = {768, 3* 1024, 3* 1024 * 1024}; + + //small sized + testAndTime(test_size[0], 8) ; + + //medium sized + testAndTime(test_size[1], 8) ; + + //large sized + testAndTime(test_size[2], 8) ; +} + +void LLPrivateMemoryPoolTester::fragmentationtest() +{ + //for internal fragmentation statistics: + //every time when asking for a new chunk during correctness test, and performance test, + //print out the chunk usage statistices. +} +#endif +//-------------------------------------------------------------------- diff --git a/indra/llcommon/llmemory.h b/indra/llcommon/llmemory.h index 11406f59b0..6967edd7e7 100644 --- a/indra/llcommon/llmemory.h +++ b/indra/llcommon/llmemory.h @@ -1,25 +1,25 @@ -/** +/** * @file llmemory.h * @brief Memory allocation/deallocation header-stuff goes here. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ @@ -27,13 +27,86 @@ #define LLMEMORY_H #include "llmemtype.h" +#if LL_DEBUG +inline void* ll_aligned_malloc( size_t size, int align ) +{ + void* mem = malloc( size + (align - 1) + sizeof(void*) ); + char* aligned = ((char*)mem) + sizeof(void*); + aligned += align - ((uintptr_t)aligned & (align - 1)); + + ((void**)aligned)[-1] = mem; + return aligned; +} + +inline void ll_aligned_free( void* ptr ) +{ + free( ((void**)ptr)[-1] ); +} + +inline void* ll_aligned_malloc_16(size_t size) // returned hunk MUST be freed with ll_aligned_free_16(). +{ +#if defined(LL_WINDOWS) + return _mm_malloc(size, 16); +#elif defined(LL_DARWIN) + return malloc(size); // default osx malloc is 16 byte aligned. +#else + void *rtn; + if (LL_LIKELY(0 == posix_memalign(&rtn, 16, size))) + return rtn; + else // bad alignment requested, or out of memory + return NULL; +#endif +} + +inline void ll_aligned_free_16(void *p) +{ +#if defined(LL_WINDOWS) + _mm_free(p); +#elif defined(LL_DARWIN) + return free(p); +#else + free(p); // posix_memalign() is compatible with heap deallocator +#endif +} -extern S32 gTotalDAlloc; -extern S32 gTotalDAUse; -extern S32 gDACount; +inline void* ll_aligned_malloc_32(size_t size) // returned hunk MUST be freed with ll_aligned_free_32(). +{ +#if defined(LL_WINDOWS) + return _mm_malloc(size, 32); +#elif defined(LL_DARWIN) + return ll_aligned_malloc( size, 32 ); +#else + void *rtn; + if (LL_LIKELY(0 == posix_memalign(&rtn, 32, size))) + return rtn; + else // bad alignment requested, or out of memory + return NULL; +#endif +} -extern void* ll_allocate (size_t size); -extern void ll_release (void *p); +inline void ll_aligned_free_32(void *p) +{ +#if defined(LL_WINDOWS) + _mm_free(p); +#elif defined(LL_DARWIN) + ll_aligned_free( p ); +#else + free(p); // posix_memalign() is compatible with heap deallocator +#endif +} +#else // LL_DEBUG +// ll_aligned_foo are noops now that we use tcmalloc everywhere (tcmalloc aligns automatically at appropriate intervals) +#define ll_aligned_malloc( size, align ) malloc(size) +#define ll_aligned_free( ptr ) free(ptr) +#define ll_aligned_malloc_16 malloc +#define ll_aligned_free_16 free +#define ll_aligned_malloc_32 malloc +#define ll_aligned_free_32 free +#endif // LL_DEBUG + +#ifndef __DEBUG_PRIVATE_MEM__ +#define __DEBUG_PRIVATE_MEM__ 0 +#endif class LL_COMMON_API LLMemory { @@ -45,8 +118,24 @@ public: // Return value is zero if not known. static U64 getCurrentRSS(); static U32 getWorkingSetSize(); + static void* tryToAlloc(void* address, U32 size); + static void initMaxHeapSizeGB(F32 max_heap_size_gb, BOOL prevent_heap_failure); + static void updateMemoryInfo() ; + static void logMemoryInfo(BOOL update = FALSE); + static bool isMemoryPoolLow(); + + static U32 getAvailableMemKB() ; + static U32 getMaxMemKB() ; + static U32 getAllocatedMemKB() ; private: static char* reserveMem; + static U32 sAvailPhysicalMemInKB ; + static U32 sMaxPhysicalMemInKB ; + static U32 sAllocatedMemInKB; + static U32 sAllocatedPageSizeInKB ; + + static U32 sMaxHeapSizeInKB; + static BOOL sEnableMemoryFailurePrevention; }; //---------------------------------------------------------------------------- @@ -93,6 +182,327 @@ private: //---------------------------------------------------------------------------- + +// +//class LLPrivateMemoryPool defines a private memory pool for an application to use, so the application does not +//need to access the heap directly fro each memory allocation. Throught this, the allocation speed is faster, +//and reduces virtaul address space gragmentation problem. +//Note: this class is thread-safe by passing true to the constructor function. However, you do not need to do this unless +//you are sure the memory allocation and de-allocation will happen in different threads. To make the pool thread safe +//increases allocation and deallocation cost. +// +class LL_COMMON_API LLPrivateMemoryPool +{ + friend class LLPrivateMemoryPoolManager ; + +public: + class LL_COMMON_API LLMemoryBlock //each block is devided into slots uniformly + { + public: + LLMemoryBlock() ; + ~LLMemoryBlock() ; + + void init(char* buffer, U32 buffer_size, U32 slot_size) ; + void setBuffer(char* buffer, U32 buffer_size) ; + + char* allocate() ; + void freeMem(void* addr) ; + + bool empty() {return !mAllocatedSlots;} + bool isFull() {return mAllocatedSlots == mTotalSlots;} + bool isFree() {return !mTotalSlots;} + + U32 getSlotSize()const {return mSlotSize;} + U32 getTotalSlots()const {return mTotalSlots;} + U32 getBufferSize()const {return mBufferSize;} + char* getBuffer() const {return mBuffer;} + + //debug use + void resetBitMap() ; + private: + char* mBuffer; + U32 mSlotSize ; //when the block is not initialized, it is the buffer size. + U32 mBufferSize ; + U32 mUsageBits ; + U8 mTotalSlots ; + U8 mAllocatedSlots ; + U8 mDummySize ; //size of extra bytes reserved for mUsageBits. + + public: + LLMemoryBlock* mPrev ; + LLMemoryBlock* mNext ; + LLMemoryBlock* mSelf ; + + struct CompareAddress + { + bool operator()(const LLMemoryBlock* const& lhs, const LLMemoryBlock* const& rhs) + { + return (U32)lhs->getBuffer() < (U32)rhs->getBuffer(); + } + }; + }; + + class LL_COMMON_API LLMemoryChunk //is divided into memory blocks. + { + public: + LLMemoryChunk() ; + ~LLMemoryChunk() ; + + void init(char* buffer, U32 buffer_size, U32 min_slot_size, U32 max_slot_size, U32 min_block_size, U32 max_block_size) ; + void setBuffer(char* buffer, U32 buffer_size) ; + + bool empty() ; + + char* allocate(U32 size) ; + void freeMem(void* addr) ; + + char* getBuffer() const {return mBuffer;} + U32 getBufferSize() const {return mBufferSize;} + U32 getAllocatedSize() const {return mAlloatedSize;} + + bool containsAddress(const char* addr) const; + + static U32 getMaxOverhead(U32 data_buffer_size, U32 min_slot_size, + U32 max_slot_size, U32 min_block_size, U32 max_block_size) ; + + void dump() ; + + private: + U32 getPageIndex(U32 addr) ; + U32 getBlockLevel(U32 size) ; + U16 getPageLevel(U32 size) ; + LLMemoryBlock* addBlock(U32 blk_idx) ; + void popAvailBlockList(U32 blk_idx) ; + void addToFreeSpace(LLMemoryBlock* blk) ; + void removeFromFreeSpace(LLMemoryBlock* blk) ; + void removeBlock(LLMemoryBlock* blk) ; + void addToAvailBlockList(LLMemoryBlock* blk) ; + U32 calcBlockSize(U32 slot_size); + LLMemoryBlock* createNewBlock(LLMemoryBlock* blk, U32 buffer_size, U32 slot_size, U32 blk_idx) ; + + private: + LLMemoryBlock** mAvailBlockList ;//256 by mMinSlotSize + LLMemoryBlock** mFreeSpaceList; + LLMemoryBlock* mBlocks ; //index of blocks by address. + + char* mBuffer ; + U32 mBufferSize ; + char* mDataBuffer ; + char* mMetaBuffer ; + U32 mMinBlockSize ; + U32 mMinSlotSize ; + U32 mMaxSlotSize ; + U32 mAlloatedSize ; + U16 mBlockLevels; + U16 mPartitionLevels; + + public: + //form a linked list + LLMemoryChunk* mNext ; + LLMemoryChunk* mPrev ; + } ; + +private: + LLPrivateMemoryPool(S32 type, U32 max_pool_size) ; + ~LLPrivateMemoryPool() ; + + char *allocate(U32 size) ; + void freeMem(void* addr) ; + + void dump() ; + U32 getTotalAllocatedSize() ; + U32 getTotalReservedSize() {return mReservedPoolSize;} + S32 getType() const {return mType; } + bool isEmpty() const {return !mNumOfChunks; } + +private: + void lock() ; + void unlock() ; + S32 getChunkIndex(U32 size) ; + LLMemoryChunk* addChunk(S32 chunk_index) ; + bool checkSize(U32 asked_size) ; + void removeChunk(LLMemoryChunk* chunk) ; + U16 findHashKey(const char* addr); + void addToHashTable(LLMemoryChunk* chunk) ; + void removeFromHashTable(LLMemoryChunk* chunk) ; + void rehash() ; + bool fillHashTable(U16 start, U16 end, LLMemoryChunk* chunk) ; + LLMemoryChunk* findChunk(const char* addr) ; + + void destroyPool() ; + +public: + enum + { + SMALL_ALLOCATION = 0, //from 8 bytes to 2KB(exclusive), page size 2KB, max chunk size is 4MB. + MEDIUM_ALLOCATION, //from 2KB to 512KB(exclusive), page size 32KB, max chunk size 4MB + LARGE_ALLOCATION, //from 512KB to 4MB(inclusive), page size 64KB, max chunk size 16MB + SUPER_ALLOCATION //allocation larger than 4MB. + }; + + enum + { + STATIC = 0 , //static pool(each alllocation stays for a long time) without threading support + VOLATILE, //Volatile pool(each allocation stays for a very short time) without threading support + STATIC_THREADED, //static pool with threading support + VOLATILE_THREADED, //volatile pool with threading support + MAX_TYPES + }; //pool types + +private: + LLMutex* mMutexp ; + U32 mMaxPoolSize; + U32 mReservedPoolSize ; + + LLMemoryChunk* mChunkList[SUPER_ALLOCATION] ; //all memory chunks reserved by this pool, sorted by address + U16 mNumOfChunks ; + U16 mHashFactor ; + + S32 mType ; + + class LLChunkHashElement + { + public: + LLChunkHashElement() {mFirst = NULL ; mSecond = NULL ;} + + bool add(LLMemoryChunk* chunk) ; + void remove(LLMemoryChunk* chunk) ; + LLMemoryChunk* findChunk(const char* addr) ; + + bool empty() {return !mFirst && !mSecond; } + bool full() {return mFirst && mSecond; } + bool hasElement(LLMemoryChunk* chunk) {return mFirst == chunk || mSecond == chunk;} + + private: + LLMemoryChunk* mFirst ; + LLMemoryChunk* mSecond ; + }; + std::vector<LLChunkHashElement> mChunkHashList ; +}; + +class LL_COMMON_API LLPrivateMemoryPoolManager +{ +private: + LLPrivateMemoryPoolManager(BOOL enabled, U32 max_pool_size) ; + ~LLPrivateMemoryPoolManager() ; + +public: + static LLPrivateMemoryPoolManager* getInstance() ; + static void initClass(BOOL enabled, U32 pool_size) ; + static void destroyClass() ; + + LLPrivateMemoryPool* newPool(S32 type) ; + void deletePool(LLPrivateMemoryPool* pool) ; + +private: + static LLPrivateMemoryPoolManager* sInstance ; + std::vector<LLPrivateMemoryPool*> mPoolList ; + BOOL mPrivatePoolEnabled; + U32 mMaxPrivatePoolSize; + +public: + //debug and statistics info. + void updateStatistics() ; + + U32 mTotalReservedSize ; + U32 mTotalAllocatedSize ; + +public: +#if __DEBUG_PRIVATE_MEM__ + static char* allocate(LLPrivateMemoryPool* poolp, U32 size, const char* function, const int line) ; + + typedef std::map<char*, std::string> mem_allocation_info_t ; + static mem_allocation_info_t sMemAllocationTracker; +#else + static char* allocate(LLPrivateMemoryPool* poolp, U32 size) ; +#endif + static void freeMem(LLPrivateMemoryPool* poolp, void* addr) ; +}; + +//------------------------------------------------------------------------------------- +#if __DEBUG_PRIVATE_MEM__ +#define ALLOCATE_MEM(poolp, size) LLPrivateMemoryPoolManager::allocate((poolp), (size), __FUNCTION__, __LINE__) +#else +#define ALLOCATE_MEM(poolp, size) LLPrivateMemoryPoolManager::allocate((poolp), (size)) +#endif +#define FREE_MEM(poolp, addr) LLPrivateMemoryPoolManager::freeMem((poolp), (addr)) +//------------------------------------------------------------------------------------- + +// +//the below singleton is used to test the private memory pool. +// +#if 0 +class LL_COMMON_API LLPrivateMemoryPoolTester +{ +private: + LLPrivateMemoryPoolTester() ; + ~LLPrivateMemoryPoolTester() ; + +public: + static LLPrivateMemoryPoolTester* getInstance() ; + static void destroy() ; + + void run(S32 type) ; + +private: + void correctnessTest() ; + void performanceTest() ; + void fragmentationtest() ; + + void test(U32 min_size, U32 max_size, U32 stride, U32 times, bool random_deletion, bool output_statistics) ; + void testAndTime(U32 size, U32 times) ; + +#if 0 +public: + void* operator new(size_t size) + { + return (void*)sPool->allocate(size) ; + } + void operator delete(void* addr) + { + sPool->freeMem(addr) ; + } + void* operator new[](size_t size) + { + return (void*)sPool->allocate(size) ; + } + void operator delete[](void* addr) + { + sPool->freeMem(addr) ; + } +#endif + +private: + static LLPrivateMemoryPoolTester* sInstance; + static LLPrivateMemoryPool* sPool ; + static LLPrivateMemoryPool* sThreadedPool ; +}; +#if 0 +//static +void* LLPrivateMemoryPoolTester::operator new(size_t size) +{ + return (void*)sPool->allocate(size) ; +} + +//static +void LLPrivateMemoryPoolTester::operator delete(void* addr) +{ + sPool->free(addr) ; +} + +//static +void* LLPrivateMemoryPoolTester::operator new[](size_t size) +{ + return (void*)sPool->allocate(size) ; +} + +//static +void LLPrivateMemoryPoolTester::operator delete[](void* addr) +{ + sPool->free(addr) ; +} +#endif +#endif // LLRefCount moved to llrefcount.h // LLPointer moved to llpointer.h diff --git a/indra/llcommon/llmetricperformancetester.cpp b/indra/llcommon/llmetricperformancetester.cpp index 5fa3a5ea07..41d3eb0bf3 100644 --- a/indra/llcommon/llmetricperformancetester.cpp +++ b/indra/llcommon/llmetricperformancetester.cpp @@ -63,7 +63,18 @@ BOOL LLMetricPerformanceTesterBasic::addTester(LLMetricPerformanceTesterBasic* t sTesterMap.insert(std::make_pair(name, tester)); return TRUE; } - + +/*static*/ +void LLMetricPerformanceTesterBasic::deleteTester(std::string name) +{ + name_tester_map_t::iterator tester = sTesterMap.find(name); + if (tester != sTesterMap.end()) + { + delete tester->second; + sTesterMap.erase(tester); + } +} + /*static*/ LLMetricPerformanceTesterBasic* LLMetricPerformanceTesterBasic::getTester(std::string name) { @@ -83,7 +94,78 @@ BOOL LLMetricPerformanceTesterBasic::isMetricLogRequested(std::string name) return (LLFastTimer::sMetricLog && ((LLFastTimer::sLogName == name) || (LLFastTimer::sLogName == DEFAULT_METRIC_NAME))); } +/*static*/ +LLSD LLMetricPerformanceTesterBasic::analyzeMetricPerformanceLog(std::istream& is) +{ + LLSD ret; + LLSD cur; + + while (!is.eof() && LLSDSerialize::fromXML(cur, is)) + { + for (LLSD::map_iterator iter = cur.beginMap(); iter != cur.endMap(); ++iter) + { + std::string label = iter->first; + + LLMetricPerformanceTesterBasic* tester = LLMetricPerformanceTesterBasic::getTester(iter->second["Name"].asString()) ; + if(tester) + { + ret[label]["Name"] = iter->second["Name"] ; + + S32 num_of_metrics = tester->getNumberOfMetrics() ; + for(S32 index = 0 ; index < num_of_metrics ; index++) + { + ret[label][ tester->getMetricName(index) ] = iter->second[ tester->getMetricName(index) ] ; + } + } + } + } + + return ret; +} + +/*static*/ +void LLMetricPerformanceTesterBasic::doAnalysisMetrics(std::string baseline, std::string target, std::string output) +{ + if(!LLMetricPerformanceTesterBasic::hasMetricPerformanceTesters()) + { + return ; + } + + // Open baseline and current target, exit if one is inexistent + std::ifstream base_is(baseline.c_str()); + std::ifstream target_is(target.c_str()); + if (!base_is.is_open() || !target_is.is_open()) + { + llwarns << "'-analyzeperformance' error : baseline or current target file inexistent" << llendl; + base_is.close(); + target_is.close(); + return; + } + //analyze baseline + LLSD base = analyzeMetricPerformanceLog(base_is); + base_is.close(); + + //analyze current + LLSD current = analyzeMetricPerformanceLog(target_is); + target_is.close(); + + //output comparision + std::ofstream os(output.c_str()); + + os << "Label, Metric, Base(B), Target(T), Diff(T-B), Percentage(100*T/B)\n"; + for(LLMetricPerformanceTesterBasic::name_tester_map_t::iterator iter = LLMetricPerformanceTesterBasic::sTesterMap.begin() ; + iter != LLMetricPerformanceTesterBasic::sTesterMap.end() ; ++iter) + { + LLMetricPerformanceTesterBasic* tester = ((LLMetricPerformanceTesterBasic*)iter->second) ; + tester->analyzePerformance(&os, &base, ¤t) ; + } + + os.flush(); + os.close(); +} + + //---------------------------------------------------------------------------------------------- // LLMetricPerformanceTesterBasic : Tester instance methods //---------------------------------------------------------------------------------------------- diff --git a/indra/llcommon/llmetricperformancetester.h b/indra/llcommon/llmetricperformancetester.h index 925010ac96..1a18cdf36f 100644 --- a/indra/llcommon/llmetricperformancetester.h +++ b/indra/llcommon/llmetricperformancetester.h @@ -27,7 +27,7 @@ #ifndef LL_METRICPERFORMANCETESTER_H #define LL_METRICPERFORMANCETESTER_H -const std::string DEFAULT_METRIC_NAME("metric"); +char const* const DEFAULT_METRIC_NAME = "metric"; /** * @class LLMetricPerformanceTesterBasic @@ -62,6 +62,8 @@ public: */ virtual void analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) ; + static void doAnalysisMetrics(std::string baseline, std::string target, std::string output) ; + /** * @return Returns the number of the test metrics in this tester instance. */ @@ -116,6 +118,7 @@ protected: private: void preOutputTestResults(LLSD* sd) ; void postOutputTestResults(LLSD* sd) ; + static LLSD analyzeMetricPerformanceLog(std::istream& is) ; std::string mName ; // Name of this tester instance S32 mCount ; // Current record count @@ -135,6 +138,12 @@ public: static LLMetricPerformanceTesterBasic* getTester(std::string name) ; /** + * @return Delete the named tester from the list + * @param[in] name - Name of the tester instance to delete. + */ + static void deleteTester(std::string name); + + /** * @return Returns TRUE if that metric *or* the default catch all metric has been requested to be logged * @param[in] name - Name of the tester queried. */ diff --git a/indra/llcommon/llprocesslauncher.cpp b/indra/llcommon/llprocesslauncher.cpp index 4b0f6b0251..10950181fd 100644 --- a/indra/llcommon/llprocesslauncher.cpp +++ b/indra/llcommon/llprocesslauncher.cpp @@ -103,10 +103,30 @@ int LLProcessLauncher::launch(void) char *args2 = new char[args.size() + 1]; strcpy(args2, args.c_str()); - if( ! CreateProcessA( NULL, args2, NULL, NULL, FALSE, 0, NULL, NULL, &sinfo, &pinfo ) ) + const char * working_directory = 0; + if(!mWorkingDir.empty()) working_directory = mWorkingDir.c_str(); + if( ! CreateProcessA( NULL, args2, NULL, NULL, FALSE, 0, NULL, working_directory, &sinfo, &pinfo ) ) { - // TODO: do better than returning the OS-specific error code on failure... result = GetLastError(); + + LPTSTR error_str = 0; + if( + FormatMessage( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + result, + 0, + (LPTSTR)&error_str, + 0, + NULL) + != 0) + { + char message[256]; + wcstombs(message, error_str, 256); + message[255] = 0; + llwarns << "CreateProcessA failed: " << message << llendl; + LocalFree(error_str); + } + if(result == 0) { // Make absolutely certain we return a non-zero value on failure. diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index efd9c4b68f..5dee7a3541 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -32,7 +32,7 @@ //============================================================================ // MAIN THREAD -LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded) : +LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded, bool should_pause) : LLThread(name), mThreaded(threaded), mIdleThread(TRUE), @@ -41,6 +41,11 @@ LLQueuedThread::LLQueuedThread(const std::string& name, bool threaded) : { if (mThreaded) { + if(should_pause) + { + pause() ; //call this before start the thread. + } + start(); } } diff --git a/indra/llcommon/llqueuedthread.h b/indra/llcommon/llqueuedthread.h index a53b22f6fc..499d13a792 100644 --- a/indra/llcommon/llqueuedthread.h +++ b/indra/llcommon/llqueuedthread.h @@ -149,7 +149,7 @@ public: static handle_t nullHandle() { return handle_t(0); } public: - LLQueuedThread(const std::string& name, bool threaded = true); + LLQueuedThread(const std::string& name, bool threaded = true, bool should_pause = false); virtual ~LLQueuedThread(); virtual void shutdown(); diff --git a/indra/llcommon/llrefcount.cpp b/indra/llcommon/llrefcount.cpp index 55d0c85cbd..e1876599fc 100644 --- a/indra/llcommon/llrefcount.cpp +++ b/indra/llcommon/llrefcount.cpp @@ -29,9 +29,25 @@ #include "llerror.h" +#if LL_REF_COUNT_DEBUG +#include "llthread.h" +#include "llapr.h" +#endif + LLRefCount::LLRefCount(const LLRefCount& other) : mRef(0) { +#if LL_REF_COUNT_DEBUG + if(gAPRPoolp) + { + mMutexp = new LLMutex(gAPRPoolp) ; + } + else + { + mMutexp = NULL ; + } + mCrashAtUnlock = FALSE ; +#endif } LLRefCount& LLRefCount::operator=(const LLRefCount&) @@ -43,6 +59,17 @@ LLRefCount& LLRefCount::operator=(const LLRefCount&) LLRefCount::LLRefCount() : mRef(0) { +#if LL_REF_COUNT_DEBUG + if(gAPRPoolp) + { + mMutexp = new LLMutex(gAPRPoolp) ; + } + else + { + mMutexp = NULL ; + } + mCrashAtUnlock = FALSE ; +#endif } LLRefCount::~LLRefCount() @@ -51,4 +78,87 @@ LLRefCount::~LLRefCount() { llerrs << "deleting non-zero reference" << llendl; } + +#if LL_REF_COUNT_DEBUG + if(gAPRPoolp) + { + delete mMutexp ; + } +#endif } + +#if LL_REF_COUNT_DEBUG +void LLRefCount::ref() const +{ + if(mMutexp) + { + if(mMutexp->isLocked()) + { + mCrashAtUnlock = TRUE ; + llerrs << "the mutex is locked by the thread: " << mLockedThreadID + << " Current thread: " << LLThread::currentID() << llendl ; + } + + mMutexp->lock() ; + mLockedThreadID = LLThread::currentID() ; + + mRef++; + + if(mCrashAtUnlock) + { + while(1); //crash here. + } + mMutexp->unlock() ; + } + else + { + mRef++; + } +} + +S32 LLRefCount::unref() const +{ + if(mMutexp) + { + if(mMutexp->isLocked()) + { + mCrashAtUnlock = TRUE ; + llerrs << "the mutex is locked by the thread: " << mLockedThreadID + << " Current thread: " << LLThread::currentID() << llendl ; + } + + mMutexp->lock() ; + mLockedThreadID = LLThread::currentID() ; + + llassert(mRef >= 1); + if (0 == --mRef) + { + if(mCrashAtUnlock) + { + while(1); //crash here. + } + mMutexp->unlock() ; + + delete this; + return 0; + } + + if(mCrashAtUnlock) + { + while(1); //crash here. + } + mMutexp->unlock() ; + return mRef; + } + else + { + llassert(mRef >= 1); + if (0 == --mRef) + { + delete this; + return 0; + } + return mRef; + } +} +#endif diff --git a/indra/llcommon/llrefcount.h b/indra/llcommon/llrefcount.h index 19f008b15c..8eb5d53f3f 100644 --- a/indra/llcommon/llrefcount.h +++ b/indra/llcommon/llrefcount.h @@ -28,6 +28,11 @@ #include <boost/noncopyable.hpp> +#define LL_REF_COUNT_DEBUG 0 +#if LL_REF_COUNT_DEBUG +class LLMutex ; +#endif + //---------------------------------------------------------------------------- // RefCount objects should generally only be accessed by way of LLPointer<>'s // see llthread.h for LLThreadSafeRefCount @@ -43,12 +48,16 @@ protected: public: LLRefCount(); - void ref() const +#if LL_REF_COUNT_DEBUG + void ref() const ; + S32 unref() const ; +#else + inline void ref() const { mRef++; } - S32 unref() const + inline S32 unref() const { llassert(mRef >= 1); if (0 == --mRef) @@ -58,6 +67,7 @@ public: } return mRef; } +#endif //NOTE: when passing around a const LLRefCount object, this can return different results // at different types, since mRef is mutable @@ -68,6 +78,12 @@ public: private: mutable S32 mRef; + +#if LL_REF_COUNT_DEBUG + LLMutex* mMutexp ; + mutable U32 mLockedThreadID ; + mutable BOOL mCrashAtUnlock ; +#endif }; #endif diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 10f460e8a6..bf62600514 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -34,6 +34,12 @@ #include <iostream> #include "apr_base64.h" +#ifdef LL_STANDALONE +# include <zlib.h> +#else +# include "zlib/zlib.h" // for davep's dirty little zip functions +#endif + #if !LL_WINDOWS #include <netinet/in.h> // htonl & ntohl #endif @@ -1983,3 +1989,182 @@ std::ostream& operator<<(std::ostream& s, const LLSD& llsd) return s; } + +//dirty little zippers -- yell at davep if these are horrid + +//return a string containing gzipped bytes of binary serialized LLSD +// VERY inefficient -- creates several copies of LLSD block in memory +std::string zip_llsd(LLSD& data) +{ + std::stringstream llsd_strm; + + LLSDSerialize::toBinary(data, llsd_strm); + + const U32 CHUNK = 65536; + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + S32 ret = deflateInit(&strm, Z_BEST_COMPRESSION); + if (ret != Z_OK) + { + llwarns << "Failed to compress LLSD block." << llendl; + return std::string(); + } + + std::string source = llsd_strm.str(); + + U8 out[CHUNK]; + + strm.avail_in = source.size(); + strm.next_in = (U8*) source.data(); + U8* output = NULL; + + U32 cur_size = 0; + + U32 have = 0; + + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + + ret = deflate(&strm, Z_FINISH); + if (ret == Z_OK || ret == Z_STREAM_END) + { //copy result into output + if (strm.avail_out >= CHUNK) + { + free(output); + llwarns << "Failed to compress LLSD block." << llendl; + return std::string(); + } + + have = CHUNK-strm.avail_out; + output = (U8*) realloc(output, cur_size+have); + memcpy(output+cur_size, out, have); + cur_size += have; + } + else + { + free(output); + llwarns << "Failed to compress LLSD block." << llendl; + return std::string(); + } + } + while (ret == Z_OK); + + std::string::size_type size = cur_size; + + std::string result((char*) output, size); + deflateEnd(&strm); + free(output); + +#if 0 //verify results work with unzip_llsd + std::istringstream test(result); + LLSD test_sd; + if (!unzip_llsd(test_sd, test, result.size())) + { + llerrs << "Invalid compression result!" << llendl; + } +#endif + + return result; +} + +//decompress a block of LLSD from provided istream +// not very efficient -- creats a copy of decompressed LLSD block in memory +// and deserializes from that copy using LLSDSerialize +bool unzip_llsd(LLSD& data, std::istream& is, S32 size) +{ + U8* result = NULL; + U32 cur_size = 0; + z_stream strm; + + const U32 CHUNK = 65536; + + U8 *in = new U8[size]; + is.read((char*) in, size); + + U8 out[CHUNK]; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = size; + strm.next_in = in; + + S32 ret = inflateInit(&strm); + + do + { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR) + { + inflateEnd(&strm); + free(result); + delete [] in; + return false; + } + + switch (ret) + { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; + case Z_DATA_ERROR: + case Z_MEM_ERROR: + inflateEnd(&strm); + free(result); + delete [] in; + return false; + break; + } + + U32 have = CHUNK-strm.avail_out; + + result = (U8*) realloc(result, cur_size + have); + memcpy(result+cur_size, out, have); + cur_size += have; + + } while (ret == Z_OK); + + inflateEnd(&strm); + delete [] in; + + if (ret != Z_STREAM_END) + { + free(result); + return false; + } + + //result now points to the decompressed LLSD block + { + std::string res_str((char*) result, cur_size); + + std::string deprecated_header("<? LLSD/Binary ?>"); + + if (res_str.substr(0, deprecated_header.size()) == deprecated_header) + { + res_str = res_str.substr(deprecated_header.size()+1, cur_size); + } + cur_size = res_str.size(); + + std::istringstream istr(res_str); + + if (!LLSDSerialize::fromBinary(data, istr, cur_size)) + { + llwarns << "Failed to unzip LLSD block" << llendl; + free(result); + return false; + } + } + + free(result); + return true; +} + + + diff --git a/indra/llcommon/llsdserialize.h b/indra/llcommon/llsdserialize.h index 1f096b5254..99a3ea3cd4 100644 --- a/indra/llcommon/llsdserialize.h +++ b/indra/llcommon/llsdserialize.h @@ -790,4 +790,8 @@ public: } }; +//dirty little zip functions -- yell at davep +LL_COMMON_API std::string zip_llsd(LLSD& data); +LL_COMMON_API bool unzip_llsd(LLSD& data, std::istream& is, S32 size); + #endif // LL_LLSDSERIALIZE_H diff --git a/indra/llcommon/llsdserialize_xml.cpp b/indra/llcommon/llsdserialize_xml.cpp index c5a7c6fc15..be9db53906 100644 --- a/indra/llcommon/llsdserialize_xml.cpp +++ b/indra/llcommon/llsdserialize_xml.cpp @@ -373,10 +373,13 @@ S32 LLSDXMLParser::Impl::parse(std::istream& input, LLSD& data) { break; } - count = get_till_eol(input, (char *)buffer, BUFFER_SIZE); - if (!count) { - break; + + count = get_till_eol(input, (char *)buffer, BUFFER_SIZE); + if (!count) + { + break; + } } status = XML_ParseBuffer(mParser, count, false); @@ -716,6 +719,7 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name) case ELEMENT_INTEGER: { S32 i; + // sscanf okay here with different locales - ints don't change for different locale settings like floats do. if ( sscanf(mCurrentContent.c_str(), "%d", &i ) == 1 ) { // See if sscanf works - it's faster value = i; @@ -729,15 +733,19 @@ void LLSDXMLParser::Impl::endElementHandler(const XML_Char* name) case ELEMENT_REAL: { - F64 r; - if ( sscanf(mCurrentContent.c_str(), "%lf", &r ) == 1 ) - { // See if sscanf works - it's faster - value = r; - } - else - { - value = LLSD(mCurrentContent).asReal(); - } + value = LLSD(mCurrentContent).asReal(); + // removed since this breaks when locale has decimal separator that isn't '.' + // investigated changing local to something compatible each time but deemed higher + // risk that just using LLSD.asReal() each time. + //F64 r; + //if ( sscanf(mCurrentContent.c_str(), "%lf", &r ) == 1 ) + //{ // See if sscanf works - it's faster + // value = r; + //} + //else + //{ + // value = LLSD(mCurrentContent).asReal(); + //} } break; diff --git a/indra/llcommon/llsdutil.cpp b/indra/llcommon/llsdutil.cpp index f8f9ece058..803417d368 100644 --- a/indra/llcommon/llsdutil.cpp +++ b/indra/llcommon/llsdutil.cpp @@ -41,6 +41,7 @@ #include "llsdserialize.h" #include "stringize.h" +#include "is_approx_equal_fraction.h" #include <map> #include <set> @@ -571,7 +572,7 @@ std::string llsd_matches(const LLSD& prototype, const LLSD& data, const std::str return match_types(prototype.type(), TypeVector(), data.type(), pfx); } -bool llsd_equals(const LLSD& lhs, const LLSD& rhs) +bool llsd_equals(const LLSD& lhs, const LLSD& rhs, unsigned bits) { // We're comparing strict equality of LLSD representation rather than // performing any conversions. So if the types aren't equal, the LLSD @@ -588,6 +589,20 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) // Both are TypeUndefined. There's nothing more to know. return true; + case LLSD::TypeReal: + // This is where the 'bits' argument comes in handy. If passed + // explicitly, it means to use is_approx_equal_fraction() to compare. + if (bits >= 0) + { + return is_approx_equal_fraction(lhs.asReal(), rhs.asReal(), bits); + } + // Otherwise we compare bit representations, and the usual caveats + // about comparing floating-point numbers apply. Omitting 'bits' when + // comparing Real values is only useful when we expect identical bit + // representation for a given Real value, e.g. for integer-valued + // Reals. + return (lhs.asReal() == rhs.asReal()); + #define COMPARE_SCALAR(type) \ case LLSD::Type##type: \ /* LLSD::URI has operator!=() but not operator==() */ \ @@ -596,10 +611,6 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) COMPARE_SCALAR(Boolean); COMPARE_SCALAR(Integer); - // The usual caveats about comparing floating-point numbers apply. This is - // only useful when we expect identical bit representation for a given - // Real value, e.g. for integer-valued Reals. - COMPARE_SCALAR(Real); COMPARE_SCALAR(String); COMPARE_SCALAR(UUID); COMPARE_SCALAR(Date); @@ -617,7 +628,7 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) for ( ; lai != laend && rai != raend; ++lai, ++rai) { // If any one array element is unequal, the arrays are unequal. - if (! llsd_equals(*lai, *rai)) + if (! llsd_equals(*lai, *rai, bits)) return false; } // Here we've reached the end of one or the other array. They're equal @@ -644,7 +655,7 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) if (rhskeys.erase(lmi->first) != 1) return false; // Both maps have the current key. Compare values. - if (! llsd_equals(lmi->second, rhs[lmi->first])) + if (! llsd_equals(lmi->second, rhs[lmi->first], bits)) return false; } // We've now established that all the lhs keys have equal values in @@ -657,7 +668,7 @@ bool llsd_equals(const LLSD& lhs, const LLSD& rhs) // We expect that every possible type() value is specifically handled // above. Failing to extend this switch to support a new LLSD type is // an error that must be brought to the coder's attention. - LL_ERRS("llsd_equals") << "llsd_equals(" << lhs << ", " << rhs << "): " + LL_ERRS("llsd_equals") << "llsd_equals(" << lhs << ", " << rhs << ", " << bits << "): " "unknown type " << lhs.type() << LL_ENDL; return false; // pacify the compiler } diff --git a/indra/llcommon/llsdutil.h b/indra/llcommon/llsdutil.h index bb8c0690b1..65c7297cbf 100644 --- a/indra/llcommon/llsdutil.h +++ b/indra/llcommon/llsdutil.h @@ -123,8 +123,10 @@ LL_COMMON_API BOOL compare_llsd_with_template( */ LL_COMMON_API std::string llsd_matches(const LLSD& prototype, const LLSD& data, const std::string& pfx=""); -/// Deep equality -LL_COMMON_API bool llsd_equals(const LLSD& lhs, const LLSD& rhs); +/// Deep equality. If you want to compare LLSD::Real values for approximate +/// equality rather than bitwise equality, pass @a bits as for +/// is_approx_equal_fraction(). +LL_COMMON_API bool llsd_equals(const LLSD& lhs, const LLSD& rhs, unsigned bits=-1); // Simple function to copy data out of input & output iterators if // there is no need for casting. @@ -138,4 +140,283 @@ template<typename Input> LLSD llsd_copy_array(Input iter, Input end) return dest; } +/***************************************************************************** +* LLSDArray +*****************************************************************************/ +/** + * Construct an LLSD::Array inline, with implicit conversion to LLSD. Usage: + * + * @code + * void somefunc(const LLSD&); + * ... + * somefunc(LLSDArray("text")(17)(3.14)); + * @endcode + * + * For completeness, LLSDArray() with no args constructs an empty array, so + * <tt>LLSDArray()("text")(17)(3.14)</tt> produces an array equivalent to the + * above. But for most purposes, LLSD() is already equivalent to an empty + * array, and if you explicitly want an empty isArray(), there's + * LLSD::emptyArray(). However, supporting a no-args LLSDArray() constructor + * follows the principle of least astonishment. + */ +class LLSDArray +{ +public: + LLSDArray(): + _data(LLSD::emptyArray()) + {} + + /** + * Need an explicit copy constructor. Consider the following: + * + * @code + * LLSD array_of_arrays(LLSDArray(LLSDArray(17)(34)) + * (LLSDArray("x")("y"))); + * @endcode + * + * The coder intends to construct [[17, 34], ["x", "y"]]. + * + * With the compiler's implicit copy constructor, s/he gets instead + * [17, 34, ["x", "y"]]. + * + * The expression LLSDArray(17)(34) constructs an LLSDArray with those two + * values. The reader assumes it should be converted to LLSD, as we always + * want with LLSDArray, before passing it to the @em outer LLSDArray + * constructor! This copy constructor makes that happen. + */ + LLSDArray(const LLSDArray& inner): + _data(LLSD::emptyArray()) + { + _data.append(inner); + } + + LLSDArray(const LLSD& value): + _data(LLSD::emptyArray()) + { + _data.append(value); + } + + LLSDArray& operator()(const LLSD& value) + { + _data.append(value); + return *this; + } + + operator LLSD() const { return _data; } + LLSD get() const { return _data; } + +private: + LLSD _data; +}; + +/***************************************************************************** +* LLSDMap +*****************************************************************************/ +/** + * Construct an LLSD::Map inline, with implicit conversion to LLSD. Usage: + * + * @code + * void somefunc(const LLSD&); + * ... + * somefunc(LLSDMap("alpha", "abc")("number", 17)("pi", 3.14)); + * @endcode + * + * For completeness, LLSDMap() with no args constructs an empty map, so + * <tt>LLSDMap()("alpha", "abc")("number", 17)("pi", 3.14)</tt> produces a map + * equivalent to the above. But for most purposes, LLSD() is already + * equivalent to an empty map, and if you explicitly want an empty isMap(), + * there's LLSD::emptyMap(). However, supporting a no-args LLSDMap() + * constructor follows the principle of least astonishment. + */ +class LLSDMap +{ +public: + LLSDMap(): + _data(LLSD::emptyMap()) + {} + LLSDMap(const LLSD::String& key, const LLSD& value): + _data(LLSD::emptyMap()) + { + _data[key] = value; + } + + LLSDMap& operator()(const LLSD::String& key, const LLSD& value) + { + _data[key] = value; + return *this; + } + + operator LLSD() const { return _data; } + LLSD get() const { return _data; } + +private: + LLSD _data; +}; + +/***************************************************************************** +* LLSDParam +*****************************************************************************/ +/** + * LLSDParam is a customization point for passing LLSD values to function + * parameters of more or less arbitrary type. LLSD provides a small set of + * native conversions; but if a generic algorithm explicitly constructs an + * LLSDParam object in the function's argument list, a consumer can provide + * LLSDParam specializations to support more different parameter types than + * LLSD's native conversions. + * + * Usage: + * + * @code + * void somefunc(const paramtype&); + * ... + * somefunc(..., LLSDParam<paramtype>(someLLSD), ...); + * @endcode + */ +template <typename T> +class LLSDParam +{ +public: + /** + * Default implementation converts to T on construction, saves converted + * value for later retrieval + */ + LLSDParam(const LLSD& value): + _value(value) + {} + + operator T() const { return _value; } + +private: + T _value; +}; + +/** + * Turns out that several target types could accept an LLSD param using any of + * a few different conversions, e.g. LLUUID's constructor can accept LLUUID or + * std::string. Therefore, the compiler can't decide which LLSD conversion + * operator to choose, even though to us it seems obvious. But that's okay, we + * can specialize LLSDParam for such target types, explicitly specifying the + * desired conversion -- that's part of what LLSDParam is all about. Turns out + * we have to do that enough to make it worthwhile generalizing. Use a macro + * because I need to specify one of the asReal, etc., explicit conversion + * methods as well as a type. If I'm overlooking a clever way to implement + * that using a template instead, feel free to reimplement. + */ +#define LLSDParam_for(T, AS) \ +template <> \ +class LLSDParam<T> \ +{ \ +public: \ + LLSDParam(const LLSD& value): \ + _value(value.AS()) \ + {} \ + \ + operator T() const { return _value; } \ + \ +private: \ + T _value; \ +} + +LLSDParam_for(float, asReal); +LLSDParam_for(LLUUID, asUUID); +LLSDParam_for(LLDate, asDate); +LLSDParam_for(LLURI, asURI); +LLSDParam_for(LLSD::Binary, asBinary); + +/** + * LLSDParam<const char*> is an example of the kind of conversion you can + * support with LLSDParam beyond native LLSD conversions. Normally you can't + * pass an LLSD object to a function accepting const char* -- but you can + * safely pass an LLSDParam<const char*>(yourLLSD). + */ +template <> +class LLSDParam<const char*> +{ +private: + // The difference here is that we store a std::string rather than a const + // char*. It's important that the LLSDParam object own the std::string. + std::string _value; + // We don't bother storing the incoming LLSD object, but we do have to + // distinguish whether _value is an empty string because the LLSD object + // contains an empty string or because it's isUndefined(). + bool _undefined; + +public: + LLSDParam(const LLSD& value): + _value(value), + _undefined(value.isUndefined()) + {} + + // The const char* we retrieve is for storage owned by our _value member. + // That's how we guarantee that the const char* is valid for the lifetime + // of this LLSDParam object. Constructing your LLSDParam in the argument + // list should ensure that the LLSDParam object will persist for the + // duration of the function call. + operator const char*() const + { + if (_undefined) + { + // By default, an isUndefined() LLSD object's asString() method + // will produce an empty string. But for a function accepting + // const char*, it's often important to be able to pass NULL, and + // isUndefined() seems like the best way. If you want to pass an + // empty string, you can still pass LLSD(""). Without this special + // case, though, no LLSD value could pass NULL. + return NULL; + } + return _value.c_str(); + } +}; + +namespace llsd +{ + +/***************************************************************************** +* BOOST_FOREACH() helpers for LLSD +*****************************************************************************/ +/// Usage: BOOST_FOREACH(LLSD item, inArray(someLLSDarray)) { ... } +class inArray +{ +public: + inArray(const LLSD& array): + _array(array) + {} + + typedef LLSD::array_const_iterator const_iterator; + typedef LLSD::array_iterator iterator; + + iterator begin() { return _array.beginArray(); } + iterator end() { return _array.endArray(); } + const_iterator begin() const { return _array.beginArray(); } + const_iterator end() const { return _array.endArray(); } + +private: + LLSD _array; +}; + +/// MapEntry is what you get from dereferencing an LLSD::map_[const_]iterator. +typedef std::map<LLSD::String, LLSD>::value_type MapEntry; + +/// Usage: BOOST_FOREACH([const] MapEntry& e, inMap(someLLSDmap)) { ... } +class inMap +{ +public: + inMap(const LLSD& map): + _map(map) + {} + + typedef LLSD::map_const_iterator const_iterator; + typedef LLSD::map_iterator iterator; + + iterator begin() { return _map.beginMap(); } + iterator end() { return _map.endMap(); } + const_iterator begin() const { return _map.beginMap(); } + const_iterator end() const { return _map.endMap(); } + +private: + LLSD _map; +}; + +} // namespace llsd + #endif // LL_LLSDUTIL_H diff --git a/indra/llcommon/llsingleton.h b/indra/llcommon/llsingleton.h index 7aee1bb85f..49d99f2cd0 100644 --- a/indra/llcommon/llsingleton.h +++ b/indra/llcommon/llsingleton.h @@ -100,12 +100,6 @@ private: DELETED } EInitState; - static void deleteSingleton() - { - delete getData().mSingletonInstance; - getData().mSingletonInstance = NULL; - } - // stores pointer to singleton instance // and tracks initialization state of singleton struct SingletonInstanceData @@ -120,7 +114,10 @@ private: ~SingletonInstanceData() { - deleteSingleton(); + if (mInitState != DELETED) + { + deleteSingleton(); + } } }; @@ -132,6 +129,33 @@ public: data.mInitState = DELETED; } + /** + * @brief Immediately delete the singleton. + * + * A subsequent call to LLProxy::getInstance() will construct a new + * instance of the class. + * + * LLSingletons are normally destroyed after main() has exited and the C++ + * runtime is cleaning up statically-constructed objects. Some classes + * derived from LLSingleton have objects that are part of a runtime system + * that is terminated before main() exits. Calling the destructor of those + * objects after the termination of their respective systems can cause + * crashes and other problems during termination of the project. Using this + * method to destroy the singleton early can prevent these crashes. + * + * An example where this is needed is for a LLSingleton that has an APR + * object as a member that makes APR calls on destruction. The APR system is + * shut down explicitly before main() exits. This causes a crash on exit. + * Using this method before the call to apr_terminate() and NOT calling + * getInstance() again will prevent the crash. + */ + static void deleteSingleton() + { + delete getData().mSingletonInstance; + getData().mSingletonInstance = NULL; + getData().mInitState = DELETED; + } + static SingletonInstanceData& getData() { // this is static to cache the lookup results diff --git a/indra/llcommon/llstat.cpp b/indra/llcommon/llstat.cpp index 8ba97d7730..b2c495d093 100644 --- a/indra/llcommon/llstat.cpp +++ b/indra/llcommon/llstat.cpp @@ -737,7 +737,7 @@ void LLPerfBlock::addStatsToLLSDandReset( LLSD & stats, } } else - { // WTF? Shouldn't have a NULL pointer in the map. + { // Shouldn't have a NULL pointer in the map. llwarns << "Unexpected NULL dynamic stat at '" << stats_full_path << "'" << llendl; } } diff --git a/indra/llcommon/llstring.cpp b/indra/llcommon/llstring.cpp index f3b48b0156..e7fe656808 100644 --- a/indra/llcommon/llstring.cpp +++ b/indra/llcommon/llstring.cpp @@ -936,13 +936,18 @@ LLStringUtil::size_type LLStringUtil::getSubstitution(const std::string& instr, { const std::string delims (","); - // Find the first ] - size_type pos2 = instr.find(']', start); + // Find the first [ + size_type pos1 = instr.find('[', start); + if (pos1 == std::string::npos) + return std::string::npos; + + //Find the first ] after the initial [ + size_type pos2 = instr.find(']', pos1); if (pos2 == std::string::npos) return std::string::npos; - // Find the last [ before ] - size_type pos1 = instr.find_last_of('[', pos2-1); + // Find the last [ before ] in case of nested [[]] + pos1 = instr.find_last_of('[', pos2-1); if (pos1 == std::string::npos || pos1 < start) return std::string::npos; diff --git a/indra/llcommon/llsys.cpp b/indra/llcommon/llsys.cpp index 10cdc7087b..d781687175 100644 --- a/indra/llcommon/llsys.cpp +++ b/indra/llcommon/llsys.cpp @@ -1,6 +1,6 @@ /** * @file llsys.cpp - * @brief Impelementation of the basic system query functions. + * @brief Implementation of the basic system query functions. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -24,6 +24,10 @@ * $/LicenseInfo$ */ +#if LL_WINDOWS +#pragma warning (disable : 4355) // 'this' used in initializer list: yes, intentionally +#endif + #include "linden_common.h" #include "llsys.h" @@ -36,21 +40,45 @@ #endif #include "llprocessor.h" +#include "llerrorcontrol.h" +#include "llevents.h" +#include "lltimer.h" +#include "llsdserialize.h" +#include "llsdutil.h" +#include <boost/bind.hpp> +#include <boost/circular_buffer.hpp> +#include <boost/regex.hpp> +#include <boost/foreach.hpp> +#include <boost/lexical_cast.hpp> +#include <boost/range.hpp> +#include <boost/utility/enable_if.hpp> +#include <boost/type_traits/is_integral.hpp> +#include <boost/type_traits/is_float.hpp> + +using namespace llsd; #if LL_WINDOWS # define WIN32_LEAN_AND_MEAN # include <winsock2.h> # include <windows.h> +# include <psapi.h> // GetPerformanceInfo() et al. #elif LL_DARWIN # include <errno.h> # include <sys/sysctl.h> # include <sys/utsname.h> # include <stdint.h> +# include <Carbon/Carbon.h> +# include <stdexcept> +# include <mach/host_info.h> +# include <mach/mach_host.h> +# include <mach/task.h> +# include <mach/task_info.h> #elif LL_LINUX # include <errno.h> # include <sys/utsname.h> # include <unistd.h> # include <sys/sysinfo.h> +# include <stdexcept> const char MEMINFO_FILE[] = "/proc/meminfo"; #elif LL_SOLARIS # include <stdio.h> @@ -69,6 +97,15 @@ extern int errno; static const S32 CPUINFO_BUFFER_SIZE = 16383; LLCPUInfo gSysCPU; +// Don't log memory info any more often than this. It also serves as our +// framerate sample size. +static const F32 MEM_INFO_THROTTLE = 20; +// Sliding window of samples. We intentionally limit the length of time we +// remember "the slowest" framerate because framerate is very slow at login. +// If we only triggered FrameWatcher logging when the session framerate +// dropped below the login framerate, we'd have very little additional data. +static const F32 MEM_INFO_WINDOW = 10*60; + #if LL_WINDOWS #ifndef DLLVERSIONINFO typedef struct _DllVersionInfo @@ -187,22 +224,30 @@ LLOSInfo::LLOSInfo() : if(osvi.wProductType == VER_NT_WORKSTATION) mOSStringSimple = "Microsoft Windows XP x64 Edition "; else - mOSStringSimple = "Microsoft Windows Server 2003 "; + mOSStringSimple = "Microsoft Windows Server 2003 "; } - else if(osvi.dwMajorVersion == 6 && osvi.dwMinorVersion <= 1) + else if(osvi.dwMajorVersion == 6 && osvi.dwMinorVersion <= 2) { if(osvi.dwMinorVersion == 0) { - mOSStringSimple = "Microsoft Windows Vista "; + if(osvi.wProductType == VER_NT_WORKSTATION) + mOSStringSimple = "Microsoft Windows Vista "; + else + mOSStringSimple = "Windows Server 2008 "; } else if(osvi.dwMinorVersion == 1) { - mOSStringSimple = "Microsoft Windows 7 "; + if(osvi.wProductType == VER_NT_WORKSTATION) + mOSStringSimple = "Microsoft Windows 7 "; + else + mOSStringSimple = "Windows Server 2008 R2 "; } - - if(osvi.wProductType != VER_NT_WORKSTATION) + else if(osvi.dwMinorVersion == 2) { - mOSStringSimple += "Server "; + if(osvi.wProductType == VER_NT_WORKSTATION) + mOSStringSimple = "Microsoft Windows 8 "; + else + mOSStringSimple = "Windows Server 2012 "; } ///get native system info if available.. @@ -307,8 +352,7 @@ LLOSInfo::LLOSInfo() : std::string compatibility_mode; if(got_shell32_version) { - if(osvi.dwMajorVersion != shell32_major - || osvi.dwMinorVersion != shell32_minor) + if(osvi.dwMajorVersion != shell32_major || osvi.dwMinorVersion != shell32_minor) { compatibility_mode = llformat(" compatibility mode. real ver: %d.%d (Build %d)", shell32_major, @@ -318,7 +362,58 @@ LLOSInfo::LLOSInfo() : } mOSString += compatibility_mode; +#elif LL_DARWIN + + // Initialize mOSStringSimple to something like: + // "Mac OS X 10.6.7" + { + const char * DARWIN_PRODUCT_NAME = "Mac OS X"; + + SInt32 major_version, minor_version, bugfix_version; + OSErr r1 = Gestalt(gestaltSystemVersionMajor, &major_version); + OSErr r2 = Gestalt(gestaltSystemVersionMinor, &minor_version); + OSErr r3 = Gestalt(gestaltSystemVersionBugFix, &bugfix_version); + + if((r1 == noErr) && (r2 == noErr) && (r3 == noErr)) + { + mMajorVer = major_version; + mMinorVer = minor_version; + mBuild = bugfix_version; + + std::stringstream os_version_string; + os_version_string << DARWIN_PRODUCT_NAME << " " << mMajorVer << "." << mMinorVer << "." << mBuild; + + // Put it in the OS string we are compiling + mOSStringSimple.append(os_version_string.str()); + } + else + { + mOSStringSimple.append("Unable to collect OS info"); + } + } + + // Initialize mOSString to something like: + // "Mac OS X 10.6.7 Darwin Kernel Version 10.7.0: Sat Jan 29 15:17:16 PST 2011; root:xnu-1504.9.37~1/RELEASE_I386 i386" + struct utsname un; + if(uname(&un) != -1) + { + mOSString = mOSStringSimple; + mOSString.append(" "); + mOSString.append(un.sysname); + mOSString.append(" "); + mOSString.append(un.release); + mOSString.append(" "); + mOSString.append(un.version); + mOSString.append(" "); + mOSString.append(un.machine); + } + else + { + mOSString = mOSStringSimple; + } + #else + struct utsname un; if(uname(&un) != -1) { @@ -334,15 +429,7 @@ LLOSInfo::LLOSInfo() : // Simplify 'Simple' std::string ostype = mOSStringSimple.substr(0, mOSStringSimple.find_first_of(" ", 0)); - if (ostype == "Darwin") - { - // Only care about major Darwin versions, truncate at first '.' - S32 idx1 = mOSStringSimple.find_first_of(".", 0); - std::string simple = mOSStringSimple.substr(0, idx1); - if (simple.length() > 0) - mOSStringSimple = simple; - } - else if (ostype == "Linux") + if (ostype == "Linux") { // Only care about major and minor Linux versions, truncate at second '.' std::string::size_type idx1 = mOSStringSimple.find_first_of(".", 0); @@ -562,8 +649,78 @@ void LLCPUInfo::stream(std::ostream& s) const s << "->mCPUString: " << mCPUString << std::endl; } +// Helper class for LLMemoryInfo: accumulate stats in the form we store for +// LLMemoryInfo::getStatsMap(). +class Stats +{ +public: + Stats(): + mStats(LLSD::emptyMap()) + {} + + // Store every integer type as LLSD::Integer. + template <class T> + void add(const LLSD::String& name, const T& value, + typename boost::enable_if<boost::is_integral<T> >::type* = 0) + { + mStats[name] = LLSD::Integer(value); + } + + // Store every floating-point type as LLSD::Real. + template <class T> + void add(const LLSD::String& name, const T& value, + typename boost::enable_if<boost::is_float<T> >::type* = 0) + { + mStats[name] = LLSD::Real(value); + } + + // Hope that LLSD::Date values are sufficiently unambiguous. + void add(const LLSD::String& name, const LLSD::Date& value) + { + mStats[name] = value; + } + + LLSD get() const { return mStats; } + +private: + LLSD mStats; +}; + +// Wrap boost::regex_match() with a function that doesn't throw. +template <typename S, typename M, typename R> +static bool regex_match_no_exc(const S& string, M& match, const R& regex) +{ + try + { + return boost::regex_match(string, match, regex); + } + catch (const std::runtime_error& e) + { + LL_WARNS("LLMemoryInfo") << "error matching with '" << regex.str() << "': " + << e.what() << ":\n'" << string << "'" << LL_ENDL; + return false; + } +} + +// Wrap boost::regex_search() with a function that doesn't throw. +template <typename S, typename M, typename R> +static bool regex_search_no_exc(const S& string, M& match, const R& regex) +{ + try + { + return boost::regex_search(string, match, regex); + } + catch (const std::runtime_error& e) + { + LL_WARNS("LLMemoryInfo") << "error searching with '" << regex.str() << "': " + << e.what() << ":\n'" << string << "'" << LL_ENDL; + return false; + } +} + LLMemoryInfo::LLMemoryInfo() { + refresh(); } #if LL_WINDOWS @@ -587,11 +744,7 @@ static U32 LLMemoryAdjustKBResult(U32 inKB) U32 LLMemoryInfo::getPhysicalMemoryKB() const { #if LL_WINDOWS - MEMORYSTATUSEX state; - state.dwLength = sizeof(state); - GlobalMemoryStatusEx(&state); - - return LLMemoryAdjustKBResult((U32)(state.ullTotalPhys >> 10)); + return LLMemoryAdjustKBResult(mStatsMap["Total Physical KB"].asInteger()); #elif LL_DARWIN // This might work on Linux as well. Someone check... @@ -639,12 +792,82 @@ U32 LLMemoryInfo::getPhysicalMemoryClamped() const void LLMemoryInfo::getAvailableMemoryKB(U32& avail_physical_mem_kb, U32& avail_virtual_mem_kb) { #if LL_WINDOWS - MEMORYSTATUSEX state; - state.dwLength = sizeof(state); - GlobalMemoryStatusEx(&state); + // Sigh, this shouldn't be a static method, then we wouldn't have to + // reload this data separately from refresh() + LLSD statsMap(loadStatsMap()); + + avail_physical_mem_kb = statsMap["Avail Physical KB"].asInteger(); + avail_virtual_mem_kb = statsMap["Avail Virtual KB"].asInteger(); - avail_physical_mem_kb = (U32)(state.ullAvailPhys/1024) ; - avail_virtual_mem_kb = (U32)(state.ullAvailVirtual/1024) ; +#elif LL_DARWIN + // mStatsMap is derived from vm_stat, look for (e.g.) "kb free": + // $ vm_stat + // Mach Virtual Memory Statistics: (page size of 4096 bytes) + // Pages free: 462078. + // Pages active: 142010. + // Pages inactive: 220007. + // Pages wired down: 159552. + // "Translation faults": 220825184. + // Pages copy-on-write: 2104153. + // Pages zero filled: 167034876. + // Pages reactivated: 65153. + // Pageins: 2097212. + // Pageouts: 41759. + // Object cache: 841598 hits of 7629869 lookups (11% hit rate) + avail_physical_mem_kb = -1 ; + avail_virtual_mem_kb = -1 ; + +#elif LL_LINUX + // mStatsMap is derived from MEMINFO_FILE: + // $ cat /proc/meminfo + // MemTotal: 4108424 kB + // MemFree: 1244064 kB + // Buffers: 85164 kB + // Cached: 1990264 kB + // SwapCached: 0 kB + // Active: 1176648 kB + // Inactive: 1427532 kB + // Active(anon): 529152 kB + // Inactive(anon): 15924 kB + // Active(file): 647496 kB + // Inactive(file): 1411608 kB + // Unevictable: 16 kB + // Mlocked: 16 kB + // HighTotal: 3266316 kB + // HighFree: 721308 kB + // LowTotal: 842108 kB + // LowFree: 522756 kB + // SwapTotal: 6384632 kB + // SwapFree: 6384632 kB + // Dirty: 28 kB + // Writeback: 0 kB + // AnonPages: 528820 kB + // Mapped: 89472 kB + // Shmem: 16324 kB + // Slab: 159624 kB + // SReclaimable: 145168 kB + // SUnreclaim: 14456 kB + // KernelStack: 2560 kB + // PageTables: 5560 kB + // NFS_Unstable: 0 kB + // Bounce: 0 kB + // WritebackTmp: 0 kB + // CommitLimit: 8438844 kB + // Committed_AS: 1271596 kB + // VmallocTotal: 122880 kB + // VmallocUsed: 65252 kB + // VmallocChunk: 52356 kB + // HardwareCorrupted: 0 kB + // HugePages_Total: 0 + // HugePages_Free: 0 + // HugePages_Rsvd: 0 + // HugePages_Surp: 0 + // Hugepagesize: 2048 kB + // DirectMap4k: 434168 kB + // DirectMap2M: 477184 kB + // (could also run 'free', but easier to read a file than run a program) + avail_physical_mem_kb = -1 ; + avail_virtual_mem_kb = -1 ; #else //do not know how to collect available memory info for other systems. @@ -657,56 +880,283 @@ void LLMemoryInfo::getAvailableMemoryKB(U32& avail_physical_mem_kb, U32& avail_v void LLMemoryInfo::stream(std::ostream& s) const { + // We want these memory stats to be easy to grep from the log, along with + // the timestamp. So preface each line with the timestamp and a + // distinctive marker. Without that, we'd have to search the log for the + // introducer line, then read subsequent lines, etc... + std::string pfx(LLError::utcTime() + " <mem> "); + + // Max key length + size_t key_width(0); + BOOST_FOREACH(const MapEntry& pair, inMap(mStatsMap)) + { + size_t len(pair.first.length()); + if (len > key_width) + { + key_width = len; + } + } + + // Now stream stats + BOOST_FOREACH(const MapEntry& pair, inMap(mStatsMap)) + { + s << pfx << std::setw(key_width+1) << (pair.first + ':') << ' '; + LLSD value(pair.second); + if (value.isInteger()) + s << std::setw(12) << value.asInteger(); + else if (value.isReal()) + s << std::fixed << std::setprecision(1) << value.asReal(); + else if (value.isDate()) + value.asDate().toStream(s); + else + s << value; // just use default LLSD formatting + s << std::endl; + } +} + +LLSD LLMemoryInfo::getStatsMap() const +{ + return mStatsMap; +} + +LLMemoryInfo& LLMemoryInfo::refresh() +{ + mStatsMap = loadStatsMap(); + + LL_DEBUGS("LLMemoryInfo") << "Populated mStatsMap:\n"; + LLSDSerialize::toPrettyXML(mStatsMap, LL_CONT); + LL_ENDL; + + return *this; +} + +LLSD LLMemoryInfo::loadStatsMap() +{ + // This implementation is derived from stream() code (as of 2011-06-29). + Stats stats; + + // associate timestamp for analysis over time + stats.add("timestamp", LLDate::now()); + #if LL_WINDOWS MEMORYSTATUSEX state; state.dwLength = sizeof(state); GlobalMemoryStatusEx(&state); - s << "Percent Memory use: " << (U32)state.dwMemoryLoad << '%' << std::endl; - s << "Total Physical KB: " << (U32)(state.ullTotalPhys/1024) << std::endl; - s << "Avail Physical KB: " << (U32)(state.ullAvailPhys/1024) << std::endl; - s << "Total page KB: " << (U32)(state.ullTotalPageFile/1024) << std::endl; - s << "Avail page KB: " << (U32)(state.ullAvailPageFile/1024) << std::endl; - s << "Total Virtual KB: " << (U32)(state.ullTotalVirtual/1024) << std::endl; - s << "Avail Virtual KB: " << (U32)(state.ullAvailVirtual/1024) << std::endl; + stats.add("Percent Memory use", state.dwMemoryLoad); + stats.add("Total Physical KB", state.ullTotalPhys/1024); + stats.add("Avail Physical KB", state.ullAvailPhys/1024); + stats.add("Total page KB", state.ullTotalPageFile/1024); + stats.add("Avail page KB", state.ullAvailPageFile/1024); + stats.add("Total Virtual KB", state.ullTotalVirtual/1024); + stats.add("Avail Virtual KB", state.ullAvailVirtual/1024); + + PERFORMANCE_INFORMATION perf; + perf.cb = sizeof(perf); + GetPerformanceInfo(&perf, sizeof(perf)); + + SIZE_T pagekb(perf.PageSize/1024); + stats.add("CommitTotal KB", perf.CommitTotal * pagekb); + stats.add("CommitLimit KB", perf.CommitLimit * pagekb); + stats.add("CommitPeak KB", perf.CommitPeak * pagekb); + stats.add("PhysicalTotal KB", perf.PhysicalTotal * pagekb); + stats.add("PhysicalAvail KB", perf.PhysicalAvailable * pagekb); + stats.add("SystemCache KB", perf.SystemCache * pagekb); + stats.add("KernelTotal KB", perf.KernelTotal * pagekb); + stats.add("KernelPaged KB", perf.KernelPaged * pagekb); + stats.add("KernelNonpaged KB", perf.KernelNonpaged * pagekb); + stats.add("PageSize KB", pagekb); + stats.add("HandleCount", perf.HandleCount); + stats.add("ProcessCount", perf.ProcessCount); + stats.add("ThreadCount", perf.ThreadCount); + + PROCESS_MEMORY_COUNTERS_EX pmem; + pmem.cb = sizeof(pmem); + // GetProcessMemoryInfo() is documented to accept either + // PROCESS_MEMORY_COUNTERS* or PROCESS_MEMORY_COUNTERS_EX*, presumably + // using the redundant size info to distinguish. But its prototype + // specifically accepts PROCESS_MEMORY_COUNTERS*, and since this is a + // classic-C API, PROCESS_MEMORY_COUNTERS_EX isn't a subclass. Cast the + // pointer. + GetProcessMemoryInfo(GetCurrentProcess(), PPROCESS_MEMORY_COUNTERS(&pmem), sizeof(pmem)); + + stats.add("Page Fault Count", pmem.PageFaultCount); + stats.add("PeakWorkingSetSize KB", pmem.PeakWorkingSetSize/1024); + stats.add("WorkingSetSize KB", pmem.WorkingSetSize/1024); + stats.add("QutaPeakPagedPoolUsage KB", pmem.QuotaPeakPagedPoolUsage/1024); + stats.add("QuotaPagedPoolUsage KB", pmem.QuotaPagedPoolUsage/1024); + stats.add("QuotaPeakNonPagedPoolUsage KB", pmem.QuotaPeakNonPagedPoolUsage/1024); + stats.add("QuotaNonPagedPoolUsage KB", pmem.QuotaNonPagedPoolUsage/1024); + stats.add("PagefileUsage KB", pmem.PagefileUsage/1024); + stats.add("PeakPagefileUsage KB", pmem.PeakPagefileUsage/1024); + stats.add("PrivateUsage KB", pmem.PrivateUsage/1024); + #elif LL_DARWIN - uint64_t phys = 0; - size_t len = sizeof(phys); + const vm_size_t pagekb(vm_page_size / 1024); + + // + // Collect the vm_stat's + // - if(sysctlbyname("hw.memsize", &phys, &len, NULL, 0) == 0) { - s << "Total Physical KB: " << phys/1024 << std::endl; - } - else + vm_statistics_data_t vmstat; + mach_msg_type_number_t vmstatCount = HOST_VM_INFO_COUNT; + + if (host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t) &vmstat, &vmstatCount) != KERN_SUCCESS) { - s << "Unable to collect memory information"; + LL_WARNS("LLMemoryInfo") << "Unable to collect memory information" << LL_ENDL; + } + else + { + stats.add("Pages free KB", pagekb * vmstat.free_count); + stats.add("Pages active KB", pagekb * vmstat.active_count); + stats.add("Pages inactive KB", pagekb * vmstat.inactive_count); + stats.add("Pages wired KB", pagekb * vmstat.wire_count); + + stats.add("Pages zero fill", vmstat.zero_fill_count); + stats.add("Page reactivations", vmstat.reactivations); + stats.add("Page-ins", vmstat.pageins); + stats.add("Page-outs", vmstat.pageouts); + + stats.add("Faults", vmstat.faults); + stats.add("Faults copy-on-write", vmstat.cow_faults); + + stats.add("Cache lookups", vmstat.lookups); + stats.add("Cache hits", vmstat.hits); + + stats.add("Page purgeable count", vmstat.purgeable_count); + stats.add("Page purges", vmstat.purges); + + stats.add("Page speculative reads", vmstat.speculative_count); + } + } + + // + // Collect the misc task info + // + + { + task_events_info_data_t taskinfo; + unsigned taskinfoSize = sizeof(taskinfo); + + if (task_info(mach_task_self(), TASK_EVENTS_INFO, (task_info_t) &taskinfo, &taskinfoSize) != KERN_SUCCESS) + { + LL_WARNS("LLMemoryInfo") << "Unable to collect task information" << LL_ENDL; + } + else + { + stats.add("Task page-ins", taskinfo.pageins); + stats.add("Task copy-on-write faults", taskinfo.cow_faults); + stats.add("Task messages sent", taskinfo.messages_sent); + stats.add("Task messages received", taskinfo.messages_received); + stats.add("Task mach system call count", taskinfo.syscalls_mach); + stats.add("Task unix system call count", taskinfo.syscalls_unix); + stats.add("Task context switch count", taskinfo.csw); + } + } + + // + // Collect the basic task info + // + + { + task_basic_info_64_data_t taskinfo; + unsigned taskinfoSize = sizeof(taskinfo); + + if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t) &taskinfo, &taskinfoSize) != KERN_SUCCESS) + { + LL_WARNS("LLMemoryInfo") << "Unable to collect task information" << LL_ENDL; + } + else + { + stats.add("Basic suspend count", taskinfo.suspend_count); + stats.add("Basic virtual memory KB", taskinfo.virtual_size / 1024); + stats.add("Basic resident memory KB", taskinfo.resident_size / 1024); + stats.add("Basic new thread policy", taskinfo.policy); + } } + #elif LL_SOLARIS - U64 phys = 0; + U64 phys = 0; - phys = (U64)(sysconf(_SC_PHYS_PAGES)) * (U64)(sysconf(_SC_PAGESIZE)/1024); + phys = (U64)(sysconf(_SC_PHYS_PAGES)) * (U64)(sysconf(_SC_PAGESIZE)/1024); - s << "Total Physical KB: " << phys << std::endl; -#else - // *NOTE: This works on linux. What will it do on other systems? - LLFILE* meminfo = LLFile::fopen(MEMINFO_FILE,"rb"); - if(meminfo) + stats.add("Total Physical KB", phys); + +#elif LL_LINUX + std::ifstream meminfo(MEMINFO_FILE); + if (meminfo.is_open()) { - char line[MAX_STRING]; /* Flawfinder: ignore */ - memset(line, 0, MAX_STRING); - while(fgets(line, MAX_STRING, meminfo)) + // MemTotal: 4108424 kB + // MemFree: 1244064 kB + // Buffers: 85164 kB + // Cached: 1990264 kB + // SwapCached: 0 kB + // Active: 1176648 kB + // Inactive: 1427532 kB + // ... + // VmallocTotal: 122880 kB + // VmallocUsed: 65252 kB + // VmallocChunk: 52356 kB + // HardwareCorrupted: 0 kB + // HugePages_Total: 0 + // HugePages_Free: 0 + // HugePages_Rsvd: 0 + // HugePages_Surp: 0 + // Hugepagesize: 2048 kB + // DirectMap4k: 434168 kB + // DirectMap2M: 477184 kB + + // Intentionally don't pass the boost::no_except flag. This + // boost::regex object is constructed with a string literal, so it + // should be valid every time. If it becomes invalid, we WANT an + // exception, hopefully even before the dev checks in. + boost::regex stat_rx("(.+): +([0-9]+)( kB)?"); + boost::smatch matched; + + std::string line; + while (std::getline(meminfo, line)) { - line[strlen(line)-1] = ' '; /*Flawfinder: ignore*/ - s << line; + LL_DEBUGS("LLMemoryInfo") << line << LL_ENDL; + if (regex_match_no_exc(line, matched, stat_rx)) + { + // e.g. "MemTotal: 4108424 kB" + LLSD::String key(matched[1].first, matched[1].second); + LLSD::String value_str(matched[2].first, matched[2].second); + LLSD::Integer value(0); + try + { + value = boost::lexical_cast<LLSD::Integer>(value_str); + } + catch (const boost::bad_lexical_cast&) + { + LL_WARNS("LLMemoryInfo") << "couldn't parse '" << value_str + << "' in " << MEMINFO_FILE << " line: " + << line << LL_ENDL; + continue; + } + // Store this statistic. + stats.add(key, value); + } + else + { + LL_WARNS("LLMemoryInfo") << "unrecognized " << MEMINFO_FILE << " line: " + << line << LL_ENDL; + } } - fclose(meminfo); } else { - s << "Unable to collect memory information"; + LL_WARNS("LLMemoryInfo") << "Unable to collect memory information" << LL_ENDL; } + +#else + LL_WARNS("LLMemoryInfo") << "Unknown system; unable to collect memory information" << LL_ENDL; + #endif + + return stats.get(); } std::ostream& operator<<(std::ostream& s, const LLOSInfo& info) @@ -727,6 +1177,143 @@ std::ostream& operator<<(std::ostream& s, const LLMemoryInfo& info) return s; } +class FrameWatcher +{ +public: + FrameWatcher(): + // Hooking onto the "mainloop" event pump gets us one call per frame. + mConnection(LLEventPumps::instance() + .obtain("mainloop") + .listen("FrameWatcher", boost::bind(&FrameWatcher::tick, this, _1))), + // Initializing mSampleStart to an invalid timestamp alerts us to skip + // trying to compute framerate on the first call. + mSampleStart(-1), + // Initializing mSampleEnd to 0 ensures that we treat the first call + // as the completion of a sample window. + mSampleEnd(0), + mFrames(0), + // Both MEM_INFO_WINDOW and MEM_INFO_THROTTLE are in seconds. We need + // the number of integer MEM_INFO_THROTTLE sample slots that will fit + // in MEM_INFO_WINDOW. Round up. + mSamples(int((MEM_INFO_WINDOW / MEM_INFO_THROTTLE) + 0.7)), + // Initializing to F32_MAX means that the first real frame will become + // the slowest ever, which sounds like a good idea. + mSlowest(F32_MAX) + {} + + bool tick(const LLSD&) + { + F32 timestamp(mTimer.getElapsedTimeF32()); + + // Count this frame in the interval just completed. + ++mFrames; + + // Have we finished a sample window yet? + if (timestamp < mSampleEnd) + { + // no, just keep waiting + return false; + } + + // Set up for next sample window. Capture values for previous frame in + // local variables and reset data members. + U32 frames(mFrames); + F32 sampleStart(mSampleStart); + // No frames yet in next window + mFrames = 0; + // which starts right now + mSampleStart = timestamp; + // and ends MEM_INFO_THROTTLE seconds in the future + mSampleEnd = mSampleStart + MEM_INFO_THROTTLE; + + // On the very first call, that's all we can do, no framerate + // computation is possible. + if (sampleStart < 0) + { + return false; + } + + // How long did this actually take? As framerate slows, the duration + // of the frame we just finished could push us WELL beyond our desired + // sample window size. + F32 elapsed(timestamp - sampleStart); + F32 framerate(frames/elapsed); + + // Remember previous slowest framerate because we're just about to + // update it. + F32 slowest(mSlowest); + // Remember previous number of samples. + boost::circular_buffer<F32>::size_type prevSize(mSamples.size()); + + // Capture new framerate in our samples buffer. Once the buffer is + // full (after MEM_INFO_WINDOW seconds), this will displace the oldest + // sample. ("So they all rolled over, and one fell out...") + mSamples.push_back(framerate); + + // Calculate the new minimum framerate. I know of no way to update a + // rolling minimum without ever rescanning the buffer. But since there + // are only a few tens of items in this buffer, rescanning it is + // probably cheaper (and certainly easier to reason about) than + // attempting to optimize away some of the scans. + mSlowest = framerate; // pick an arbitrary entry to start + for (boost::circular_buffer<F32>::const_iterator si(mSamples.begin()), send(mSamples.end()); + si != send; ++si) + { + if (*si < mSlowest) + { + mSlowest = *si; + } + } + + // We're especially interested in memory as framerate drops. Only log + // when framerate drops below the slowest framerate we remember. + // (Should always be true for the end of the very first sample + // window.) + if (framerate >= slowest) + { + return false; + } + // Congratulations, we've hit a new low. :-P + + LL_INFOS("FrameWatcher") << ' '; + if (! prevSize) + { + LL_CONT << "initial framerate "; + } + else + { + LL_CONT << "slowest framerate for last " << int(prevSize * MEM_INFO_THROTTLE) + << " seconds "; + } + LL_CONT << std::fixed << std::setprecision(1) << framerate << '\n' + << LLMemoryInfo() << LL_ENDL; + + return false; + } + +private: + // Storing the connection in an LLTempBoundListener ensures it will be + // disconnected when we're destroyed. + LLTempBoundListener mConnection; + // Track elapsed time + LLTimer mTimer; + // Some of what you see here is in fact redundant with functionality you + // can get from LLTimer. Unfortunately the LLTimer API is missing the + // feature we need: has at least the stated interval elapsed, and if so, + // exactly how long has passed? So we have to do it by hand, sigh. + // Time at start, end of sample window + F32 mSampleStart, mSampleEnd; + // Frames this sample window + U32 mFrames; + // Sliding window of framerate samples + boost::circular_buffer<F32> mSamples; + // Slowest framerate in mSamples + F32 mSlowest; +}; + +// Need an instance of FrameWatcher before it does any good +static FrameWatcher sFrameWatcher; + BOOL gunzip_file(const std::string& srcfile, const std::string& dstfile) { std::string tmpfile; diff --git a/indra/llcommon/llsys.h b/indra/llcommon/llsys.h index 41a4f25000..739e795d3a 100644 --- a/indra/llcommon/llsys.h +++ b/indra/llcommon/llsys.h @@ -36,6 +36,7 @@ // llinfos << info << llendl; // +#include "llsd.h" #include <iosfwd> #include <string> @@ -117,6 +118,27 @@ public: //get the available memory infomation in KiloBytes. static void getAvailableMemoryKB(U32& avail_physical_mem_kb, U32& avail_virtual_mem_kb); + + // Retrieve a map of memory statistics. The keys of the map are platform- + // dependent. The values are in kilobytes to try to avoid integer overflow. + LLSD getStatsMap() const; + + // Re-fetch memory data (as reported by stream() and getStatsMap()) from the + // system. Normally this is fetched at construction time. Return (*this) + // to permit usage of the form: + // @code + // LLMemoryInfo info; + // ... + // info.refresh().getStatsMap(); + // @endcode + LLMemoryInfo& refresh(); + +private: + // set mStatsMap + static LLSD loadStatsMap(); + + // Memory stats for getStatsMap(). + LLSD mStatsMap; }; diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index 49d05ef411..ed4f9eb376 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -36,6 +36,12 @@ #include <sched.h> #endif +#if !LL_DARWIN +U32 ll_thread_local local_thread_ID = 0; +#endif + +U32 LLThread::sIDIter = 0; + //---------------------------------------------------------------------------- // Usage: // void run_func(LLThread* thread) @@ -56,6 +62,15 @@ // //---------------------------------------------------------------------------- +LL_COMMON_API void assert_main_thread() +{ + static U32 s_thread_id = LLThread::currentID(); + if (LLThread::currentID() != s_thread_id) + { + llerrs << "Illegal execution outside main thread." << llendl; + } +} + // // Handed to the APR thread creation function // @@ -63,10 +78,14 @@ void *APR_THREAD_FUNC LLThread::staticRun(apr_thread_t *apr_threadp, void *datap { LLThread *threadp = (LLThread *)datap; +#if !LL_DARWIN + local_thread_ID = threadp->mID; +#endif + // Run the user supplied function threadp->run(); - llinfos << "LLThread::staticRun() Exiting: " << threadp->mName << llendl; + //llinfos << "LLThread::staticRun() Exiting: " << threadp->mName << llendl; // We're done with the run function, this thread is done executing now. threadp->mStatus = STOPPED; @@ -121,7 +140,7 @@ void LLThread::shutdown() // First, set the flag that indicates that we're ready to die setQuitting(); - llinfos << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << llendl; + //llinfos << "LLThread::~LLThread() Killing thread " << mName << " Status: " << mStatus << llendl; // Now wait a bit for the thread to exit // It's unclear whether I should even bother doing this - this destructor // should netver get called unless we're already stopped, really... @@ -143,7 +162,7 @@ void LLThread::shutdown() if (!isStopped()) { // This thread just wouldn't stop, even though we gave it time - llwarns << "LLThread::~LLThread() exiting thread before clean exit!" << llendl; + //llwarns << "LLThread::shutdown() exiting thread before clean exit!" << llendl; // Put a stake in its heart. apr_thread_exit(mAPRThreadp, -1); return; @@ -385,6 +404,44 @@ void LLCondition::broadcast() } //============================================================================ +LLMutexBase::LLMutexBase() : + mLockingThread(NO_THREAD), + mCount(0) +{ +} + +void LLMutexBase::lock() +{ +#if LL_DARWIN + if (mLockingThread == LLThread::currentID()) +#else + if (mLockingThread == local_thread_ID) +#endif + { //redundant lock + mCount++; + return; + } + + apr_thread_mutex_lock(mAPRMutexp); + +#if LL_DARWIN + mLockingThread = LLThread::currentID(); +#else + mLockingThread = local_thread_ID; +#endif +} + +void LLMutexBase::unlock() +{ + if (mCount > 0) + { //not the root unlock + mCount--; + return; + } + mLockingThread = NO_THREAD; + + apr_thread_mutex_unlock(mAPRMutexp); +} //---------------------------------------------------------------------------- diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index f1c6cd75af..ad4a6523a1 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -35,8 +35,17 @@ class LLThread; class LLMutex; class LLCondition; +#if LL_WINDOWS +#define ll_thread_local __declspec(thread) +#else +#define ll_thread_local __thread +#endif + class LL_COMMON_API LLThread { +private: + static U32 sIDIter; + public: typedef enum e_thread_status { @@ -91,7 +100,8 @@ protected: apr_pool_t *mAPRPoolp; BOOL mIsLocalPool; EThreadStatus mStatus; - + U32 mID; + //a local apr_pool for APRFile operations in this thread. If it exists, LLAPRFile::sAPRFilePoolp should not be used. //Note: this pool is used by APRFile ONLY, do NOT use it for any other purposes. // otherwise it will cause severe memory leaking!!! --bao diff --git a/indra/llcommon/llversionviewer.h b/indra/llcommon/llversionviewer.h index 7d5afe92dc..3377465bb6 100644 --- a/indra/llcommon/llversionviewer.h +++ b/indra/llcommon/llversionviewer.h @@ -27,8 +27,8 @@ #ifndef LL_LLVERSIONVIEWER_H #define LL_LLVERSIONVIEWER_H -const S32 LL_VERSION_MAJOR = 2; -const S32 LL_VERSION_MINOR = 6; +const S32 LL_VERSION_MAJOR = 3; +const S32 LL_VERSION_MINOR = 1; const S32 LL_VERSION_PATCH = 0; const S32 LL_VERSION_BUILD = 0; diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index 3ac50832fd..4988bdf570 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -34,8 +34,8 @@ //============================================================================ // Run on MAIN thread -LLWorkerThread::LLWorkerThread(const std::string& name, bool threaded) : - LLQueuedThread(name, threaded) +LLWorkerThread::LLWorkerThread(const std::string& name, bool threaded, bool should_pause) : + LLQueuedThread(name, threaded, should_pause) { mDeleteMutex = new LLMutex(NULL); diff --git a/indra/llcommon/llworkerthread.h b/indra/llcommon/llworkerthread.h index 9bff18303e..78a4781d15 100644 --- a/indra/llcommon/llworkerthread.h +++ b/indra/llcommon/llworkerthread.h @@ -83,7 +83,7 @@ private: LLMutex* mDeleteMutex; public: - LLWorkerThread(const std::string& name, bool threaded = true); + LLWorkerThread(const std::string& name, bool threaded = true, bool should_pause = false); ~LLWorkerThread(); /*virtual*/ S32 update(U32 max_time_ms); diff --git a/indra/llcommon/stdenums.h b/indra/llcommon/stdenums.h index 9f86de124e..556eff8370 100644 --- a/indra/llcommon/stdenums.h +++ b/indra/llcommon/stdenums.h @@ -49,7 +49,8 @@ enum EDragAndDropType DAD_ANIMATION = 12, DAD_GESTURE = 13, DAD_LINK = 14, - DAD_COUNT = 15, // number of types in this enum + DAD_MESH = 15, + DAD_COUNT = 16, // number of types in this enum }; // Reasons for drags to be denied. diff --git a/indra/llcommon/tests/lldependencies_test.cpp b/indra/llcommon/tests/lldependencies_test.cpp index e40743ccf7..5395d785b6 100644 --- a/indra/llcommon/tests/lldependencies_test.cpp +++ b/indra/llcommon/tests/lldependencies_test.cpp @@ -258,10 +258,10 @@ namespace tut ++const_iterator; ensure_equals(const_iterator->first, "def"); ensure_equals(const_iterator->second, 2); - NameIndexDeps::node_range node_range(nideps.get_node_range()); - ensure_equals(instance_from_range<std::vector<int> >(node_range), make< std::vector<int> >(list_of(1)(2)(3))); - *node_range.begin() = 0; - *node_range.begin() = 1; +// NameIndexDeps::node_range node_range(nideps.get_node_range()); +// ensure_equals(instance_from_range<std::vector<int> >(node_range), make< std::vector<int> >(list_of(1)(2)(3))); +// *node_range.begin() = 0; +// *node_range.begin() = 1; NameIndexDeps::const_node_range const_node_range(const_nideps.get_node_range()); ensure_equals(instance_from_range<std::vector<int> >(const_node_range), make< std::vector<int> >(list_of(1)(2)(3))); NameIndexDeps::const_key_range const_key_range(const_nideps.get_key_range()); @@ -278,8 +278,8 @@ namespace tut def); ensure_equals(instance_from_range<StringList>(const_nideps.get_after_range(const_nideps.get_range().begin())), def); - ensure_equals(instance_from_range<StringList>(nideps.get_after_range(nideps.get_node_range().begin())), - def); +// ensure_equals(instance_from_range<StringList>(nideps.get_after_range(nideps.get_node_range().begin())), +// def); ensure_equals(instance_from_range<StringList>(const_nideps.get_after_range(const_nideps.get_node_range().begin())), def); ensure_equals(instance_from_range<StringList>(nideps.get_after_range(nideps.get_key_range().begin())), diff --git a/indra/llcommon/tests/llerror_test.cpp b/indra/llcommon/tests/llerror_test.cpp index 1ef8fc9712..09a20231de 100644 --- a/indra/llcommon/tests/llerror_test.cpp +++ b/indra/llcommon/tests/llerror_test.cpp @@ -48,7 +48,10 @@ namespace { static bool fatalWasCalled; void fatalCall(const std::string&) { fatalWasCalled = true; } +} +namespace tut +{ class TestRecorder : public LLError::Recorder { public: @@ -56,7 +59,7 @@ namespace ~TestRecorder() { LLError::removeRecorder(this); } void recordMessage(LLError::ELevel level, - const std::string& message) + const std::string& message) { mMessages.push_back(message); } @@ -66,12 +69,12 @@ namespace void setWantsTime(bool t) { mWantsTime = t; } bool wantsTime() { return mWantsTime; } - + std::string message(int n) { std::ostringstream test_name; test_name << "testing message " << n << ", not enough messages"; - + tut::ensure(test_name.str(), n < countMessages()); return mMessages[n]; } @@ -82,10 +85,7 @@ namespace bool mWantsTime; }; -} - -namespace tut -{ + struct ErrorTestData { TestRecorder mRecorder; @@ -381,7 +381,7 @@ namespace } typedef std::string (*LogFromFunction)(bool); - void testLogName(TestRecorder& recorder, LogFromFunction f, + void testLogName(tut::TestRecorder& recorder, LogFromFunction f, const std::string& class_name = "") { recorder.clearMessages(); diff --git a/indra/llcommon/tests/lleventdispatcher_test.cpp b/indra/llcommon/tests/lleventdispatcher_test.cpp new file mode 100644 index 0000000000..263c9b171f --- /dev/null +++ b/indra/llcommon/tests/lleventdispatcher_test.cpp @@ -0,0 +1,1324 @@ +/** + * @file lleventdispatcher_test.cpp + * @author Nat Goodspeed + * @date 2011-01-20 + * @brief Test for lleventdispatcher. + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Copyright (c) 2011, Linden Research, Inc. + * $/LicenseInfo$ + */ + +// Precompiled header +#include "linden_common.h" +// associated header +#include "lleventdispatcher.h" +// STL headers +// std headers +// external library headers +// other Linden headers +#include "../test/lltut.h" +#include "llsd.h" +#include "llsdutil.h" +#include "stringize.h" +#include "tests/wrapllerrs.h" + +#include <map> +#include <string> +#include <stdexcept> + +#include <boost/bind.hpp> +#include <boost/function.hpp> +#include <boost/range.hpp> +#include <boost/foreach.hpp> +#define foreach BOOST_FOREACH + +#include <boost/lambda/lambda.hpp> + +#include <iostream> +#include <iomanip> + +using boost::lambda::constant; +using boost::lambda::constant_ref; +using boost::lambda::var; + +using namespace llsd; + +/***************************************************************************** +* Output control +*****************************************************************************/ +#ifdef DEBUG_ON +using std::cout; +#else +static std::ostringstream cout; +#endif + +/***************************************************************************** +* Example data, functions, classes +*****************************************************************************/ +// We don't need a whole lot of different arbitrary-params methods, just (no | +// (const LLSD&) | arbitrary) args (function | static method | non-static +// method), where 'arbitrary' is (every LLSD datatype + (const char*)). +// But we need to register each one under different names for the different +// registration styles. Don't forget LLEventDispatcher subclass methods(const +// LLSD&). + +// However, the number of target parameter conversions we want to try exceeds +// boost::fusion::invoke()'s supported parameter-list size. Break out two +// different lists. +#define NPARAMSa bool b, int i, float f, double d, const char* cp +#define NPARAMSb const std::string& s, const LLUUID& uuid, const LLDate& date, \ + const LLURI& uri, const std::vector<U8>& bin +#define NARGSa b, i, f, d, cp +#define NARGSb s, uuid, date, uri, bin + +// For some registration methods we need methods on a subclass of +// LLEventDispatcher. To simplify things, we'll use this Dispatcher subclass +// for all our testing, including testing its own methods. +class Dispatcher: public LLEventDispatcher +{ +public: + Dispatcher(const std::string& name, const std::string& key): + LLEventDispatcher(name, key) + {} + + // sensing member, mutable because we want to know when we've reached our + // const method too + mutable LLSD llsd; + + void method1(const LLSD& obj) { llsd = obj; } + void cmethod1(const LLSD& obj) const { llsd = obj; } +}; + +// sensing vars, captured in a struct to make it convenient to clear them +struct Vars +{ + LLSD llsd; + bool b; + int i; + float f; + double d; + // Capture param passed as char*. But merely storing a char* received from + // our caller, possibly the .c_str() from a concatenation expression, + // would be Bad: the pointer will be invalidated long before we can query + // it. We could allocate a new chunk of memory, copy the string data and + // point to that instead -- but hey, guess what, we already have a class + // that does that! + std::string cp; + std::string s; + LLUUID uuid; + LLDate date; + LLURI uri; + std::vector<U8> bin; + + Vars(): + // Only need to initialize the POD types, the rest should take care of + // default-constructing themselves. + b(false), + i(0), + f(0), + d(0) + {} + + // Detect any non-default values for convenient testing + LLSD inspect() const + { + LLSD result; + + if (llsd.isDefined()) + result["llsd"] = llsd; + if (b) + result["b"] = b; + if (i) + result["i"] = i; + if (f) + result["f"] = f; + if (d) + result["d"] = d; + if (! cp.empty()) + result["cp"] = cp; + if (! s.empty()) + result["s"] = s; + if (uuid != LLUUID()) + result["uuid"] = uuid; + if (date != LLDate()) + result["date"] = date; + if (uri != LLURI()) + result["uri"] = uri; + if (! bin.empty()) + result["bin"] = bin; + + return result; + } + + /*------------- no-args (non-const, const, static) methods -------------*/ + void method0() + { + cout << "method0()\n"; + i = 17; + } + + void cmethod0() const + { + cout << 'c'; + const_cast<Vars*>(this)->method0(); + } + + static void smethod0(); + + /*------------ Callable (non-const, const, static) methods -------------*/ + void method1(const LLSD& obj) + { + cout << "method1(" << obj << ")\n"; + llsd = obj; + } + + void cmethod1(const LLSD& obj) const + { + cout << 'c'; + const_cast<Vars*>(this)->method1(obj); + } + + static void smethod1(const LLSD& obj); + + /*-------- Arbitrary-params (non-const, const, static) methods ---------*/ + void methodna(NPARAMSa) + { + // Because our const char* param cp might be NULL, and because we + // intend to capture the value in a std::string, have to distinguish + // between the NULL value and any non-NULL value. Use a convention + // easy for a human reader: enclose any non-NULL value in single + // quotes, reserving the unquoted string "NULL" to represent a NULL ptr. + std::string vcp; + if (cp == NULL) + vcp = "NULL"; + else + vcp = std::string("'") + cp + "'"; + + cout << "methodna(" << b + << ", " << i + << ", " << f + << ", " << d + << ", " << vcp + << ")\n"; + + this->b = b; + this->i = i; + this->f = f; + this->d = d; + this->cp = vcp; + } + + void methodnb(NPARAMSb) + { + std::ostringstream vbin; + foreach(U8 byte, bin) + { + vbin << std::hex << std::setfill('0') << std::setw(2) << unsigned(byte); + } + + cout << "methodnb(" << "'" << s << "'" + << ", " << uuid + << ", " << date + << ", '" << uri << "'" + << ", " << vbin.str() + << ")\n"; + + this->s = s; + this->uuid = uuid; + this->date = date; + this->uri = uri; + this->bin = bin; + } + + void cmethodna(NPARAMSa) const + { + cout << 'c'; + const_cast<Vars*>(this)->methodna(NARGSa); + } + + void cmethodnb(NPARAMSb) const + { + cout << 'c'; + const_cast<Vars*>(this)->methodnb(NARGSb); + } + + static void smethodna(NPARAMSa); + static void smethodnb(NPARAMSb); +}; +/*------- Global Vars instance for free functions and static methods -------*/ +static Vars g; + +/*------------ Static Vars method implementations reference 'g' ------------*/ +void Vars::smethod0() +{ + cout << "smethod0() -> "; + g.method0(); +} + +void Vars::smethod1(const LLSD& obj) +{ + cout << "smethod1(" << obj << ") -> "; + g.method1(obj); +} + +void Vars::smethodna(NPARAMSa) +{ + cout << "smethodna(...) -> "; + g.methodna(NARGSa); +} + +void Vars::smethodnb(NPARAMSb) +{ + cout << "smethodnb(...) -> "; + g.methodnb(NARGSb); +} + +/*--------------------------- Reset global Vars ----------------------------*/ +void clear() +{ + g = Vars(); +} + +/*------------------- Free functions also reference 'g' --------------------*/ +void free0() +{ + cout << "free0() -> "; + g.method0(); +} + +void free1(const LLSD& obj) +{ + cout << "free1(" << obj << ") -> "; + g.method1(obj); +} + +void freena(NPARAMSa) +{ + cout << "freena(...) -> "; + g.methodna(NARGSa); +} + +void freenb(NPARAMSb) +{ + cout << "freenb(...) -> "; + g.methodnb(NARGSb); +} + +/***************************************************************************** +* TUT +*****************************************************************************/ +namespace tut +{ + struct lleventdispatcher_data + { + WrapLL_ERRS redirect; + Dispatcher work; + Vars v; + std::string name, desc; + // Capture our own copy of all registered functions' descriptions + typedef std::map<std::string, std::string> DescMap; + DescMap descs; + // Capture the Vars instance on which we expect each function to operate + typedef std::map<std::string, Vars*> VarsMap; + VarsMap funcvars; + // Required structure for Callables with requirements + LLSD required; + // Parameter names for freena(), freenb() + LLSD params; + // Full, partial defaults arrays for params for freena(), freenb() + LLSD dft_array_full, dft_array_partial; + // Start index of partial defaults arrays + const LLSD::Integer partial_offset; + // Full, partial defaults maps for params for freena(), freenb() + LLSD dft_map_full, dft_map_partial; + // Most of the above are indexed by "a" or "b". Useful to have an + // array containing those strings for iterating. + std::vector<LLSD::String> ab; + + lleventdispatcher_data(): + work("test dispatcher", "op"), + // map {d=double, array=[3 elements]} + required(LLSDMap("d", LLSD::Real(0))("array", LLSDArray(LLSD())(LLSD())(LLSD()))), + // first several params are required, last couple optional + partial_offset(3) + { + // This object is reconstructed for every test<n> method. But + // clear global variables every time too. + ::clear(); + + const char* abs[] = { "a", "b" }; + ab.assign(boost::begin(abs), boost::end(abs)); + + // Registration cases: + // - (Callable | subclass const method | subclass non-const method | + // non-subclass method) (with | without) required + // - (Free function | static method | non-static method), (no | arbitrary) params, + // array style + // - (Free function | static method | non-static method), (no | arbitrary) params, + // map style, (empty | partial | full) (array | map) defaults + // - Map-style errors: + // - (scalar | map) param names + // - defaults scalar + // - defaults array longer than params array + // - defaults map with plural unknown param names + + // I hate to have to write things twice, because of having to keep + // them consistent. If we had variadic functions, addf() would be + // a variadic method, capturing the name and desc and passing them + // plus "everything else" to work.add(). If I could return a pair + // and use that pair as the first two args to work.add(), I'd do + // that. But the best I can do with present C++ is to set two + // instance variables as a side effect of addf(), and pass those + // variables to each work.add() call. :-P + + /*------------------------- Callables --------------------------*/ + + // Arbitrary Callable with/out required params + addf("free1", "free1", &g); + work.add(name, desc, free1); + addf("free1_req", "free1", &g); + work.add(name, desc, free1, required); + // Subclass non-const method with/out required params + addf("Dmethod1", "method1", NULL); + work.add(name, desc, &Dispatcher::method1); + addf("Dmethod1_req", "method1", NULL); + work.add(name, desc, &Dispatcher::method1, required); + // Subclass const method with/out required params + addf("Dcmethod1", "cmethod1", NULL); + work.add(name, desc, &Dispatcher::cmethod1); + addf("Dcmethod1_req", "cmethod1", NULL); + work.add(name, desc, &Dispatcher::cmethod1, required); + // Non-subclass method with/out required params + addf("method1", "method1", &v); + work.add(name, desc, boost::bind(&Vars::method1, boost::ref(v), _1)); + addf("method1_req", "method1", &v); + work.add(name, desc, boost::bind(&Vars::method1, boost::ref(v), _1), required); + + /*--------------- Arbitrary params, array style ----------------*/ + + // (Free function | static method) with (no | arbitrary) params, array style + addf("free0_array", "free0", &g); + work.add(name, desc, free0); + addf("freena_array", "freena", &g); + work.add(name, desc, freena); + addf("freenb_array", "freenb", &g); + work.add(name, desc, freenb); + addf("smethod0_array", "smethod0", &g); + work.add(name, desc, &Vars::smethod0); + addf("smethodna_array", "smethodna", &g); + work.add(name, desc, &Vars::smethodna); + addf("smethodnb_array", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb); + // Non-static method with (no | arbitrary) params, array style + addf("method0_array", "method0", &v); + work.add(name, desc, &Vars::method0, boost::lambda::var(v)); + addf("methodna_array", "methodna", &v); + work.add(name, desc, &Vars::methodna, boost::lambda::var(v)); + addf("methodnb_array", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, boost::lambda::var(v)); + + /*---------------- Arbitrary params, map style -----------------*/ + + // We lay out each params list as an array, also each array of + // default values we'll register. We'll zip these into + // (param=value) maps. Why not define them as maps and just + // extract the keys and values to arrays? Because that wouldn't + // give us the right params-list order. + + // freena(), methodna(), cmethodna(), smethodna() all take same param list. + // Same for freenb() et al. + params = LLSDMap("a", LLSDArray("b")("i")("f")("d")("cp")) + ("b", LLSDArray("s")("uuid")("date")("uri")("bin")); + cout << "params:\n" << params << "\nparams[\"a\"]:\n" << params["a"] << "\nparams[\"b\"]:\n" << params["b"] << std::endl; + // default LLSD::Binary value + std::vector<U8> binary; + for (size_t ix = 0, h = 0xaa; ix < 6; ++ix, h += 0x11) + { + binary.push_back(h); + } + // Full defaults arrays. We actually don't care what the LLUUID or + // LLDate values are, as long as they're different from the + // LLUUID() and LLDate() default values so inspect() will report + // them. + dft_array_full = LLSDMap("a", LLSDArray(true)(17)(3.14)(123456.78)("classic")) + ("b", LLSDArray("string") + (LLUUID::generateNewID()) + (LLDate::now()) + (LLURI("http://www.ietf.org/rfc/rfc3986.txt")) + (binary)); + cout << "dft_array_full:\n" << dft_array_full << std::endl; + // Partial defaults arrays. + foreach(LLSD::String a, ab) + { + LLSD::Integer partition(std::min(partial_offset, dft_array_full[a].size())); + dft_array_partial[a] = + llsd_copy_array(dft_array_full[a].beginArray() + partition, + dft_array_full[a].endArray()); + } + cout << "dft_array_partial:\n" << dft_array_partial << std::endl; + + foreach(LLSD::String a, ab) + { + // Generate full defaults maps by zipping (params, dft_array_full). + dft_map_full[a] = zipmap(params[a], dft_array_full[a]); + + // Generate partial defaults map by zipping alternate entries from + // (params, dft_array_full). Part of the point of using map-style + // defaults is to allow any subset of the target function's + // parameters to be optional, not just the rightmost. + for (LLSD::Integer ix = 0, ixend = params[a].size(); ix < ixend; ix += 2) + { + dft_map_partial[a][params[a][ix].asString()] = dft_array_full[a][ix]; + } + } + cout << "dft_map_full:\n" << dft_map_full << "\ndft_map_partial:\n" << dft_map_partial << '\n'; + + // (Free function | static method) with (no | arbitrary) params, + // map style, no (empty array) defaults + addf("free0_map", "free0", &g); + work.add(name, desc, free0, LLSD::emptyArray()); + addf("smethod0_map", "smethod0", &g); + work.add(name, desc, &Vars::smethod0, LLSD::emptyArray()); + addf("freena_map_allreq", "freena", &g); + work.add(name, desc, freena, params["a"]); + addf("freenb_map_allreq", "freenb", &g); + work.add(name, desc, freenb, params["b"]); + addf("smethodna_map_allreq", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"]); + addf("smethodnb_map_allreq", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"]); + // Non-static method with (no | arbitrary) params, map style, no + // (empty array) defaults + addf("method0_map", "method0", &v); + work.add(name, desc, &Vars::method0, var(v), LLSD::emptyArray()); + addf("methodna_map_allreq", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"]); + addf("methodnb_map_allreq", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"]); + + // Except for the "more (array | map) defaults than params" error + // cases, tested separately below, the (partial | full)(array | + // map) defaults cases don't apply to no-params functions/methods. + // So eliminate free0, smethod0, method0 from the cases below. + + // (Free function | static method) with arbitrary params, map + // style, partial (array | map) defaults + addf("freena_map_leftreq", "freena", &g); + work.add(name, desc, freena, params["a"], dft_array_partial["a"]); + addf("freenb_map_leftreq", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_array_partial["b"]); + addf("smethodna_map_leftreq", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_array_partial["a"]); + addf("smethodnb_map_leftreq", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_array_partial["b"]); + addf("freena_map_skipreq", "freena", &g); + work.add(name, desc, freena, params["a"], dft_map_partial["a"]); + addf("freenb_map_skipreq", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_map_partial["b"]); + addf("smethodna_map_skipreq", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_map_partial["a"]); + addf("smethodnb_map_skipreq", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_map_partial["b"]); + // Non-static method with arbitrary params, map style, partial + // (array | map) defaults + addf("methodna_map_leftreq", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_array_partial["a"]); + addf("methodnb_map_leftreq", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_array_partial["b"]); + addf("methodna_map_skipreq", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_map_partial["a"]); + addf("methodnb_map_skipreq", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_map_partial["b"]); + + // (Free function | static method) with arbitrary params, map + // style, full (array | map) defaults + addf("freena_map_adft", "freena", &g); + work.add(name, desc, freena, params["a"], dft_array_full["a"]); + addf("freenb_map_adft", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_array_full["b"]); + addf("smethodna_map_adft", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_array_full["a"]); + addf("smethodnb_map_adft", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_array_full["b"]); + addf("freena_map_mdft", "freena", &g); + work.add(name, desc, freena, params["a"], dft_map_full["a"]); + addf("freenb_map_mdft", "freenb", &g); + work.add(name, desc, freenb, params["b"], dft_map_full["b"]); + addf("smethodna_map_mdft", "smethodna", &g); + work.add(name, desc, &Vars::smethodna, params["a"], dft_map_full["a"]); + addf("smethodnb_map_mdft", "smethodnb", &g); + work.add(name, desc, &Vars::smethodnb, params["b"], dft_map_full["b"]); + // Non-static method with arbitrary params, map style, full + // (array | map) defaults + addf("methodna_map_adft", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_array_full["a"]); + addf("methodnb_map_adft", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_array_full["b"]); + addf("methodna_map_mdft", "methodna", &v); + work.add(name, desc, &Vars::methodna, var(v), params["a"], dft_map_full["a"]); + addf("methodnb_map_mdft", "methodnb", &v); + work.add(name, desc, &Vars::methodnb, var(v), params["b"], dft_map_full["b"]); + + // All the above are expected to succeed, and are setup for the + // tests to follow. Registration error cases are exercised as + // tests rather than as test setup. + } + + void addf(const std::string& n, const std::string& d, Vars* v) + { + // This method is to capture in our own DescMap the name and + // description of every registered function, for metadata query + // testing. + descs[n] = d; + // Also capture the Vars instance on which each function should operate. + funcvars[n] = v; + // See constructor for rationale for setting these instance vars. + this->name = n; + this->desc = d; + } + + void verify_descs() + { + // Copy descs to a temp map of same type. + DescMap forgotten(descs.begin(), descs.end()); + // LLEventDispatcher intentionally provides only const_iterator: + // since dereferencing that iterator generates values on the fly, + // it's meaningless to have a modifiable iterator. But since our + // 'work' object isn't const, by default BOOST_FOREACH() wants to + // use non-const iterators. Persuade it to use the const_iterator. + foreach(LLEventDispatcher::NameDesc nd, const_cast<const Dispatcher&>(work)) + { + DescMap::iterator found = forgotten.find(nd.first); + ensure(STRINGIZE("LLEventDispatcher records function '" << nd.first + << "' we didn't enter"), + found != forgotten.end()); + ensure_equals(STRINGIZE("LLEventDispatcher desc '" << nd.second << + "' doesn't match what we entered: '" << found->second << "'"), + nd.second, found->second); + // found in our map the name from LLEventDispatcher, good, erase + // our map entry + forgotten.erase(found); + } + if (! forgotten.empty()) + { + std::ostringstream out; + out << "LLEventDispatcher failed to report"; + const char* delim = ": "; + foreach(const DescMap::value_type& fme, forgotten) + { + out << delim << fme.first; + delim = ", "; + } + ensure(out.str(), false); + } + } + + Vars* varsfor(const std::string& name) + { + VarsMap::const_iterator found = funcvars.find(name); + ensure(STRINGIZE("No Vars* for " << name), found != funcvars.end()); + ensure(STRINGIZE("NULL Vars* for " << name), found->second); + return found->second; + } + + void ensure_has(const std::string& outer, const std::string& inner) + { + ensure(STRINGIZE("'" << outer << "' does not contain '" << inner << "'").c_str(), + outer.find(inner) != std::string::npos); + } + + void call_exc(const std::string& func, const LLSD& args, const std::string& exc_frag) + { + std::string threw; + try + { + work(func, args); + } + catch (const std::runtime_error& e) + { + cout << "*** " << e.what() << '\n'; + threw = e.what(); + } + ensure_has(threw, exc_frag); + } + + LLSD getMetadata(const std::string& name) + { + LLSD meta(work.getMetadata(name)); + ensure(STRINGIZE("No metadata for " << name), meta.isDefined()); + return meta; + } + + // From two related LLSD arrays, e.g. a param-names array and a values + // array, zip them together into an LLSD map. + LLSD zipmap(const LLSD& keys, const LLSD& values) + { + LLSD map; + for (LLSD::Integer i = 0, iend = keys.size(); i < iend; ++i) + { + // Have to select asString() since you can index an LLSD + // object with either String or Integer. + map[keys[i].asString()] = values[i]; + } + return map; + } + + // If I call this ensure_equals(), it blocks visibility of all other + // ensure_equals() overloads. Normally I could say 'using + // baseclass::ensure_equals;' and fix that, but I don't know what the + // base class is! + void ensure_llsd(const std::string& msg, const LLSD& actual, const LLSD& expected, U32 bits) + { + std::ostringstream out; + if (! msg.empty()) + { + out << msg << ": "; + } + out << "expected " << expected << ", actual " << actual; + ensure(out.str(), llsd_equals(actual, expected, bits)); + } + + void ensure_llsd(const LLSD& actual, const LLSD& expected, U32 bits) + { + ensure_llsd("", actual, expected, bits); + } + }; + typedef test_group<lleventdispatcher_data> lleventdispatcher_group; + typedef lleventdispatcher_group::object object; + lleventdispatcher_group lleventdispatchergrp("lleventdispatcher"); + + // Call cases: + // - (try_call | call) (explicit name | event key) (real | bogus) name + // - Callable with args that (do | do not) match required + // - (Free function | non-static method), no args, (array | map) style + // - (Free function | non-static method), arbitrary args, + // (array style with (scalar | map) | map style with scalar) + // - (Free function | non-static method), arbitrary args, array style with + // array (too short | too long | just right) + // [trap LL_WARNS for too-long case?] + // - (Free function | non-static method), arbitrary args, map style with + // (array | map) (all | too many | holes (with | without) defaults) + // - const char* param gets ("" | NULL) + + // Query cases: + // - Iterate over all (with | without) remove() + // - getDispatchKey() + // - Callable style (with | without) required + // - (Free function | non-static method), array style, (no | arbitrary) params + // - (Free function | non-static method), map style, (no | arbitrary) params, + // (empty | full | partial (array | map)) defaults + + template<> template<> + void object::test<1>() + { + set_test_name("map-style registration with non-array params"); + // Pass "param names" as scalar or as map + LLSD attempts(LLSDArray(17)(LLSDMap("pi", 3.14)("two", 2))); + foreach(LLSD ae, inArray(attempts)) + { + std::string threw; + try + { + work.add("freena_err", "freena", freena, ae); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "must be an array"); + } + } + + template<> template<> + void object::test<2>() + { + set_test_name("map-style registration with badly-formed defaults"); + std::string threw; + try + { + work.add("freena_err", "freena", freena, LLSDArray("a")("b"), 17); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "must be a map or an array"); + } + + template<> template<> + void object::test<3>() + { + set_test_name("map-style registration with too many array defaults"); + std::string threw; + try + { + work.add("freena_err", "freena", freena, + LLSDArray("a")("b"), + LLSDArray(17)(0.9)("gack")); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "shorter than"); + } + + template<> template<> + void object::test<4>() + { + set_test_name("map-style registration with too many map defaults"); + std::string threw; + try + { + work.add("freena_err", "freena", freena, + LLSDArray("a")("b"), + LLSDMap("b", 17)("foo", 3.14)("bar", "sinister")); + } + catch (const std::exception& e) + { + threw = e.what(); + } + ensure_has(threw, "nonexistent params"); + ensure_has(threw, "foo"); + ensure_has(threw, "bar"); + } + + template<> template<> + void object::test<5>() + { + set_test_name("query all"); + verify_descs(); + } + + template<> template<> + void object::test<6>() + { + set_test_name("query all with remove()"); + ensure("remove('bogus') returned true", ! work.remove("bogus")); + ensure("remove('real') returned false", work.remove("free1")); + // Of course, remove that from 'descs' too... + descs.erase("free1"); + verify_descs(); + } + + template<> template<> + void object::test<7>() + { + set_test_name("getDispatchKey()"); + ensure_equals(work.getDispatchKey(), "op"); + } + + template<> template<> + void object::test<8>() + { + set_test_name("query Callables with/out required params"); + LLSD names(LLSDArray("free1")("Dmethod1")("Dcmethod1")("method1")); + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(metadata["desc"].asString(), descs[nm]); + ensure("should not have required structure", metadata["required"].isUndefined()); + ensure("should not have optional", metadata["optional"].isUndefined()); + + std::string name_req(nm.asString() + "_req"); + metadata = getMetadata(name_req); + ensure_equals(metadata["name"].asString(), name_req); + ensure_equals(metadata["desc"].asString(), descs[name_req]); + ensure_equals("required mismatch", required, metadata["required"]); + ensure("should not have optional", metadata["optional"].isUndefined()); + } + } + + template<> template<> + void object::test<9>() + { + set_test_name("query array-style functions/methods"); + // Associate each registered name with expected arity. + LLSD expected(LLSDArray + (LLSDArray + (0)(LLSDArray("free0_array")("smethod0_array")("method0_array"))) + (LLSDArray + (5)(LLSDArray("freena_array")("smethodna_array")("methodna_array"))) + (LLSDArray + (5)(LLSDArray("freenb_array")("smethodnb_array")("methodnb_array")))); + foreach(LLSD ae, inArray(expected)) + { + LLSD::Integer arity(ae[0].asInteger()); + LLSD names(ae[1]); + LLSD req(LLSD::emptyArray()); + if (arity) + req[arity - 1] = LLSD(); + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(metadata["desc"].asString(), descs[nm]); + ensure_equals(STRINGIZE("mismatched required for " << nm.asString()), + metadata["required"], req); + ensure("should not have optional", metadata["optional"].isUndefined()); + } + } + } + + template<> template<> + void object::test<10>() + { + set_test_name("query map-style no-params functions/methods"); + // - (Free function | non-static method), map style, no params (ergo + // no defaults) + LLSD names(LLSDArray("free0_map")("smethod0_map")("method0_map")); + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(metadata["desc"].asString(), descs[nm]); + ensure("should not have required", + (metadata["required"].isUndefined() || metadata["required"].size() == 0)); + ensure("should not have optional", metadata["optional"].isUndefined()); + } + } + + template<> template<> + void object::test<11>() + { + set_test_name("query map-style arbitrary-params functions/methods: " + "full array defaults vs. full map defaults"); + // With functions registered with no defaults ("_allreq" suffixes), + // there is of course no difference between array defaults and map + // defaults. (We don't even bother registering with LLSD::emptyArray() + // vs. LLSD::emptyMap().) With functions registered with all defaults, + // there should (!) be no difference beween array defaults and map + // defaults. Verify, so we can ignore the distinction for all other + // tests. + LLSD equivalences(LLSDArray + (LLSDArray("freena_map_adft")("freena_map_mdft")) + (LLSDArray("freenb_map_adft")("freenb_map_mdft")) + (LLSDArray("smethodna_map_adft")("smethodna_map_mdft")) + (LLSDArray("smethodnb_map_adft")("smethodnb_map_mdft")) + (LLSDArray("methodna_map_adft")("methodna_map_mdft")) + (LLSDArray("methodnb_map_adft")("methodnb_map_mdft"))); + foreach(LLSD eq, inArray(equivalences)) + { + LLSD adft(eq[0]); + LLSD mdft(eq[1]); + // We can't just compare the results of the two getMetadata() + // calls, because they contain ["name"], which are different. So + // capture them, verify that each ["name"] is as expected, then + // remove for comparing the rest. + LLSD ameta(getMetadata(adft)); + LLSD mmeta(getMetadata(mdft)); + ensure_equals("adft name", adft, ameta["name"]); + ensure_equals("mdft name", mdft, mmeta["name"]); + ameta.erase("name"); + mmeta.erase("name"); + ensure_equals(STRINGIZE("metadata for " << adft.asString() + << " vs. " << mdft.asString()), + ameta, mmeta); + } + } + + template<> template<> + void object::test<12>() + { + set_test_name("query map-style arbitrary-params functions/methods"); + // - (Free function | non-static method), map style, arbitrary params, + // (empty | full | partial (array | map)) defaults + + // Generate maps containing all parameter names for cases in which all + // params are required. Also maps containing left requirements for + // partial defaults arrays. Also defaults maps from defaults arrays. + LLSD allreq, leftreq, rightdft; + foreach(LLSD::String a, ab) + { + // The map in which all params are required uses params[a] as + // keys, with all isUndefined() as values. We can accomplish that + // by passing zipmap() an empty values array. + allreq[a] = zipmap(params[a], LLSD::emptyArray()); + // Same for leftreq, save that we use the subset of the params not + // supplied by dft_array_partial[a]. + LLSD::Integer partition(params[a].size() - dft_array_partial[a].size()); + leftreq[a] = zipmap(llsd_copy_array(params[a].beginArray(), + params[a].beginArray() + partition), + LLSD::emptyArray()); + // Generate map pairing dft_array_partial[a] values with their + // param names. + rightdft[a] = zipmap(llsd_copy_array(params[a].beginArray() + partition, + params[a].endArray()), + dft_array_partial[a]); + } + cout << "allreq:\n" << allreq << "\nleftreq:\n" << leftreq << "\nrightdft:\n" << rightdft << std::endl; + + // Generate maps containing parameter names not provided by the + // dft_map_partial maps. + LLSD skipreq(allreq); + foreach(LLSD::String a, ab) + { + foreach(const MapEntry& me, inMap(dft_map_partial[a])) + { + skipreq[a].erase(me.first); + } + } + cout << "skipreq:\n" << skipreq << std::endl; + + LLSD groups(LLSDArray // array of groups + + (LLSDArray // group + (LLSDArray("freena_map_allreq")("smethodna_map_allreq")("methodna_map_allreq")) + (LLSDArray(allreq["a"])(LLSD()))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_allreq")("smethodnb_map_allreq")("methodnb_map_allreq")) + (LLSDArray(allreq["b"])(LLSD()))) // required, optional + + (LLSDArray // group + (LLSDArray("freena_map_leftreq")("smethodna_map_leftreq")("methodna_map_leftreq")) + (LLSDArray(leftreq["a"])(rightdft["a"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_leftreq")("smethodnb_map_leftreq")("methodnb_map_leftreq")) + (LLSDArray(leftreq["b"])(rightdft["b"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freena_map_skipreq")("smethodna_map_skipreq")("methodna_map_skipreq")) + (LLSDArray(skipreq["a"])(dft_map_partial["a"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_skipreq")("smethodnb_map_skipreq")("methodnb_map_skipreq")) + (LLSDArray(skipreq["b"])(dft_map_partial["b"]))) // required, optional + + // We only need mention the full-map-defaults ("_mdft" suffix) + // registrations, having established their equivalence with the + // full-array-defaults ("_adft" suffix) registrations in another test. + (LLSDArray // group + (LLSDArray("freena_map_mdft")("smethodna_map_mdft")("methodna_map_mdft")) + (LLSDArray(LLSD::emptyMap())(dft_map_full["a"]))) // required, optional + + (LLSDArray // group + (LLSDArray("freenb_map_mdft")("smethodnb_map_mdft")("methodnb_map_mdft")) + (LLSDArray(LLSD::emptyMap())(dft_map_full["b"])))); // required, optional + + foreach(LLSD grp, inArray(groups)) + { + // Internal structure of each group in 'groups': + LLSD names(grp[0]); + LLSD required(grp[1][0]); + LLSD optional(grp[1][1]); + cout << "For " << names << ",\n" << "required:\n" << required << "\noptional:\n" << optional << std::endl; + + // Loop through 'names' + foreach(LLSD nm, inArray(names)) + { + LLSD metadata(getMetadata(nm)); + ensure_equals("name mismatch", metadata["name"], nm); + ensure_equals(nm.asString(), metadata["desc"].asString(), descs[nm]); + ensure_equals(STRINGIZE(nm << " required mismatch"), + metadata["required"], required); + ensure_equals(STRINGIZE(nm << " optional mismatch"), + metadata["optional"], optional); + } + } + } + + template<> template<> + void object::test<13>() + { + set_test_name("try_call()"); + ensure("try_call(bogus name, LLSD()) returned true", ! work.try_call("freek", LLSD())); + ensure("try_call(bogus name) returned true", ! work.try_call(LLSDMap("op", "freek"))); + ensure("try_call(real name, LLSD()) returned false", work.try_call("free0_array", LLSD())); + ensure("try_call(real name) returned false", work.try_call(LLSDMap("op", "free0_map"))); + } + + template<> template<> + void object::test<14>() + { + set_test_name("call with bad name"); + call_exc("freek", LLSD(), "not found"); + // We don't have a comparable helper function for the one-arg + // operator() method, and it's not worth building one just for this + // case. Write it out. + std::string threw; + try + { + work(LLSDMap("op", "freek")); + } + catch (const std::runtime_error& e) + { + cout << "*** " << e.what() << "\n"; + threw = e.what(); + } + ensure_has(threw, "bad"); + ensure_has(threw, "op"); + ensure_has(threw, "freek"); + } + + template<> template<> + void object::test<15>() + { + set_test_name("call with event key"); + // We don't need a separate test for operator()(string, LLSD) with + // valid name, because all the rest of the tests exercise that case. + // The one we don't exercise elsewhere is operator()(LLSD) with valid + // name, so here it is. + work(LLSDMap("op", "free0_map")); + ensure_equals(g.i, 17); + } + + // Cannot be defined inside function body... remind me again why we use C++... :-P + struct CallablesTriple + { + std::string name, name_req; + LLSD& llsd; + }; + + template<> template<> + void object::test<16>() + { + set_test_name("call Callables"); + CallablesTriple tests[] = + { + { "free1", "free1_req", g.llsd }, + { "Dmethod1", "Dmethod1_req", work.llsd }, + { "Dcmethod1", "Dcmethod1_req", work.llsd }, + { "method1", "method1_req", v.llsd } + }; + // Arbitrary LLSD value that we should be able to pass to Callables + // without 'required', but should not be able to pass to Callables + // with 'required'. + LLSD answer(42); + // LLSD value matching 'required' according to llsd_matches() rules. + LLSD matching(LLSDMap("d", 3.14)("array", LLSDArray("answer")(true)(answer))); + // Okay, walk through 'tests'. + foreach(const CallablesTriple& tr, tests) + { + // Should be able to pass 'answer' to Callables registered + // without 'required'. + work(tr.name, answer); + ensure_equals("answer mismatch", tr.llsd, answer); + // Should NOT be able to pass 'answer' to Callables registered + // with 'required'. + call_exc(tr.name_req, answer, "bad request"); + // But SHOULD be able to pass 'matching' to Callables registered + // with 'required'. + work(tr.name_req, matching); + ensure_equals("matching mismatch", tr.llsd, matching); + } + } + + template<> template<> + void object::test<17>() + { + set_test_name("passing wrong args to (map | array)-style registrations"); + + // Pass scalar/map to array-style functions, scalar/array to map-style + // functions. As that validation happens well before we engage the + // argument magic, it seems pointless to repeat this with every + // variation: (free function | non-static method), (no | arbitrary) + // args. We should only need to engage it for one map-style + // registration and one array-style registration. + std::string array_exc("needs an args array"); + call_exc("free0_array", 17, array_exc); + call_exc("free0_array", LLSDMap("pi", 3.14), array_exc); + + std::string map_exc("needs a map"); + call_exc("free0_map", 17, map_exc); + // Passing an array to a map-style function works now! No longer an + // error case! +// call_exc("free0_map", LLSDArray("a")("b"), map_exc); + } + + template<> template<> + void object::test<18>() + { + set_test_name("call no-args functions"); + LLSD names(LLSDArray + ("free0_array")("free0_map") + ("smethod0_array")("smethod0_map") + ("method0_array")("method0_map")); + foreach(LLSD name, inArray(names)) + { + // Look up the Vars instance for this function. + Vars* vars(varsfor(name)); + // Both the global and stack Vars instances are automatically + // cleared at the start of each test<n> method. But since we're + // calling these things several different times in the same + // test<n> method, manually reset the Vars between each. + *vars = Vars(); + ensure_equals(vars->i, 0); + // call function with empty array (or LLSD(), should be equivalent) + work(name, LLSD()); + ensure_equals(vars->i, 17); + } + } + + // Break out this data because we use it in a couple different tests. + LLSD array_funcs(LLSDArray + (LLSDMap("a", "freena_array") ("b", "freenb_array")) + (LLSDMap("a", "smethodna_array")("b", "smethodnb_array")) + (LLSDMap("a", "methodna_array") ("b", "methodnb_array"))); + + template<> template<> + void object::test<19>() + { + set_test_name("call array-style functions with too-short arrays"); + // Could have two different too-short arrays, one for *na and one for + // *nb, but since they both take 5 params... + LLSD tooshort(LLSDArray("this")("array")("too")("short")); + foreach(const LLSD& funcsab, inArray(array_funcs)) + { + foreach(const llsd::MapEntry& e, inMap(funcsab)) + { + call_exc(e.second, tooshort, "requires more arguments"); + } + } + } + + template<> template<> + void object::test<20>() + { + set_test_name("call array-style functions with (just right | too long) arrays"); + std::vector<U8> binary; + for (size_t h(0x01), i(0); i < 5; h+= 0x22, ++i) + { + binary.push_back(h); + } + LLSD args(LLSDMap("a", LLSDArray(true)(17)(3.14)(123.456)("char*")) + ("b", LLSDArray("string") + (LLUUID("01234567-89ab-cdef-0123-456789abcdef")) + (LLDate("2011-02-03T15:07:00Z")) + (LLURI("http://secondlife.com")) + (binary))); + LLSD argsplus(args); + argsplus["a"].append("bogus"); + argsplus["b"].append("bogus"); + LLSD expect; + foreach(LLSD::String a, ab) + { + expect[a] = zipmap(params[a], args[a]); + } + // Adjust expect["a"]["cp"] for special Vars::cp treatment. + expect["a"]["cp"] = std::string("'") + expect["a"]["cp"].asString() + "'"; + cout << "expect: " << expect << '\n'; + + // Use substantially the same logic for args and argsplus + LLSD argsarrays(LLSDArray(args)(argsplus)); + // So i==0 selects 'args', i==1 selects argsplus + for (LLSD::Integer i(0), iend(argsarrays.size()); i < iend; ++i) + { + foreach(const LLSD& funcsab, inArray(array_funcs)) + { + foreach(LLSD::String a, ab) + { + // Reset the Vars instance before each call + Vars* vars(varsfor(funcsab[a])); + *vars = Vars(); + work(funcsab[a], argsarrays[i][a]); + ensure_llsd(STRINGIZE(funcsab[a].asString() << + ": expect[\"" << a << "\"] mismatch"), + vars->inspect(), expect[a], 7); // 7 bits ~= 2 decimal digits + + // TODO: in the i==1 or argsplus case, intercept LL_WARNS + // output? Even without that, using argsplus verifies that + // passing too many args isn't fatal; it works -- but + // would be nice to notice the warning too. + } + } + } + } + + template<> template<> + void object::test<21>() + { + set_test_name("verify that passing LLSD() to const char* sends NULL"); + + ensure_equals("Vars::cp init", v.cp, ""); + work("methodna_map_mdft", LLSDMap("cp", LLSD())); + ensure_equals("passing LLSD()", v.cp, "NULL"); + work("methodna_map_mdft", LLSDMap("cp", "")); + ensure_equals("passing \"\"", v.cp, "''"); + work("methodna_map_mdft", LLSDMap("cp", "non-NULL")); + ensure_equals("passing \"non-NULL\"", v.cp, "'non-NULL'"); + } + + template<> template<> + void object::test<22>() + { + set_test_name("call map-style functions with (full | oversized) (arrays | maps)"); + const char binary[] = "\x99\x88\x77\x66\x55"; + LLSD array_full(LLSDMap + ("a", LLSDArray(false)(255)(98.6)(1024.5)("pointer")) + ("b", LLSDArray("object")(LLUUID::generateNewID())(LLDate::now())(LLURI("http://wiki.lindenlab.com/wiki"))(LLSD::Binary(boost::begin(binary), boost::end(binary))))); + LLSD array_overfull(array_full); + foreach(LLSD::String a, ab) + { + array_overfull[a].append("bogus"); + } + cout << "array_full: " << array_full << "\narray_overfull: " << array_overfull << std::endl; + // We rather hope that LLDate::now() will generate a timestamp + // distinct from the one it generated in the constructor, moments ago. + ensure_not_equals("Timestamps too close", + array_full["b"][2].asDate(), dft_array_full["b"][2].asDate()); + // We /insist/ that LLUUID::generateNewID() do so. + ensure_not_equals("UUID collision", + array_full["b"][1].asUUID(), dft_array_full["b"][1].asUUID()); + LLSD map_full, map_overfull; + foreach(LLSD::String a, ab) + { + map_full[a] = zipmap(params[a], array_full[a]); + map_overfull[a] = map_full[a]; + map_overfull[a]["extra"] = "ignore"; + } + cout << "map_full: " << map_full << "\nmap_overfull: " << map_overfull << std::endl; + LLSD expect(map_full); + // Twiddle the const char* param. + expect["a"]["cp"] = std::string("'") + expect["a"]["cp"].asString() + "'"; + // Another adjustment. For each data type, we're trying to distinguish + // three values: the Vars member's initial value (member wasn't + // stored; control never reached the set function), the registered + // default param value from dft_array_full, and the array_full value + // in this test. But bool can only distinguish two values. In this + // case, we want to differentiate the local array_full value from the + // dft_array_full value, so we use 'false'. However, that means + // Vars::inspect() doesn't differentiate it from the initial value, + // so won't bother returning it. Predict that behavior to match the + // LLSD values. + expect["a"].erase("b"); + cout << "expect: " << expect << std::endl; + // For this test, calling functions registered with different sets of + // parameter defaults should make NO DIFFERENCE WHATSOEVER. Every call + // should pass all params. + LLSD names(LLSDMap + ("a", LLSDArray + ("freena_map_allreq") ("smethodna_map_allreq") ("methodna_map_allreq") + ("freena_map_leftreq")("smethodna_map_leftreq")("methodna_map_leftreq") + ("freena_map_skipreq")("smethodna_map_skipreq")("methodna_map_skipreq") + ("freena_map_adft") ("smethodna_map_adft") ("methodna_map_adft") + ("freena_map_mdft") ("smethodna_map_mdft") ("methodna_map_mdft")) + ("b", LLSDArray + ("freenb_map_allreq") ("smethodnb_map_allreq") ("methodnb_map_allreq") + ("freenb_map_leftreq")("smethodnb_map_leftreq")("methodnb_map_leftreq") + ("freenb_map_skipreq")("smethodnb_map_skipreq")("methodnb_map_skipreq") + ("freenb_map_adft") ("smethodnb_map_adft") ("methodnb_map_adft") + ("freenb_map_mdft") ("smethodnb_map_mdft") ("methodnb_map_mdft"))); + // Treat (full | overfull) (array | map) the same. + LLSD argssets(LLSDArray(array_full)(array_overfull)(map_full)(map_overfull)); + foreach(const LLSD& args, inArray(argssets)) + { + foreach(LLSD::String a, ab) + { + foreach(LLSD::String name, inArray(names[a])) + { + // Reset the Vars instance + Vars* vars(varsfor(name)); + *vars = Vars(); + work(name, args[a]); + ensure_llsd(STRINGIZE(name << ": expect[\"" << a << "\"] mismatch"), + vars->inspect(), expect[a], 7); // 7 bits, 2 decimal digits + // intercept LL_WARNS for the two overfull cases? + } + } + } + } +} // namespace tut diff --git a/indra/llcommon/tests/llinstancetracker_test.cpp b/indra/llcommon/tests/llinstancetracker_test.cpp index c7cb488ca1..b34d1c5fd3 100644 --- a/indra/llcommon/tests/llinstancetracker_test.cpp +++ b/indra/llcommon/tests/llinstancetracker_test.cpp @@ -40,6 +40,7 @@ #include <boost/scoped_ptr.hpp> // other Linden headers #include "../test/lltut.h" +#include "wrapllerrs.h" struct Keyed: public LLInstanceTracker<Keyed, std::string> { @@ -151,33 +152,81 @@ namespace tut { Unkeyed one, two, three; typedef std::set<Unkeyed*> KeySet; - KeySet keys; - keys.insert(&one); - keys.insert(&two); - keys.insert(&three); - { - Unkeyed::LLInstanceTrackerScopedGuard guard; - for (Unkeyed::key_iter ki(guard.beginKeys()), kend(guard.endKeys()); - ki != kend; ++ki) - { - ensure_equals("spurious key", keys.erase(*ki), 1); - } - } - ensure_equals("unreported key", keys.size(), 0); - + KeySet instances; instances.insert(&one); instances.insert(&two); instances.insert(&three); - { - Unkeyed::LLInstanceTrackerScopedGuard guard; - for (Unkeyed::instance_iter ii(guard.beginInstances()), iend(guard.endInstances()); - ii != iend; ++ii) + + for (Unkeyed::instance_iter ii(Unkeyed::beginInstances()), iend(Unkeyed::endInstances()); ii != iend; ++ii) { Unkeyed& ref = *ii; ensure_equals("spurious instance", instances.erase(&ref), 1); } - } + ensure_equals("unreported instance", instances.size(), 0); } + + template<> template<> + void object::test<5>() + { + set_test_name("delete Keyed with outstanding instance_iter"); + std::string what; + Keyed* keyed = new Keyed("one"); + { + WrapLL_ERRS wrapper; + Keyed::instance_iter i(Keyed::beginInstances()); + try + { + delete keyed; + } + catch (const WrapLL_ERRS::FatalException& e) + { + what = e.what(); + } + } + ensure(! what.empty()); + } + + template<> template<> + void object::test<6>() + { + set_test_name("delete Keyed with outstanding key_iter"); + std::string what; + Keyed* keyed = new Keyed("one"); + { + WrapLL_ERRS wrapper; + Keyed::key_iter i(Keyed::beginKeys()); + try + { + delete keyed; + } + catch (const WrapLL_ERRS::FatalException& e) + { + what = e.what(); + } + } + ensure(! what.empty()); + } + + template<> template<> + void object::test<7>() + { + set_test_name("delete Unkeyed with outstanding instance_iter"); + std::string what; + Unkeyed* unkeyed = new Unkeyed; + { + WrapLL_ERRS wrapper; + Unkeyed::instance_iter i(Unkeyed::beginInstances()); + try + { + delete unkeyed; + } + catch (const WrapLL_ERRS::FatalException& e) + { + what = e.what(); + } + } + ensure(! what.empty()); + } } // namespace tut diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index 770443da1d..72322c3b72 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -25,35 +25,293 @@ * $/LicenseInfo$ */ -#if !LL_WINDOWS + +#include "linden_common.h" + +#if LL_WINDOWS +#include <winsock2.h> +typedef U32 uint32_t; +#include <process.h> +#include <io.h> +#else +#include <unistd.h> #include <netinet/in.h> +#include <errno.h> +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include "llprocesslauncher.h" #endif -#include "linden_common.h" +#include <sstream> + +/*==========================================================================*| +// Whoops, seems Linden's Boost package and the viewer are built with +// different settings of VC's /Zc:wchar_t switch! Using Boost.Filesystem +// pathname operations produces Windows link errors: +// unresolved external symbol "private: static class std::codecvt<unsigned short, +// char,int> const * & __cdecl boost::filesystem3::path::wchar_t_codecvt_facet()" +// unresolved external symbol "void __cdecl boost::filesystem3::path_traits::convert()" +// See: +// http://boost.2283326.n4.nabble.com/filesystem-v3-unicode-and-std-codecvt-linker-error-td3455549.html +// which points to: +// http://msdn.microsoft.com/en-us/library/dh8che7s%28v=VS.100%29.aspx + +// As we're not trying to preserve compatibility with old Boost.Filesystem +// code, but rather writing brand-new code, use the newest available +// Filesystem API. +#define BOOST_FILESYSTEM_VERSION 3 +#include "boost/filesystem.hpp" +#include "boost/filesystem/v3/fstream.hpp" +|*==========================================================================*/ +#include "boost/range.hpp" +#include "boost/foreach.hpp" +#include "boost/function.hpp" +#include "boost/lambda/lambda.hpp" +#include "boost/lambda/bind.hpp" +namespace lambda = boost::lambda; +/*==========================================================================*| +// Aaaarrgh, Linden's Boost package doesn't even include Boost.Iostreams! +#include "boost/iostreams/stream.hpp" +#include "boost/iostreams/device/file_descriptor.hpp" +|*==========================================================================*/ + #include "../llsd.h" #include "../llsdserialize.h" +#include "llsdutil.h" #include "../llformat.h" #include "../test/lltut.h" +#include "stringize.h" +std::vector<U8> string_to_vector(const std::string& str) +{ + return std::vector<U8>(str.begin(), str.end()); +} -#if LL_WINDOWS -#include <winsock2.h> -typedef U32 uint32_t; -#endif +#if ! LL_WINDOWS +// We want to call strerror_r(), but alarmingly, there are two different +// variants. The one that returns int always populates the passed buffer +// (except in case of error), whereas the other one always returns a valid +// char* but might or might not populate the passed buffer. How do we know +// which one we're getting? Define adapters for each and let the compiler +// select the applicable adapter. -std::vector<U8> string_to_vector(std::string str) +// strerror_r() returns char* +std::string message_from(int /*orig_errno*/, const char* /*buffer*/, const char* strerror_ret) { - // bc LLSD can't... - size_t len = (size_t)str.length(); - std::vector<U8> v(len); - for (size_t i = 0; i < len ; i++) - { - v[i] = str[i]; - } - return v; + return strerror_ret; } +// strerror_r() returns int +std::string message_from(int orig_errno, const char* buffer, int strerror_ret) +{ + if (strerror_ret == 0) + { + return buffer; + } + // Here strerror_r() has set errno. Since strerror_r() has already failed, + // seems like a poor bet to call it again to diagnose its own error... + int stre_errno = errno; + if (stre_errno == ERANGE) + { + return STRINGIZE("strerror_r() can't explain errno " << orig_errno + << " (buffer too small)"); + } + if (stre_errno == EINVAL) + { + return STRINGIZE("unknown errno " << orig_errno); + } + // Here we don't even understand the errno from strerror_r()! + return STRINGIZE("strerror_r() can't explain errno " << orig_errno + << " (error " << stre_errno << ')'); +} +#endif // ! LL_WINDOWS + +// boost::filesystem::temp_directory_path() isn't yet in Boost 1.45! :-( +std::string temp_directory_path() +{ +#if LL_WINDOWS + char buffer[4096]; + GetTempPathA(sizeof(buffer), buffer); + return buffer; + +#else // LL_DARWIN, LL_LINUX + static const char* vars[] = { "TMPDIR", "TMP", "TEMP", "TEMPDIR" }; + BOOST_FOREACH(const char* var, vars) + { + const char* found = getenv(var); + if (found) + return found; + } + return "/tmp"; +#endif // LL_DARWIN, LL_LINUX +} + +// Windows presents a kinda sorta compatibility layer. Code to the yucky +// Windows names because they're less likely than the Posix names to collide +// with any other names in this source. +#if LL_WINDOWS +#define _remove DeleteFileA +#else // ! LL_WINDOWS +#define _open open +#define _write write +#define _close close +#define _remove remove +#endif // ! LL_WINDOWS + +// Create a text file with specified content "somewhere in the +// filesystem," cleaning up when it goes out of scope. +class NamedTempFile +{ +public: + // Function that accepts an ostream ref and (presumably) writes stuff to + // it, e.g.: + // (lambda::_1 << "the value is " << 17 << '\n') + typedef boost::function<void(std::ostream&)> Streamer; + + NamedTempFile(const std::string& ext, const std::string& content): + mPath(temp_directory_path()) + { + createFile(ext, lambda::_1 << content); + } + + // Disambiguate when passing string literal + NamedTempFile(const std::string& ext, const char* content): + mPath(temp_directory_path()) + { + createFile(ext, lambda::_1 << content); + } + + NamedTempFile(const std::string& ext, const Streamer& func): + mPath(temp_directory_path()) + { + createFile(ext, func); + } + + ~NamedTempFile() + { + _remove(mPath.c_str()); + } + + std::string getName() const { return mPath; } + +private: + void createFile(const std::string& ext, const Streamer& func) + { + // Silly maybe, but use 'ext' as the name prefix. Strip off a leading + // '.' if present. + int pfx_offset = ((! ext.empty()) && ext[0] == '.')? 1 : 0; + +#if ! LL_WINDOWS + // Make sure mPath ends with a directory separator, if it doesn't already. + if (mPath.empty() || + ! (mPath[mPath.length() - 1] == '\\' || mPath[mPath.length() - 1] == '/')) + { + mPath.append("/"); + } + + // mkstemp() accepts and modifies a char* template string. Generate + // the template string, then copy to modifiable storage. + // mkstemp() requires its template string to end in six X's. + mPath += ext.substr(pfx_offset) + "XXXXXX"; + // Copy to vector<char> + std::vector<char> pathtemplate(mPath.begin(), mPath.end()); + // append a nul byte for classic-C semantics + pathtemplate.push_back('\0'); + // std::vector promises that a pointer to the 0th element is the same + // as a pointer to a contiguous classic-C array + int fd(mkstemp(&pathtemplate[0])); + if (fd == -1) + { + // The documented errno values (http://linux.die.net/man/3/mkstemp) + // are used in a somewhat unusual way, so provide context-specific + // errors. + if (errno == EEXIST) + { + LL_ERRS("NamedTempFile") << "mkstemp(\"" << mPath + << "\") could not create unique file " << LL_ENDL; + } + if (errno == EINVAL) + { + LL_ERRS("NamedTempFile") << "bad mkstemp() file path template '" + << mPath << "'" << LL_ENDL; + } + // Shrug, something else + int mkst_errno = errno; + char buffer[256]; + LL_ERRS("NamedTempFile") << "mkstemp(\"" << mPath << "\") failed: " + << message_from(mkst_errno, buffer, + strerror_r(mkst_errno, buffer, sizeof(buffer))) + << LL_ENDL; + } + // mkstemp() seems to have worked! Capture the modified filename. + // Avoid the nul byte we appended. + mPath.assign(pathtemplate.begin(), (pathtemplate.end()-1)); + +/*==========================================================================*| + // Define an ostream on the open fd. Tell it to close fd on destruction. + boost::iostreams::stream<boost::iostreams::file_descriptor_sink> + out(fd, boost::iostreams::close_handle); +|*==========================================================================*/ + + // Write desired content. + std::ostringstream out; + // Stream stuff to it. + func(out); + + std::string data(out.str()); + int written(_write(fd, data.c_str(), data.length())); + int closed(_close(fd)); + llassert_always(written == data.length() && closed == 0); + +#else // LL_WINDOWS + // GetTempFileName() is documented to require a MAX_PATH buffer. + char tempname[MAX_PATH]; + // Use 'ext' as filename prefix, but skip leading '.' if any. + // The 0 param is very important: requests iterating until we get a + // unique name. + if (0 == GetTempFileNameA(mPath.c_str(), ext.c_str() + pfx_offset, 0, tempname)) + { + // I always have to look up this call... :-P + LPSTR msgptr; + FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + GetLastError(), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + LPSTR(&msgptr), // have to cast (char**) to (char*) + 0, NULL ); + LL_ERRS("NamedTempFile") << "GetTempFileName(\"" << mPath << "\", \"" + << (ext.c_str() + pfx_offset) << "\") failed: " + << msgptr << LL_ENDL; + LocalFree(msgptr); + } + // GetTempFileName() appears to have worked! Capture the actual + // filename. + mPath = tempname; + // Open the file and stream content to it. Destructor will close. + std::ofstream out(tempname); + func(out); + +#endif // LL_WINDOWS + } + + void peep() + { + std::cout << "File '" << mPath << "' contains:\n"; + std::ifstream reader(mPath.c_str()); + std::string line; + while (std::getline(reader, line)) + std::cout << line << '\n'; + std::cout << "---\n"; + } + + std::string mPath; +}; + namespace tut { struct sd_xml_data @@ -452,7 +710,7 @@ namespace tut checkRoundTrip(msg + " nested arrays", v); v = LLSD::emptyMap(); - fillmap(v, 10, 6); // 10^6 maps + fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } @@ -1494,5 +1752,223 @@ namespace tut ensureBinaryAndNotation("map", test); ensureBinaryAndXML("map", test); } -} + struct TestPythonCompatible + { + TestPythonCompatible(): + // Note the peculiar insertion of __FILE__ into this string. Since + // this script is being written into a platform-dependent temp + // directory, we can't locate indra/lib/python relative to + // Python's __file__. Use __FILE__ instead, navigating relative + // to this C++ source file. Use Python raw-string syntax so + // Windows pathname backslashes won't mislead Python's string + // scanner. + import_llsd("import os.path\n" + "import sys\n" + "sys.path.insert(0,\n" + " os.path.join(os.path.dirname(r'" __FILE__ "'),\n" + " os.pardir, os.pardir, 'lib', 'python'))\n" + "try:\n" + " from llbase import llsd\n" + "except ImportError:\n" + " from indra.base import llsd\n") + {} + ~TestPythonCompatible() {} + + std::string import_llsd; + + template <typename CONTENT> + void python(const std::string& desc, const CONTENT& script, int expect=0) + { + const char* PYTHON(getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", PYTHON); + + NamedTempFile scriptfile(".py", script); + +#if LL_WINDOWS + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON, qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) + { + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } + +#else // LL_DARWIN, LL_LINUX + LLProcessLauncher py; + py.setExecutable(PYTHON); + py.addArgument(scriptfile.getName()); + ensure_equals(STRINGIZE("Couldn't launch " << desc << " script"), py.launch(), 0); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py.getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) + { + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); + } + else if (WIFSIGNALED(status)) + { + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); + } + else + { + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); + } + } +#endif + } + }; + + typedef tut::test_group<TestPythonCompatible> TestPythonCompatibleGroup; + typedef TestPythonCompatibleGroup::object TestPythonCompatibleObject; + TestPythonCompatibleGroup pycompat("LLSD serialize Python compatibility"); + + template<> template<> + void TestPythonCompatibleObject::test<1>() + { + set_test_name("verify python()"); + python("hello", + "import sys\n" + "sys.exit(17)\n", + 17); // expect nonzero rc + } + + template<> template<> + void TestPythonCompatibleObject::test<2>() + { + set_test_name("verify NamedTempFile"); + python("platform", + "import sys\n" + "print 'Running on', sys.platform\n"); + } + + template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("verify sequence to Python"); + + LLSD cdata(LLSDArray(17)(3.14) + ("This string\n" + "has several\n" + "lines.")); + + const char pydata[] = + "def verify(iterable):\n" + " it = iter(iterable)\n" + " assert it.next() == 17\n" + " assert abs(it.next() - 3.14) < 0.01\n" + " assert it.next() == '''\\\n" + "This string\n" + "has several\n" + "lines.'''\n" + " try:\n" + " it.next()\n" + " except StopIteration:\n" + " pass\n" + " else:\n" + " assert False, 'Too many data items'\n"; + + // Create a something.llsd file containing 'data' serialized to + // notation. It's important to separate with newlines because Python's + // llsd module doesn't support parsing from a file stream, only from a + // string, so we have to know how much of the file to read into a + // string. + NamedTempFile file(".llsd", + // NamedTempFile's boost::function constructor + // takes a callable. To this callable it passes the + // std::ostream with which it's writing the + // NamedTempFile. This lambda-based expression + // first calls LLSD::Serialize() with that ostream, + // then streams a newline to it, etc. + (lambda::bind(LLSDSerialize::toNotation, cdata[0], lambda::_1), + lambda::_1 << '\n', + lambda::bind(LLSDSerialize::toNotation, cdata[1], lambda::_1), + lambda::_1 << '\n', + lambda::bind(LLSDSerialize::toNotation, cdata[2], lambda::_1), + lambda::_1 << '\n')); + + python("read C++ notation", + lambda::_1 << + import_llsd << + "def parse_each(iterable):\n" + " for item in iterable:\n" + " yield llsd.parse(item)\n" << + pydata << + // Don't forget raw-string syntax for Windows pathnames. + "verify(parse_each(open(r'" << file.getName() << "')))\n"); + } + + template<> template<> + void TestPythonCompatibleObject::test<4>() + { + set_test_name("verify sequence from Python"); + + // Create an empty data file. This is just a placeholder for our + // script to write into. Create it to establish a unique name that + // we know. + NamedTempFile file(".llsd", ""); + + python("write Python notation", + lambda::_1 << + "from __future__ import with_statement\n" << + import_llsd << + "DATA = [\n" + " 17,\n" + " 3.14,\n" + " '''\\\n" + "This string\n" + "has several\n" + "lines.''',\n" + "]\n" + // Don't forget raw-string syntax for Windows pathnames. + // N.B. Using 'print' implicitly adds newlines. + "with open(r'" << file.getName() << "', 'w') as f:\n" + " for item in DATA:\n" + " print >>f, llsd.format_notation(item)\n"); + + std::ifstream inf(file.getName().c_str()); + LLSD item; + // Notice that we're not doing anything special to parse out the + // newlines: LLSDSerialize::fromNotation ignores them. While it would + // seem they're not strictly necessary, going in this direction, we + // want to ensure that notation-separated-by-newlines works in both + // directions -- since in practice, a given file might be read by + // either language. + ensure_equals("Failed to read LLSD::Integer from Python", + LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), + 1); + ensure_equals(item.asInteger(), 17); + ensure_equals("Failed to read LLSD::Real from Python", + LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), + 1); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure_equals("Failed to read LLSD::String from Python", + LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), + 1); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } +} diff --git a/indra/llcommon/tests/llsingleton_test.cpp b/indra/llcommon/tests/llsingleton_test.cpp new file mode 100644 index 0000000000..385289aefe --- /dev/null +++ b/indra/llcommon/tests/llsingleton_test.cpp @@ -0,0 +1,76 @@ +/** + * @file llsingleton_test.cpp + * @date 2011-08-11 + * @brief Unit test for the LLSingleton class + * + * $LicenseInfo:firstyear=2011&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2011, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llsingleton.h" +#include "../test/lltut.h" + +namespace tut +{ + struct singleton + { + // We need a class created with the LLSingleton template to test with. + class LLSingletonTest: public LLSingleton<LLSingletonTest> + { + + }; + }; + + typedef test_group<singleton> singleton_t; + typedef singleton_t::object singleton_object_t; + tut::singleton_t tut_singleton("LLSingleton"); + + template<> template<> + void singleton_object_t::test<1>() + { + + } + template<> template<> + void singleton_object_t::test<2>() + { + LLSingletonTest* singleton_test = LLSingletonTest::getInstance(); + ensure(singleton_test); + } + template<> template<> + void singleton_object_t::test<3>() + { + //Construct the instance + LLSingletonTest::getInstance(); + ensure(LLSingletonTest::instanceExists()); + + //Delete the instance + LLSingletonTest::deleteSingleton(); + ensure(LLSingletonTest::destroyed()); + ensure(!LLSingletonTest::instanceExists()); + + //Construct it again. + LLSingletonTest* singleton_test = LLSingletonTest::getInstance(); + ensure(singleton_test); + ensure(LLSingletonTest::instanceExists()); + } +} diff --git a/indra/llcommon/tests/llstring_test.cpp b/indra/llcommon/tests/llstring_test.cpp index 304e91ed92..6a1cbf652a 100644 --- a/indra/llcommon/tests/llstring_test.cpp +++ b/indra/llcommon/tests/llstring_test.cpp @@ -624,6 +624,14 @@ namespace tut subcount = LLStringUtil::format(s, fmt_map); ensure_equals("LLStringUtil::format: Assorted Test2 result", s, "?Am I not a long string?short[A]bbbaaaba[A]"); ensure_equals("LLStringUtil::format: Assorted Test2 result count", 9, subcount); + + // Test on nested brackets + std::string srcs6 = "[[TRICK1]][[A]][[B]][[AAA]][[BBB]][[TRICK2]][[KEYLONGER]][[KEYSHORTER]]?[[DELETE]]"; + s = srcs6; + subcount = LLStringUtil::format(s, fmt_map); + ensure_equals("LLStringUtil::format: Assorted Test2 result", s, "[[A]][a][b][aaa][bbb][[A]][short][Am I not a long string?]?[]"); + ensure_equals("LLStringUtil::format: Assorted Test2 result count", 9, subcount); + // Test an assorted substitution std::string srcs8 = "foo[DELETE]bar?"; diff --git a/indra/llcommon/tests/setpython.py b/indra/llcommon/tests/setpython.py new file mode 100644 index 0000000000..df7b90428e --- /dev/null +++ b/indra/llcommon/tests/setpython.py @@ -0,0 +1,19 @@ +#!/usr/bin/python +"""\ +@file setpython.py +@author Nat Goodspeed +@date 2011-07-13 +@brief Set PYTHON environment variable for tests that care. + +$LicenseInfo:firstyear=2011&license=viewerlgpl$ +Copyright (c) 2011, Linden Research, Inc. +$/LicenseInfo$ +""" + +import os +import sys +import subprocess + +if __name__ == "__main__": + os.environ["PYTHON"] = sys.executable + sys.exit(subprocess.call(sys.argv[1:])) |