diff options
Diffstat (limited to 'indra/llcommon')
41 files changed, 1783 insertions, 905 deletions
diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 3d073b009c..54020a4231 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -105,6 +105,7 @@ set(llcommon_SOURCE_FILES lluriparser.cpp lluuid.cpp llworkerthread.cpp + hbxxh.cpp u64.cpp threadpool.cpp workqueue.cpp @@ -241,6 +242,7 @@ set(llcommon_HEADER_FILES llwin32headers.h llwin32headerslean.h llworkerthread.h + hbxxh.h lockstatic.h stdtypes.h stringize.h diff --git a/indra/llcommon/hbxxh.cpp b/indra/llcommon/hbxxh.cpp new file mode 100644 index 0000000000..388269d6c8 --- /dev/null +++ b/indra/llcommon/hbxxh.cpp @@ -0,0 +1,377 @@ +/** + * @file hbxxh.cpp + * @brief High performances vectorized hashing based on xxHash. + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (c) 2023, Henri Beauchamp. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +// This define ensures that xxHash will be compiled within this module, with +// vectorized (*) and inlined functions (with no exported API symbol); our +// xxhash "pre-built library" package actually only contains the xxhash.h +// header (no library needed at link time). +// (*) SSE2 is normally used for x86(_64) builds, unless you enabled AVX2 +// in your build, in which case the latter would be used instead. For ARM64 +// builds, this would also automatically enable NEON vectorization. +#define XXH_INLINE_ALL +#include "xxhash/xxhash.h" + +#include "hbxxh.h" + +// How many bytes to grab at a time when hashing files or streams +constexpr size_t BLOCK_LEN = 4096; + +/////////////////////////////////////////////////////////////////////////////// +// HBXXH64 class +/////////////////////////////////////////////////////////////////////////////// + +//static +U64 HBXXH64::digest(const void* buffer, size_t len) +{ + return XXH3_64bits(buffer, len); +} + +//static +U64 HBXXH64::digest(const char* str) +{ + return XXH3_64bits((const void*)str, strlen(str)); +} + +//static +U64 HBXXH64::digest(const std::string& str) +{ + return XXH3_64bits((const void*)str.c_str(), str.size()); +} + +// Must be called by all constructors. +void HBXXH64::init() +{ + mDigest = 0; + mState = (void*)XXH3_createState(); + if (!mState || XXH3_64bits_reset((XXH3_state_t*)mState) != XXH_OK) + { + LL_WARNS() << "Failed to initialize state !" << LL_ENDL; + } +} + +HBXXH64::~HBXXH64() +{ + if (mState) + { + XXH3_freeState((XXH3_state_t*)mState); + } +} + +void HBXXH64::update(const void* buffer, size_t len) +{ + if (mState) + { + XXH3_64bits_update((XXH3_state_t*)mState, buffer, len); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH64::update(const std::string& str) +{ + if (mState) + { + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)str.c_str(), + str.length()); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH64::update(std::istream& stream) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while (stream.good()) + { + stream.read(buffer, BLOCK_LEN); + len = stream.gcount(); + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } +} + +void HBXXH64::update(FILE* file) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while ((len = fread((void*)buffer, 1, BLOCK_LEN, file))) + { + XXH3_64bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } + fclose(file); +} + +void HBXXH64::finalize() +{ + if (!mState) + { + LL_WARNS() << "Already finalized !" << LL_ENDL; + return; + } + mDigest = XXH3_64bits_digest((XXH3_state_t*)mState); + XXH3_freeState((XXH3_state_t*)mState); + mState = NULL; +} + +U64 HBXXH64::digest() const +{ + return mState ? XXH3_64bits_digest((XXH3_state_t*)mState) : mDigest; +} + +std::ostream& operator<<(std::ostream& stream, HBXXH64 context) +{ + stream << context.digest(); + return stream; +} + +/////////////////////////////////////////////////////////////////////////////// +// HBXXH128 class +/////////////////////////////////////////////////////////////////////////////// + +//static +LLUUID HBXXH128::digest(const void* buffer, size_t len) +{ + XXH128_hash_t hash = XXH3_128bits(buffer, len); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +LLUUID HBXXH128::digest(const char* str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str, strlen(str)); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +LLUUID HBXXH128::digest(const std::string& str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str.c_str(), str.size()); + LLUUID id; + U64* data = (U64*)id.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + return id; +} + +//static +void HBXXH128::digest(LLUUID& result, const void* buffer, size_t len) +{ + XXH128_hash_t hash = XXH3_128bits(buffer, len); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +//static +void HBXXH128::digest(LLUUID& result, const char* str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str, strlen(str)); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +//static +void HBXXH128::digest(LLUUID& result, const std::string& str) +{ + XXH128_hash_t hash = XXH3_128bits((const void*)str.c_str(), str.size()); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +// Must be called by all constructors. +void HBXXH128::init() +{ + mState = (void*)XXH3_createState(); + if (!mState || XXH3_128bits_reset((XXH3_state_t*)mState) != XXH_OK) + { + LL_WARNS() << "Failed to initialize state !" << LL_ENDL; + } +} + +HBXXH128::~HBXXH128() +{ + if (mState) + { + XXH3_freeState((XXH3_state_t*)mState); + } +} + +void HBXXH128::update(const void* buffer, size_t len) +{ + if (mState) + { + XXH3_128bits_update((XXH3_state_t*)mState, buffer, len); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH128::update(const std::string& str) +{ + if (mState) + { + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)str.c_str(), + str.length()); + } + else + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + } +} + +void HBXXH128::update(std::istream& stream) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while (stream.good()) + { + stream.read(buffer, BLOCK_LEN); + len = stream.gcount(); + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } +} + +void HBXXH128::update(FILE* file) +{ + if (!mState) + { + LL_WARNS() << "Cannot update a finalized digest !" << LL_ENDL; + return; + } + + char buffer[BLOCK_LEN]; + size_t len; + while ((len = fread((void*)buffer, 1, BLOCK_LEN, file))) + { + XXH3_128bits_update((XXH3_state_t*)mState, (const void*)buffer, len); + } + fclose(file); +} + +void HBXXH128::finalize() +{ + if (!mState) + { + LL_WARNS() << "Already finalized !" << LL_ENDL; + return; + } + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + U64* data = (U64*)mDigest.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + XXH3_freeState((XXH3_state_t*)mState); + mState = NULL; +} + +const LLUUID& HBXXH128::digest() const +{ + if (mState) + { + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + // We cheat the const-ness of the method here, but this is OK, since + // mDigest is private and cannot be accessed indirectly by other + // methods than digest() ones, that do check for mState to decide + // wether mDigest's current value may be provided as is or not. This + // cheat saves us a temporary LLLUID copy. + U64* data = (U64*)mDigest.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; + } + return mDigest; +} + +void HBXXH128::digest(LLUUID& result) const +{ + if (!mState) + { + result = mDigest; + return; + } + XXH128_hash_t hash = XXH3_128bits_digest((XXH3_state_t*)mState); + U64* data = (U64*)result.mData; + // Note: we do not check endianness here and we just store in the same + // order as XXH128_hash_t, that is low word "first". + data[0] = hash.low64; + data[1] = hash.high64; +} + +std::ostream& operator<<(std::ostream& stream, HBXXH128 context) +{ + stream << context.digest(); + return stream; +} diff --git a/indra/llcommon/hbxxh.h b/indra/llcommon/hbxxh.h new file mode 100644 index 0000000000..9c0e9cf172 --- /dev/null +++ b/indra/llcommon/hbxxh.h @@ -0,0 +1,277 @@ +/** + * @file hbxxh.h + * @brief High performances vectorized hashing based on xxHash. + * + * $LicenseInfo:firstyear=2023&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (c) 2023, Henri Beauchamp. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_HBXXH_H +#define LL_HBXXH_H + +#include "lluuid.h" + +// HBXXH* classes are to be used where speed matters and cryptographic quality +// is not required (no "one-way" guarantee, though they are likely not worst in +// this respect than MD5 which got busted and is now considered too weak). The +// xxHash code they are built upon is vectorized and about 50 times faster than +// MD5. A 64 bits hash class is also provided for when 128 bits of entropy are +// not needed. The hashes collision rate is similar to MD5's. +// See https://github.com/Cyan4973/xxHash#readme for details. + +// 64 bits hashing class + +class HBXXH64 +{ + friend std::ostream& operator<<(std::ostream&, HBXXH64); + +protected: + LOG_CLASS(HBXXH64); + +public: + inline HBXXH64() { init(); } + + // Constructors for special circumstances; they all digest the first passed + // parameter. Set 'do_finalize' to false if you do not want to finalize the + // context, which is useful/needed when you want to update() it afterwards. + // Ideally, the compiler should be smart enough to get our clue and + // optimize out the const bool test during inlining... + + inline HBXXH64(const void* buffer, size_t len, + const bool do_finalize = true) + { + init(); + update(buffer, len); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(const std::string& str, const bool do_finalize = true) + { + init(); + update(str); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(std::istream& s, const bool do_finalize = true) + { + init(); + update(s); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH64(FILE* file, const bool do_finalize = true) + { + init(); + update(file); + if (do_finalize) + { + finalize(); + } + } + + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH64(const HBXXH64&) noexcept = delete; + HBXXH64& operator=(const HBXXH64&) noexcept = delete; + + ~HBXXH64(); + + void update(const void* buffer, size_t len); + void update(const std::string& str); + void update(std::istream& s); + void update(FILE* file); + + // Note that unlike what happens with LLMD5, you do not need to finalize() + // HBXXH64 before using digest(), and you may keep updating() it even after + // you got a first digest() (the next digest would of course change after + // any update). It is still useful to use finalize() when you do not want + // to store a final digest() result in a separate U64; after this method + // has been called, digest() simply returns mDigest value. + void finalize(); + + U64 digest() const; + + // Fast static methods. Use them when hashing just one contiguous block of + // data. + static U64 digest(const void* buffer, size_t len); + static U64 digest(const char* str); // str must be NUL-terminated + static U64 digest(const std::string& str); + +private: + void init(); + +private: + // We use a void pointer to avoid including xxhash.h here for XXH3_state_t + // (which cannot either be trivially forward-declared, due to complex API + // related pre-processor macros in xxhash.h). + void* mState; + U64 mDigest; +}; + +inline bool operator==(const HBXXH64& a, const HBXXH64& b) +{ + return a.digest() == b.digest(); +} + +inline bool operator!=(const HBXXH64& a, const HBXXH64& b) +{ + return a.digest() != b.digest(); +} + +// 128 bits hashing class + +class HBXXH128 +{ + friend std::ostream& operator<<(std::ostream&, HBXXH128); + +protected: + LOG_CLASS(HBXXH128); + +public: + inline HBXXH128() { init(); } + + // Constructors for special circumstances; they all digest the first passed + // parameter. Set 'do_finalize' to false if you do not want to finalize the + // context, which is useful/needed when you want to update() it afterwards. + // Ideally, the compiler should be smart enough to get our clue and + // optimize out the const bool test during inlining... + + inline HBXXH128(const void* buffer, size_t len, + const bool do_finalize = true) + { + init(); + update(buffer, len); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(const std::string& str, const bool do_finalize = true) + { + init(); + update(str); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(std::istream& s, const bool do_finalize = true) + { + init(); + update(s); + if (do_finalize) + { + finalize(); + } + } + + inline HBXXH128(FILE* file, const bool do_finalize = true) + { + init(); + update(file); + if (do_finalize) + { + finalize(); + } + } + + // Make this class no-copy (it would be possible, with custom copy + // operators, but it is not trivially copyable, because of the mState + // pointer): it does not really make sense to allow copying it anyway, + // since all we care about is the resulting digest (so you should only + // need and care about storing/copying the digest and not a class + // instance). + HBXXH128(const HBXXH128&) noexcept = delete; + HBXXH128& operator=(const HBXXH128&) noexcept = delete; + + ~HBXXH128(); + + void update(const void* buffer, size_t len); + void update(const std::string& str); + void update(std::istream& s); + void update(FILE* file); + + // Note that unlike what happens with LLMD5, you do not need to finalize() + // HBXXH128 before using digest(), and you may keep updating() it even + // after you got a first digest() (the next digest would of course change + // after any update). It is still useful to use finalize() when you do not + // want to store a final digest() result in a separate LLUUID; after this + // method has been called, digest() simply returns a reference on mDigest. + void finalize(); + + // We use an LLUUID for the digest, since this is a 128 bits wide native + // type available in the viewer code, making it easy to manipulate. It also + // allows to use HBXXH128 efficiently in LLUUID generate() and combine() + // methods. + const LLUUID& digest() const; + + // Here, we avoid an LLUUID copy whenever we already got one to store the + // result *and* we did not yet call finalize(). + void digest(LLUUID& result) const; + + // Fast static methods. Use them when hashing just one contiguous block of + // data. + static LLUUID digest(const void* buffer, size_t len); + static LLUUID digest(const char* str); // str must be NUL-terminated + static LLUUID digest(const std::string& str); + // Same as above, but saves you from an LLUUID copy when you already got + // one for storage use. + static void digest(LLUUID& result, const void* buffer, size_t len); + static void digest(LLUUID& result, const char* str); // str NUL-terminated + static void digest(LLUUID& result, const std::string& str); + +private: + void init(); + +private: + // We use a void pointer to avoid including xxhash.h here for XXH3_state_t + // (which cannot either be trivially forward-declared, due to complex API + // related pre-processor macros in xxhash.h). + void* mState; + LLUUID mDigest; +}; + +inline bool operator==(const HBXXH128& a, const HBXXH128& b) +{ + return a.digest() == b.digest(); +} + +inline bool operator!=(const HBXXH128& a, const HBXXH128& b) +{ + return a.digest() != b.digest(); +} + +#endif // LL_HBXXH_H diff --git a/indra/llcommon/llallocator_heap_profile.cpp b/indra/llcommon/llallocator_heap_profile.cpp index b2eafde1aa..c6d9542b42 100644 --- a/indra/llcommon/llallocator_heap_profile.cpp +++ b/indra/llcommon/llallocator_heap_profile.cpp @@ -130,15 +130,13 @@ void LLAllocatorHeapProfile::parse(std::string const & prof_text) void LLAllocatorHeapProfile::dump(std::ostream & out) const { - lines_t::const_iterator i; - for(i = mLines.begin(); i != mLines.end(); ++i) + for (const LLAllocatorHeapProfile::line& line : mLines) { - out << i->mLiveCount << ": " << i->mLiveSize << '[' << i->mTotalCount << ": " << i->mTotalSize << "] @"; + out << line.mLiveCount << ": " << line.mLiveSize << '[' << line.mTotalCount << ": " << line.mTotalSize << "] @"; - stack_trace::const_iterator j; - for(j = i->mTrace.begin(); j != i->mTrace.end(); ++j) + for (const stack_marker marker : line.mTrace) { - out << ' ' << *j; + out << ' ' << marker; } out << '\n'; } diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index e6cc06e8d0..4c84223dad 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -150,14 +150,12 @@ LLAssetType::EType LLAssetType::lookup(const char* name) LLAssetType::EType LLAssetType::lookup(const std::string& type_name) { const LLAssetDictionary *dict = LLAssetDictionary::getInstance(); - for (LLAssetDictionary::const_iterator iter = dict->begin(); - iter != dict->end(); - iter++) + for (const LLAssetDictionary::value_type& pair : *dict) { - const AssetEntry *entry = iter->second; + const AssetEntry *entry = pair.second; if (type_name == entry->mTypeName) { - return iter->first; + return pair.first; } } return AT_UNKNOWN; @@ -188,14 +186,12 @@ LLAssetType::EType LLAssetType::lookupHumanReadable(const char* name) LLAssetType::EType LLAssetType::lookupHumanReadable(const std::string& readable_name) { const LLAssetDictionary *dict = LLAssetDictionary::getInstance(); - for (LLAssetDictionary::const_iterator iter = dict->begin(); - iter != dict->end(); - iter++) + for (const LLAssetDictionary::value_type& pair : *dict) { - const AssetEntry *entry = iter->second; + const AssetEntry *entry = pair.second; if (entry->mHumanName && (readable_name == entry->mHumanName)) { - return iter->first; + return pair.first; } } return AT_NONE; diff --git a/indra/llcommon/llcallbacklist.cpp b/indra/llcommon/llcallbacklist.cpp index 541ff75ee4..93d0a035da 100644 --- a/indra/llcommon/llcallbacklist.cpp +++ b/indra/llcommon/llcallbacklist.cpp @@ -109,7 +109,7 @@ void LLCallbackList::deleteAllFunctions() void LLCallbackList::callFunctions() { - for (callback_list_t::iterator iter = mCallbackList.begin(); iter != mCallbackList.end(); ) + for (callback_list_t::iterator iter = mCallbackList.begin(); iter != mCallbackList.end(); ) { callback_list_t::iterator curiter = iter++; curiter->first(curiter->second); diff --git a/indra/llcommon/llcallstack.cpp b/indra/llcommon/llcallstack.cpp index 8db291eed1..83d5ae2a63 100644 --- a/indra/llcommon/llcallstack.cpp +++ b/indra/llcommon/llcallstack.cpp @@ -91,10 +91,9 @@ LLCallStack::LLCallStack(S32 skip_count, bool verbose): bool LLCallStack::contains(const std::string& str) { - for (std::vector<std::string>::const_iterator it = m_strings.begin(); - it != m_strings.end(); ++it) + for (const std::string& src_str : m_strings) { - if (it->find(str) != std::string::npos) + if (src_str.find(str) != std::string::npos) { return true; } @@ -105,10 +104,9 @@ bool LLCallStack::contains(const std::string& str) std::ostream& operator<<(std::ostream& s, const LLCallStack& call_stack) { #ifndef LL_RELEASE_FOR_DOWNLOAD - std::vector<std::string>::const_iterator it; - for (it=call_stack.m_strings.begin(); it!=call_stack.m_strings.end(); ++it) + for (const std::string& str : call_stack.m_strings) { - s << *it; + s << str; } #else s << "UNAVAILABLE IN RELEASE"; @@ -156,9 +154,9 @@ bool LLContextStrings::contains(const std::string& str) { const std::map<std::string,S32>& strings = LLThreadLocalSingletonPointer<LLContextStrings>::getInstance()->m_contextStrings; - for (std::map<std::string,S32>::const_iterator it = strings.begin(); it!=strings.end(); ++it) + for (const std::map<std::string,S32>::value_type& str_pair : strings) { - if (it->first.find(str) != std::string::npos) + if (str_pair.first.find(str) != std::string::npos) { return true; } @@ -171,9 +169,9 @@ void LLContextStrings::output(std::ostream& os) { const std::map<std::string,S32>& strings = LLThreadLocalSingletonPointer<LLContextStrings>::getInstance()->m_contextStrings; - for (std::map<std::string,S32>::const_iterator it = strings.begin(); it!=strings.end(); ++it) + for (const std::map<std::string,S32>::value_type& str_pair : strings) { - os << it->first << "[" << it->second << "]" << "\n"; + os << str_pair.first << "[" << str_pair.second << "]" << "\n"; } } diff --git a/indra/llcommon/lldefs.h b/indra/llcommon/lldefs.h index 5c46f6a796..4e25001fff 100644 --- a/indra/llcommon/lldefs.h +++ b/indra/llcommon/lldefs.h @@ -167,48 +167,34 @@ const U32 MAXADDRSTR = 17; // 123.567.901.345 = 15 chars + \0 + 1 for good luc // // defined for U16, U32, U64, S16, S32, S64, : // llclampb(a) // clamps a to [0 .. 255] -// - -template <typename T1, typename T2> -inline auto llmax(T1 d1, T2 d2) -{ - return (d1 > d2) ? d1 : d2; -} - -template <typename T1, typename T2, typename T3> -inline auto llmax(T1 d1, T2 d2, T3 d3) -{ - auto r = llmax(d1,d2); - return llmax(r, d3); -} +// -template <typename T1, typename T2, typename T3, typename T4> -inline auto llmax(T1 d1, T2 d2, T3 d3, T4 d4) +// recursion tail +template <typename T> +inline auto llmax(T data) { - auto r1 = llmax(d1,d2); - auto r2 = llmax(d3,d4); - return llmax(r1, r2); + return data; } -template <typename T1, typename T2> -inline auto llmin(T1 d1, T2 d2) +template <typename T0, typename T1, typename... Ts> +inline auto llmax(T0 d0, T1 d1, Ts... rest) { - return (d1 < d2) ? d1 : d2; + auto maxrest = llmax(d1, rest...); + return (d0 > maxrest)? d0 : maxrest; } -template <typename T1, typename T2, typename T3> -inline auto llmin(T1 d1, T2 d2, T3 d3) +// recursion tail +template <typename T> +inline auto llmin(T data) { - auto r = llmin(d1,d2); - return (r < d3 ? r : d3); + return data; } -template <typename T1, typename T2, typename T3, typename T4> -inline auto llmin(T1 d1, T2 d2, T3 d3, T4 d4) +template <typename T0, typename T1, typename... Ts> +inline auto llmin(T0 d0, T1 d1, Ts... rest) { - auto r1 = llmin(d1,d2); - auto r2 = llmin(d3,d4); - return llmin(r1, r2); + auto minrest = llmin(d1, rest...); + return (d0 < minrest) ? d0 : minrest; } template <typename A, typename MIN, typename MAX> diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index f9eda3cd4e..02cb186275 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -586,11 +586,9 @@ namespace void Globals::invalidateCallSites() { - for (CallSiteVector::const_iterator i = callSites.begin(); - i != callSites.end(); - ++i) + for (LLError::CallSite* site : callSites) { - (*i)->invalidate(); + site->invalidate(); } callSites.clear(); @@ -1224,12 +1222,8 @@ namespace std::string escaped_message; LLMutexLock lock(&s->mRecorderMutex); - for (Recorders::const_iterator i = s->mRecorders.begin(); - i != s->mRecorders.end(); - ++i) + for (LLError::RecorderPtr& r : s->mRecorders) { - LLError::RecorderPtr r = *i; - if (!r->enabled()) { continue; diff --git a/indra/llcommon/llevent.cpp b/indra/llcommon/llevent.cpp index 633df01588..501d06e3cd 100644 --- a/indra/llcommon/llevent.cpp +++ b/indra/llcommon/llevent.cpp @@ -203,10 +203,9 @@ void LLSimpleDispatcher::removeListener(LLEventListener* listener) std::vector<LLListenerEntry> LLSimpleDispatcher::getListeners() const { std::vector<LLListenerEntry> ret; - std::vector<LLListenerEntry>::const_iterator itor; - for (itor=mListeners.begin(); itor!=mListeners.end(); ++itor) + for (const LLListenerEntry& entry : mListeners) { - ret.push_back(*itor); + ret.push_back(entry); } return ret; @@ -215,14 +214,12 @@ std::vector<LLListenerEntry> LLSimpleDispatcher::getListeners() const // virtual bool LLSimpleDispatcher::fireEvent(LLPointer<LLEvent> event, LLSD filter) { - std::vector<LLListenerEntry>::iterator itor; std::string filter_string = filter.asString(); - for (itor=mListeners.begin(); itor!=mListeners.end(); ++itor) + for (LLListenerEntry& entry : mListeners) { - LLListenerEntry& entry = *itor; if (filter_string == "" || entry.filter.asString() == filter_string) { - (entry.listener)->handleEvent(event, (*itor).userdata); + (entry.listener)->handleEvent(event, entry.userdata); } } return true; @@ -276,10 +273,9 @@ void LLSimpleListener::clearDispatchers() bool LLSimpleListener::handleAttach(LLEventDispatcher *dispatcher) { // Add dispatcher if it doesn't already exist - std::vector<LLEventDispatcher *>::iterator itor; - for (itor = mDispatchers.begin(); itor != mDispatchers.end(); ++itor) + for (LLEventDispatcher* disp : mDispatchers) { - if ((*itor) == dispatcher) return true; + if (disp == dispatcher) return true; } mDispatchers.push_back(dispatcher); return true; diff --git a/indra/llcommon/llheteromap.cpp b/indra/llcommon/llheteromap.cpp index 7c19196e0c..c84e49d085 100644 --- a/indra/llcommon/llheteromap.cpp +++ b/indra/llcommon/llheteromap.cpp @@ -22,11 +22,11 @@ LLHeteroMap::~LLHeteroMap() { // For each entry in our map, we must call its deleter, which is the only // record we have of its original type. - for (TypeMap::iterator mi(mMap.begin()), me(mMap.end()); mi != me; ++mi) + for (TypeMap::value_type& pair : mMap) { - // mi->second is the std::pair; mi->second.first is the void*; - // mi->second.second points to the deleter function - (mi->second.second)(mi->second.first); - mi->second.first = NULL; + // pair.second is the std::pair; pair.second.first is the void*; + // pair.second.second points to the deleter function + (pair.second.second)(pair.second.first); + pair.second.first = NULL; } } diff --git a/indra/llcommon/llinitdestroyclass.cpp b/indra/llcommon/llinitdestroyclass.cpp index e6382a7924..e3b9e6d099 100644 --- a/indra/llcommon/llinitdestroyclass.cpp +++ b/indra/llcommon/llinitdestroyclass.cpp @@ -21,10 +21,9 @@ void LLCallbackRegistry::fireCallbacks() const { - for (FuncList::const_iterator fi = mCallbacks.begin(), fe = mCallbacks.end(); - fi != fe; ++fi) + for (FuncList::value_type pair : mCallbacks) { - LL_INFOS("LLInitDestroyClass") << "calling " << fi->first << "()" << LL_ENDL; - fi->second(); + LL_INFOS("LLInitDestroyClass") << "calling " << pair.first << "()" << LL_ENDL; + pair.second(); } } diff --git a/indra/llcommon/llinitparam.cpp b/indra/llcommon/llinitparam.cpp index aa2f4eb289..d15bd2f619 100644 --- a/indra/llcommon/llinitparam.cpp +++ b/indra/llcommon/llinitparam.cpp @@ -207,10 +207,10 @@ namespace LLInitParam if (!mValidated) { const BlockDescriptor& block_data = mostDerivedBlockDescriptor(); - for (BlockDescriptor::param_validation_list_t::const_iterator it = block_data.mValidationList.begin(); it != block_data.mValidationList.end(); ++it) + for (const BlockDescriptor::param_validation_list_t::value_type& pair : block_data.mValidationList) { - const Param* param = getParamFromHandle(it->first); - if (!it->second(param)) + const Param* param = getParamFromHandle(pair.first); + if (!pair.second(param)) { if (emit_errors) { @@ -235,13 +235,11 @@ namespace LLInitParam // unnamed param is like LLView::Params::rect - implicit const BlockDescriptor& block_data = mostDerivedBlockDescriptor(); - for (BlockDescriptor::param_list_t::const_iterator it = block_data.mUnnamedParams.begin(); - it != block_data.mUnnamedParams.end(); - ++it) + for (const ParamDescriptorPtr& ptr : block_data.mUnnamedParams) { - param_handle_t param_handle = (*it)->mParamHandle; + param_handle_t param_handle = ptr->mParamHandle; const Param* param = getParamFromHandle(param_handle); - ParamDescriptor::serialize_func_t serialize_func = (*it)->mSerializeFunc; + ParamDescriptor::serialize_func_t serialize_func = ptr->mSerializeFunc; if (serialize_func && predicate_rule.check(ll_make_predicate(PROVIDED, param->anyProvided()))) { const Param* diff_param = diff_block ? diff_block->getParamFromHandle(param_handle) : NULL; @@ -249,23 +247,19 @@ namespace LLInitParam } } - for(BlockDescriptor::param_map_t::const_iterator it = block_data.mNamedParams.begin(); - it != block_data.mNamedParams.end(); - ++it) + for (const BlockDescriptor::param_map_t::value_type& pair : block_data.mNamedParams) { - param_handle_t param_handle = it->second->mParamHandle; + param_handle_t param_handle = pair.second->mParamHandle; const Param* param = getParamFromHandle(param_handle); - ParamDescriptor::serialize_func_t serialize_func = it->second->mSerializeFunc; + ParamDescriptor::serialize_func_t serialize_func = pair.second->mSerializeFunc; if (serialize_func && predicate_rule.check(ll_make_predicate(PROVIDED, param->anyProvided()))) { // Ensure this param has not already been serialized // Prevents <rect> from being serialized as its own tag. bool duplicate = false; - for (BlockDescriptor::param_list_t::const_iterator it2 = block_data.mUnnamedParams.begin(); - it2 != block_data.mUnnamedParams.end(); - ++it2) + for (const ParamDescriptorPtr& ptr : block_data.mUnnamedParams) { - if (param_handle == (*it2)->mParamHandle) + if (param_handle == ptr->mParamHandle) { duplicate = true; break; @@ -279,7 +273,7 @@ namespace LLInitParam continue; } - name_stack.push_back(std::make_pair(it->first, !duplicate)); + name_stack.push_back(std::make_pair(pair.first, !duplicate)); const Param* diff_param = diff_block ? diff_block->getParamFromHandle(param_handle) : NULL; serialized |= serialize_func(*param, parser, name_stack, predicate_rule, diff_param); name_stack.pop_back(); @@ -300,45 +294,39 @@ namespace LLInitParam // unnamed param is like LLView::Params::rect - implicit const BlockDescriptor& block_data = mostDerivedBlockDescriptor(); - for (BlockDescriptor::param_list_t::const_iterator it = block_data.mUnnamedParams.begin(); - it != block_data.mUnnamedParams.end(); - ++it) + for (const ParamDescriptorPtr& ptr : block_data.mUnnamedParams) { - param_handle_t param_handle = (*it)->mParamHandle; + param_handle_t param_handle = ptr->mParamHandle; const Param* param = getParamFromHandle(param_handle); - ParamDescriptor::inspect_func_t inspect_func = (*it)->mInspectFunc; + ParamDescriptor::inspect_func_t inspect_func = ptr->mInspectFunc; if (inspect_func) { name_stack.push_back(std::make_pair("", true)); - inspect_func(*param, parser, name_stack, (*it)->mMinCount, (*it)->mMaxCount); + inspect_func(*param, parser, name_stack, ptr->mMinCount, ptr->mMaxCount); name_stack.pop_back(); } } - for(BlockDescriptor::param_map_t::const_iterator it = block_data.mNamedParams.begin(); - it != block_data.mNamedParams.end(); - ++it) + for(const BlockDescriptor::param_map_t::value_type& pair : block_data.mNamedParams) { - param_handle_t param_handle = it->second->mParamHandle; + param_handle_t param_handle = pair.second->mParamHandle; const Param* param = getParamFromHandle(param_handle); - ParamDescriptor::inspect_func_t inspect_func = it->second->mInspectFunc; + ParamDescriptor::inspect_func_t inspect_func = pair.second->mInspectFunc; if (inspect_func) { // Ensure this param has not already been inspected bool duplicate = false; - for (BlockDescriptor::param_list_t::const_iterator it2 = block_data.mUnnamedParams.begin(); - it2 != block_data.mUnnamedParams.end(); - ++it2) + for (const ParamDescriptorPtr &ptr : block_data.mUnnamedParams) { - if (param_handle == (*it2)->mParamHandle) + if (param_handle == ptr->mParamHandle) { duplicate = true; break; } } - name_stack.push_back(std::make_pair(it->first, !duplicate)); - inspect_func(*param, parser, name_stack, it->second->mMinCount, it->second->mMaxCount); + name_stack.push_back(std::make_pair(pair.first, !duplicate)); + inspect_func(*param, parser, name_stack, pair.second->mMinCount, pair.second->mMaxCount); name_stack.pop_back(); } } @@ -382,12 +370,10 @@ namespace LLInitParam } // try to parse unnamed parameters, in declaration order - for ( BlockDescriptor::param_list_t::iterator it = block_data.mUnnamedParams.begin(); - it != block_data.mUnnamedParams.end(); - ++it) + for (ParamDescriptorPtr& ptr : block_data.mUnnamedParams) { - Param* paramp = getParamFromHandle((*it)->mParamHandle); - ParamDescriptor::deserialize_func_t deserialize_func = (*it)->mDeserializeFunc; + Param* paramp = getParamFromHandle(ptr->mParamHandle); + ParamDescriptor::deserialize_func_t deserialize_func = ptr->mDeserializeFunc; if (deserialize_func && deserialize_func(*paramp, p, name_stack_range, new_name)) { @@ -453,12 +439,9 @@ namespace LLInitParam { param_handle_t handle = getHandleFromParam(¶m); BlockDescriptor& descriptor = mostDerivedBlockDescriptor(); - BlockDescriptor::all_params_list_t::iterator end_it = descriptor.mAllParams.end(); - for (BlockDescriptor::all_params_list_t::iterator it = descriptor.mAllParams.begin(); - it != end_it; - ++it) + for (ParamDescriptorPtr& ptr : descriptor.mAllParams) { - if ((*it)->mParamHandle == handle) return *it; + if (ptr->mParamHandle == handle) return ptr; } return ParamDescriptorPtr(); } @@ -468,17 +451,14 @@ namespace LLInitParam bool BaseBlock::mergeBlock(BlockDescriptor& block_data, const BaseBlock& other, bool overwrite) { bool some_param_changed = false; - BlockDescriptor::all_params_list_t::const_iterator end_it = block_data.mAllParams.end(); - for (BlockDescriptor::all_params_list_t::const_iterator it = block_data.mAllParams.begin(); - it != end_it; - ++it) + for (const ParamDescriptorPtr& ptr : block_data.mAllParams) { - const Param* other_paramp = other.getParamFromHandle((*it)->mParamHandle); - ParamDescriptor::merge_func_t merge_func = (*it)->mMergeFunc; + const Param* other_paramp = other.getParamFromHandle(ptr->mParamHandle); + ParamDescriptor::merge_func_t merge_func = ptr->mMergeFunc; if (merge_func) { - Param* paramp = getParamFromHandle((*it)->mParamHandle); - llassert(paramp->getEnclosingBlockOffset() == (*it)->mParamHandle); + Param* paramp = getParamFromHandle(ptr->mParamHandle); + llassert(paramp->getEnclosingBlockOffset() == ptr->mParamHandle); some_param_changed |= merge_func(*paramp, *other_paramp, overwrite); } } diff --git a/indra/llcommon/llinitparam.h b/indra/llcommon/llinitparam.h index 7f5b9b4ac2..9edc7e40f3 100644 --- a/indra/llcommon/llinitparam.h +++ b/indra/llcommon/llinitparam.h @@ -325,13 +325,11 @@ namespace LLInitParam std::string calcValueName(const value_t& value) const { value_name_map_t* map = getValueNames(); - for (typename value_name_map_t::iterator it = map->begin(), end_it = map->end(); - it != end_it; - ++it) + for (typename value_name_map_t::value_type& map_pair : *map) { - if (ParamCompare<T>::equals(it->second, value)) + if (ParamCompare<T>::equals(map_pair.second, value)) { - return it->first; + return map_pair.first; } } @@ -376,11 +374,9 @@ namespace LLInitParam static std::vector<std::string> sValues; value_name_map_t* map = getValueNames(); - for (typename value_name_map_t::iterator it = map->begin(), end_it = map->end(); - it != end_it; - ++it) + for (typename value_name_map_t::value_type& map_pair : *map) { - sValues.push_back(it->first); + sValues.push_back(map_pair.first); } return &sValues; } diff --git a/indra/llcommon/llkeybind.cpp b/indra/llcommon/llkeybind.cpp index 12e57ae94b..b89160cc55 100644 --- a/indra/llcommon/llkeybind.cpp +++ b/indra/llcommon/llkeybind.cpp @@ -207,9 +207,9 @@ bool LLKeyBind::operator!=(const LLKeyBind& rhs) bool LLKeyBind::isEmpty() const { - for (data_vector_t::const_iterator iter = mData.begin(); iter != mData.end(); iter++) + for (const LLKeyData& key_data : mData) { - if (!iter->isEmpty()) return false; + if (!key_data.isEmpty()) return false; } return true; } @@ -225,12 +225,11 @@ LLKeyBind::data_vector_t::const_iterator LLKeyBind::endNonEmpty() const LLSD LLKeyBind::asLLSD() const { LLSD data; - auto end{ endNonEmpty() }; - for (auto it = mData.begin(); it < end; ++it) + for (const LLKeyData& key_data : mData) { // append intermediate entries even if empty to not affect visual // representation - data.append(it->asLLSD()); + data.append(key_data.asLLSD()); } return data; } @@ -243,9 +242,9 @@ bool LLKeyBind::canHandle(EMouseClickType mouse, KEY key, MASK mask) const return false; } - for (data_vector_t::const_iterator iter = mData.begin(); iter != mData.end(); iter++) + for (const LLKeyData& key_data : mData) { - if (iter->canHandle(mouse, key, mask)) + if (key_data.canHandle(mouse, key, mask)) { return true; } @@ -267,12 +266,12 @@ bool LLKeyBind::hasKeyData(EMouseClickType mouse, KEY key, MASK mask, bool ignor { if (mouse != CLICK_NONE || key != KEY_NONE) { - for (data_vector_t::const_iterator iter = mData.begin(); iter != mData.end(); iter++) + for (const LLKeyData& key_data : mData) { - if (iter->mKey == key - && iter->mMask == mask - && iter->mMouse == mouse - && iter->mIgnoreMasks == ignore) + if (key_data.mKey == key + && key_data.mMask == mask + && key_data.mMouse == mouse + && key_data.mIgnoreMasks == ignore) { return true; } @@ -354,16 +353,16 @@ void LLKeyBind::replaceKeyData(const LLKeyData& data, U32 index) { // if both click and key are none (isEmpty()), we are inserting a placeholder, we don't want to reset anything // otherwise reset identical key - for (data_vector_t::iterator iter = mData.begin(); iter != mData.end(); iter++) + for (LLKeyData& key_data : mData) { - if (iter->mKey == data.mKey - && iter->mMouse == data.mMouse - && iter->mIgnoreMasks == data.mIgnoreMasks - && iter->mMask == data.mMask) + if (key_data.mKey == data.mKey + && key_data.mMouse == data.mMouse + && key_data.mIgnoreMasks == data.mIgnoreMasks + && key_data.mMask == data.mMask) { // Replacing only fully equal combinations even in case 'ignore' is set // Reason: Simplicity and user might decide to do a 'move' command as W and Shift+Ctrl+W, and 'run' as Shift+W - iter->reset(); + key_data.reset(); break; } } diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index c87c0758fe..259f5bc505 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -204,30 +204,35 @@ public: LLSD packet(LLSDMap("pump", pump)("data", data)); std::ostringstream buffer; - buffer << LLSDNotationStreamer(packet); + // SL-18330: for large data blocks, it's much faster to parse binary + // LLSD than notation LLSD. Use serialize(LLSD_BINARY) rather than + // directly calling LLSDBinaryFormatter because, unlike the latter, + // serialize() prepends the relevant header, needed by a general- + // purpose LLSD parser to distinguish binary from notation. + LLSDSerialize::serialize(packet, buffer, LLSDSerialize::LLSD_BINARY, + LLSDFormatter::OPTIONS_NONE); /*==========================================================================*| // DEBUGGING ONLY: don't copy str() if we can avoid it. std::string strdata(buffer.str()); if (std::size_t(buffer.tellp()) != strdata.length()) { - LL_ERRS("LLLeap") << "tellp() -> " << buffer.tellp() << " != " + LL_ERRS("LLLeap") << "tellp() -> " << static_cast<U64>(buffer.tellp()) << " != " << "str().length() -> " << strdata.length() << LL_ENDL; } // DEBUGGING ONLY: reading back is terribly inefficient. std::istringstream readback(strdata); LLSD echo; - LLPointer<LLSDParser> parser(new LLSDNotationParser()); - S32 parse_status(parser->parse(readback, echo, strdata.length())); - if (parse_status == LLSDParser::PARSE_FAILURE) + bool parse_status(LLSDSerialize::deserialize(echo, readback, strdata.length())); + if (! parse_status) { - LL_ERRS("LLLeap") << "LLSDNotationParser() cannot parse output of " - << "LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() cannot parse output of " + << "LLSDSerialize::serialize(LLSD_BINARY)" << LL_ENDL; } if (! llsd_equals(echo, packet)) { - LL_ERRS("LLLeap") << "LLSDNotationParser() produced different LLSD " - << "than passed to LLSDNotationStreamer()" << LL_ENDL; + LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() returned different LLSD " + << "than passed to LLSDSerialize::serialize()" << LL_ENDL; } |*==========================================================================*/ @@ -314,9 +319,17 @@ public: LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got " << childout.size() << ", parsing LLSD" << LL_ENDL; LLSD data; +#if 1 + // specifically require notation LLSD from child LLPointer<LLSDParser> parser(new LLSDNotationParser()); S32 parse_status(parser->parse(childout.get_istream(), data, mExpect)); if (parse_status == LLSDParser::PARSE_FAILURE) +#else + // SL-18330: accept any valid LLSD serialization format from child + // Unfortunately this runs into trouble we have not yet debugged. + bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect)); + if (! parse_status) +#endif { bad_protocol("unparseable LLSD data"); } diff --git a/indra/llcommon/llmd5.cpp b/indra/llcommon/llmd5.cpp index 9b2a2bab60..0abe817f1d 100644 --- a/indra/llcommon/llmd5.cpp +++ b/indra/llcommon/llmd5.cpp @@ -96,7 +96,7 @@ LLMD5::LLMD5() // operation, processing another message block, and updating the // context. -void LLMD5::update (const uint1 *input, const size_t input_length) { +void LLMD5::update (const uint8_t *input, const size_t input_length) { size_t input_index, buffer_index; size_t buffer_space; // how much space is left in buffer @@ -189,7 +189,7 @@ void LLMD5::finalize (){ unsigned char bits[8]; /* Flawfinder: ignore */ size_t index, padLen; - static uint1 PADDING[64]={ + static uint8_t PADDING[64]={ 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 @@ -201,8 +201,8 @@ void LLMD5::finalize (){ } // Save number of bits. - // Treat count, a uint64_t, as uint4[2]. - encode (bits, reinterpret_cast<uint4*>(&count), 8); + // Treat count, a uint64_t, as uint32_t[2]. + encode (bits, reinterpret_cast<uint32_t*>(&count), 8); // Pad out to 56 mod 64. index = size_t((count >> 3) & 0x3f); @@ -412,7 +412,7 @@ Rotation is separate from addition to prevent recomputation. // LLMD5 basic transformation. Transforms state based on block. void LLMD5::transform (const U8 block[64]){ - uint4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; + uint32_t a = state[0], b = state[1], c = state[2], d = state[3], x[16]; decode (x, block, 64); @@ -496,38 +496,38 @@ void LLMD5::transform (const U8 block[64]){ state[3] += d; // Zeroize sensitive information. - memset ( (uint1 *) x, 0, sizeof(x)); + memset ( (uint8_t *) x, 0, sizeof(x)); } -// Encodes input (UINT4) into output (unsigned char). Assumes len is +// Encodes input (uint32_t) into output (unsigned char). Assumes len is // a multiple of 4. -void LLMD5::encode (uint1 *output, const uint4 *input, const size_t len) { +void LLMD5::encode (uint8_t *output, const uint32_t *input, const size_t len) { size_t i, j; for (i = 0, j = 0; j < len; i++, j += 4) { - output[j] = (uint1) (input[i] & 0xff); - output[j+1] = (uint1) ((input[i] >> 8) & 0xff); - output[j+2] = (uint1) ((input[i] >> 16) & 0xff); - output[j+3] = (uint1) ((input[i] >> 24) & 0xff); + output[j] = (uint8_t) (input[i] & 0xff); + output[j+1] = (uint8_t) ((input[i] >> 8) & 0xff); + output[j+2] = (uint8_t) ((input[i] >> 16) & 0xff); + output[j+3] = (uint8_t) ((input[i] >> 24) & 0xff); } } -// Decodes input (unsigned char) into output (UINT4). Assumes len is +// Decodes input (unsigned char) into output (uint32_t). Assumes len is // a multiple of 4. -void LLMD5::decode (uint4 *output, const uint1 *input, const size_t len){ +void LLMD5::decode (uint32_t *output, const uint8_t *input, const size_t len){ size_t i, j; for (i = 0, j = 0; j < len; i++, j += 4) - output[i] = ((uint4)input[j]) | (((uint4)input[j+1]) << 8) | - (((uint4)input[j+2]) << 16) | (((uint4)input[j+3]) << 24); + output[i] = ((uint32_t)input[j]) | (((uint32_t)input[j+1]) << 8) | + (((uint32_t)input[j+2]) << 16) | (((uint32_t)input[j+3]) << 24); } diff --git a/indra/llcommon/llmd5.h b/indra/llcommon/llmd5.h index 8530dc0389..7d6373c20c 100644 --- a/indra/llcommon/llmd5.h +++ b/indra/llcommon/llmd5.h @@ -67,6 +67,8 @@ documentation and/or software. */ +#include <cstdint> // uint32_t et al. + // use for the raw digest output const int MD5RAW_BYTES = 16; @@ -75,18 +77,13 @@ const int MD5HEX_STR_SIZE = 33; // char hex[MD5HEX_STR_SIZE]; with null const int MD5HEX_STR_BYTES = 32; // message system fixed size class LL_COMMON_API LLMD5 { -// first, some types: - typedef unsigned int uint4; // assumes integer is 4 words long - typedef unsigned short int uint2; // assumes short integer is 2 words long - typedef unsigned char uint1; // assumes char is 1 word long - // how many bytes to grab at a time when checking files static const int BLOCK_LEN; public: // methods for controlled operation: LLMD5 (); // simple initializer - void update (const uint1 *input, const size_t input_length); + void update (const uint8_t *input, const size_t input_length); void update (std::istream& stream); void update (FILE *file); void update (const std::string& str); @@ -109,19 +106,19 @@ private: // next, the private data: - uint4 state[4]; + uint32_t state[4]; uint64_t count; // number of *bits*, mod 2^64 - uint1 buffer[64]; // input buffer - uint1 digest[16]; - uint1 finalized; + uint8_t buffer[64]; // input buffer + uint8_t digest[16]; + uint8_t finalized; // last, the private methods, mostly static: void init (); // called by all constructors - void transform (const uint1 *buffer); // does the real update work. Note + void transform (const uint8_t *buffer); // does the real update work. Note // that length is implied to be 64. - static void encode (uint1 *dest, const uint4 *src, const size_t length); - static void decode (uint4 *dest, const uint1 *src, const size_t length); + static void encode (uint8_t *dest, const uint32_t *src, const size_t length); + static void decode (uint32_t *dest, const uint8_t *src, const size_t length); }; diff --git a/indra/llcommon/llmetricperformancetester.cpp b/indra/llcommon/llmetricperformancetester.cpp index 864ecf650b..ab509b46eb 100644 --- a/indra/llcommon/llmetricperformancetester.cpp +++ b/indra/llcommon/llmetricperformancetester.cpp @@ -42,9 +42,9 @@ LLMetricPerformanceTesterBasic::name_tester_map_t LLMetricPerformanceTesterBasic /*static*/ void LLMetricPerformanceTesterBasic::cleanupClass() { - for (name_tester_map_t::iterator iter = sTesterMap.begin() ; iter != sTesterMap.end() ; ++iter) + for (name_tester_map_t::value_type& pair : sTesterMap) { - delete iter->second ; + delete pair.second; } sTesterMap.clear() ; } @@ -154,10 +154,9 @@ void LLMetricPerformanceTesterBasic::doAnalysisMetrics(std::string baseline, std llofstream os(output.c_str()); os << "Label, Metric, Base(B), Target(T), Diff(T-B), Percentage(100*T/B)\n"; - for(LLMetricPerformanceTesterBasic::name_tester_map_t::iterator iter = LLMetricPerformanceTesterBasic::sTesterMap.begin() ; - iter != LLMetricPerformanceTesterBasic::sTesterMap.end() ; ++iter) + for (LLMetricPerformanceTesterBasic::name_tester_map_t::value_type& pair : LLMetricPerformanceTesterBasic::sTesterMap) { - LLMetricPerformanceTesterBasic* tester = ((LLMetricPerformanceTesterBasic*)iter->second) ; + LLMetricPerformanceTesterBasic* tester = ((LLMetricPerformanceTesterBasic*)pair.second); tester->analyzePerformance(&os, &base, ¤t) ; } diff --git a/indra/llcommon/llnametable.h b/indra/llcommon/llnametable.h index d3283543f3..2c8e71263e 100644 --- a/indra/llcommon/llnametable.h +++ b/indra/llcommon/llnametable.h @@ -86,12 +86,10 @@ public: // O(N)! (currently only used in one place... (newsim/llstate.cpp)) const char *resolveData(const DATA &data) const { - const_iter_t iter = mNameMap.begin(); - const_iter_t end = mNameMap.end(); - for (; iter != end; ++iter) + for (const name_map_t::value_type& pair : mNameMap) { - if (iter->second == data) - return iter->first; + if (pair.second == data) + return pair.first; } return NULL; } diff --git a/indra/llcommon/llpriqueuemap.h b/indra/llcommon/llpriqueuemap.h index d8d3edd48a..030e2e0f21 100644 --- a/indra/llcommon/llpriqueuemap.h +++ b/indra/llcommon/llpriqueuemap.h @@ -115,9 +115,9 @@ public: LL_WARNS() << "Data not on priority queue!" << LL_ENDL; // OK, try iterating through all of the data and seeing if we just screwed up the priority // somehow. - for (iter = mMap.begin(); iter != mMap.end(); iter++) + for (pqm_pair pair : mMap) { - if ((*(iter)).second == data) + if (pair.second == data) { LL_ERRS() << "Data on priority queue but priority not matched!" << LL_ENDL; } diff --git a/indra/llcommon/llregistry.h b/indra/llcommon/llregistry.h index 750fe9fdc8..e272d7a9b8 100644 --- a/indra/llcommon/llregistry.h +++ b/indra/llcommon/llregistry.h @@ -141,11 +141,9 @@ public: ptr_value_t getValue(ref_const_key_t key) { - for(scope_list_iterator_t it = mActiveScopes.begin(); - it != mActiveScopes.end(); - ++it) + for(Registrar* scope : mActiveScopes) { - ptr_value_t valuep = (*it)->getValue(key); + ptr_value_t valuep = scope->getValue(key); if (valuep != NULL) return valuep; } return mDefaultRegistrar.getValue(key); @@ -153,11 +151,9 @@ public: ptr_const_value_t getValue(ref_const_key_t key) const { - for(scope_list_const_iterator_t it = mActiveScopes.begin(); - it != mActiveScopes.end(); - ++it) + for(const Registrar* scope : mActiveScopes) { - ptr_value_t valuep = (*it)->getValue(key); + ptr_const_value_t valuep = scope->getValue(key); if (valuep != NULL) return valuep; } return mDefaultRegistrar.getValue(key); @@ -165,11 +161,9 @@ public: bool exists(ref_const_key_t key) const { - for(scope_list_const_iterator_t it = mActiveScopes.begin(); - it != mActiveScopes.end(); - ++it) + for(const Registrar* scope : mActiveScopes) { - if ((*it)->exists(key)) return true; + if (scope->exists(key)) return true; } return mDefaultRegistrar.exists(key); @@ -177,11 +171,9 @@ public: bool empty() const { - for(scope_list_const_iterator_t it = mActiveScopes.begin(); - it != mActiveScopes.end(); - ++it) + for(const Registrar* scope : mActiveScopes) { - if (!(*it)->empty()) return false; + if (!scope->empty()) return false; } return mDefaultRegistrar.empty(); diff --git a/indra/llcommon/llsdparam.cpp b/indra/llcommon/llsdparam.cpp index af4ccf25fd..30f49b27ea 100644 --- a/indra/llcommon/llsdparam.cpp +++ b/indra/llcommon/llsdparam.cpp @@ -113,11 +113,9 @@ void LLParamSDParser::writeSDImpl(LLSD& sd, const LLInitParam::BaseBlock& block, /*virtual*/ std::string LLParamSDParser::getCurrentElementName() { std::string full_name = "sd"; - for (name_stack_t::iterator it = mNameStack.begin(); - it != mNameStack.end(); - ++it) + for (name_stack_t::value_type& stack_pair : mNameStack) { - full_name += llformat("[%s]", it->first.c_str()); + full_name += llformat("[%s]", stack_pair.first.c_str()); } return full_name; diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index c8dbcb2404..a904ce700e 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -48,6 +48,7 @@ #endif #include "lldate.h" +#include "llmemorystream.h" #include "llsd.h" #include "llstring.h" #include "lluri.h" @@ -64,6 +65,23 @@ const std::string LLSD_NOTATION_HEADER("llsd/notation"); #define windowBits 15 #define ENABLE_ZLIB_GZIP 32 +// If we published this in llsdserialize.h, we could use it in the +// implementation of LLSDOStreamer's operator<<(). +template <class Formatter> +void format_using(const LLSD& data, std::ostream& ostr, + LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY) +{ + LLPointer<Formatter> f{ new Formatter }; + f->format(data, ostr, options); +} + +template <class Parser> +S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1) +{ + LLPointer<Parser> p{ new Parser }; + return p->parse(istr, data, max_bytes, max_depth); +} + /** * LLSDSerialize */ @@ -86,10 +104,10 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize f = new LLSDXMLFormatter; break; - case LLSD_NOTATION: - str << "<? " << LLSD_NOTATION_HEADER << " ?>\n"; - f = new LLSDNotationFormatter; - break; + case LLSD_NOTATION: + str << "<? " << LLSD_NOTATION_HEADER << " ?>\n"; + f = new LLSDNotationFormatter; + break; default: LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL; @@ -104,18 +122,37 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize // static bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) { - LLPointer<LLSDParser> p = NULL; char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */ - int i; - int inbuf = 0; - bool legacy_no_header = false; bool fail_if_not_legacy = false; - std::string header; - /* - * Get the first line before anything. - */ - str.get(hdr_buf, MAX_HDR_LEN, '\n'); + /* + * Get the first line before anything. Don't read more than max_bytes: + * this get() overload reads no more than (count-1) bytes into the + * specified buffer. In the usual case when max_bytes exceeds + * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2. + */ + llssize max_hdr_read = MAX_HDR_LEN; + if (max_bytes != LLSDSerialize::SIZE_UNLIMITED) + { + max_hdr_read = llmin(max_bytes + 1, max_hdr_read); + } + str.get(hdr_buf, max_hdr_read, '\n'); + auto inbuf = str.gcount(); + + // https://en.cppreference.com/w/cpp/io/basic_istream/get + // When the get() above sees the specified delimiter '\n', it stops there + // without pulling it from the stream. If it turns out that the stream + // does NOT contain a header, and the content includes meaningful '\n', + // it's important to pull that into hdr_buf too. + if (inbuf < max_bytes && str.get(hdr_buf[inbuf])) + { + // got the delimiting '\n' + ++inbuf; + // None of the following requires that hdr_buf contain a final '\0' + // byte. We could store one if needed, since even the incremented + // inbuf won't exceed sizeof(hdr_buf)-1, but there's no need. + } + std::string header{ hdr_buf, static_cast<std::string::size_type>(inbuf) }; if (str.fail()) { str.clear(); @@ -123,79 +160,97 @@ bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes) } if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */ + { // Create a LLSD XML parser, and parse the first chunk read above. + LLSDXMLParser x; + x.parsePart(hdr_buf, inbuf); // Parse the first part that was already read + auto parsed = x.parse(str, sd, max_bytes - inbuf); // Parse the rest of it + // Formally we should probably check (parsed != PARSE_FAILURE && + // parsed > 0), but since PARSE_FAILURE is -1, this suffices. + return (parsed > 0); + } + + if (fail_if_not_legacy) { - legacy_no_header = true; - inbuf = (int)str.gcount(); + LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; + return false; } - else + + /* + * Remove the newline chars + */ + std::string::size_type lastchar = header.find_last_not_of("\r\n"); + if (lastchar != std::string::npos) { - if (fail_if_not_legacy) - goto fail; - /* - * Remove the newline chars - */ - for (i = 0; i < MAX_HDR_LEN; i++) - { - if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' || - hdr_buf[i] == '\n') - { - hdr_buf[i] = 0; - break; - } - } - header = hdr_buf; + // It's important that find_last_not_of() returns size_type, which is + // why lastchar explicitly declares the type above. erase(size_type) + // erases from that offset to the end of the string, whereas + // erase(iterator) erases only a single character. + header.erase(lastchar+1); + } - std::string::size_type start = std::string::npos; - std::string::size_type end = std::string::npos; - start = header.find_first_not_of("<? "); - if (start != std::string::npos) + // trim off the <? ... ?> header syntax + auto start = header.find_first_not_of("<? "); + if (start != std::string::npos) + { + auto end = header.find_first_of(" ?", start); + if (end != std::string::npos) { - end = header.find_first_of(" ?", start); + header = header.substr(start, end - start); + ws(str); } - if ((start == std::string::npos) || (end == std::string::npos)) - goto fail; - - header = header.substr(start, end - start); - ws(str); } /* * Create the parser as appropriate */ - if (legacy_no_header) - { // Create a LLSD XML parser, and parse the first chunk read above - LLSDXMLParser* x = new LLSDXMLParser(); - x->parsePart(hdr_buf, inbuf); // Parse the first part that was already read - x->parseLines(str, sd); // Parse the rest of it - delete x; - return true; - } - - if (header == LLSD_BINARY_HEADER) + if (0 == LLStringUtil::compareInsensitive(header, LLSD_BINARY_HEADER)) { - p = new LLSDBinaryParser; + return (parse_using<LLSDBinaryParser>(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_XML_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_XML_HEADER)) { - p = new LLSDXMLParser; + return (parse_using<LLSDXMLParser>(str, sd, max_bytes-inbuf) > 0); } - else if (header == LLSD_NOTATION_HEADER) + else if (0 == LLStringUtil::compareInsensitive(header, LLSD_NOTATION_HEADER)) { - p = new LLSDNotationParser; + return (parse_using<LLSDNotationParser>(str, sd, max_bytes-inbuf) > 0); } - else + else // no header we recognize { - LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL; - } - - if (p.notNull()) - { - p->parse(str, sd, max_bytes); - return true; + LLPointer<LLSDParser> p; + if (inbuf && hdr_buf[0] == '<') + { + // looks like XML + LL_DEBUGS() << "deserialize request with no header, assuming XML" << LL_ENDL; + p = new LLSDXMLParser; + } + else + { + // assume notation + LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL; + p = new LLSDNotationParser; + } + // Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that + // data to whatever remains in 'str'. + LLMemoryStreamBuf already(reinterpret_cast<const U8*>(hdr_buf), inbuf); + cat_streambuf prebuff(&already, str.rdbuf()); + std::istream prepend(&prebuff); +#if 1 + return (p->parse(prepend, sd, max_bytes) > 0); +#else + // debugging the reconstituted 'prepend' stream + // allocate a buffer that we hope is big enough for the whole thing + std::vector<char> wholemsg((max_bytes == size_t(SIZE_UNLIMITED))? 1024 : max_bytes); + prepend.read(wholemsg.data(), std::min(max_bytes, wholemsg.size())); + LLMemoryStream replay(reinterpret_cast<const U8*>(wholemsg.data()), prepend.gcount()); + auto success{ p->parse(replay, sd, prepend.gcount()) > 0 }; + { + LL_DEBUGS() << (success? "parsed: $$" : "failed: '") + << std::string(wholemsg.data(), llmin(prepend.gcount(), 100)) << "$$" + << LL_ENDL; + } + return success; +#endif } - -fail: - LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL; - return false; } /** diff --git a/indra/llcommon/llsdutil.cpp b/indra/llcommon/llsdutil.cpp index 8e90d1e8b8..f70bee9903 100644 --- a/indra/llcommon/llsdutil.cpp +++ b/indra/llcommon/llsdutil.cpp @@ -148,10 +148,9 @@ LLSD ll_binary_from_string(const LLSD& sd) std::vector<U8> binary_value; std::string string_value = sd.asString(); - for (std::string::iterator iter = string_value.begin(); - iter != string_value.end(); ++iter) + for (const U8 c : string_value) { - binary_value.push_back(*iter); + binary_value.push_back(c); } binary_value.push_back('\0'); diff --git a/indra/llcommon/llsdutil.h b/indra/llcommon/llsdutil.h index 1321615805..372278c51a 100644 --- a/indra/llcommon/llsdutil.h +++ b/indra/llcommon/llsdutil.h @@ -191,75 +191,6 @@ LLSD& drill_ref( LLSD& blob, const LLSD& path); } -/***************************************************************************** -* LLSDArray -*****************************************************************************/ -/** - * Construct an LLSD::Array inline, with implicit conversion to LLSD. Usage: - * - * @code - * void somefunc(const LLSD&); - * ... - * somefunc(LLSDArray("text")(17)(3.14)); - * @endcode - * - * For completeness, LLSDArray() with no args constructs an empty array, so - * <tt>LLSDArray()("text")(17)(3.14)</tt> produces an array equivalent to the - * above. But for most purposes, LLSD() is already equivalent to an empty - * array, and if you explicitly want an empty isArray(), there's - * LLSD::emptyArray(). However, supporting a no-args LLSDArray() constructor - * follows the principle of least astonishment. - */ -class LLSDArray -{ -public: - LLSDArray(): - _data(LLSD::emptyArray()) - {} - - /** - * Need an explicit copy constructor. Consider the following: - * - * @code - * LLSD array_of_arrays(LLSDArray(LLSDArray(17)(34)) - * (LLSDArray("x")("y"))); - * @endcode - * - * The coder intends to construct [[17, 34], ["x", "y"]]. - * - * With the compiler's implicit copy constructor, s/he gets instead - * [17, 34, ["x", "y"]]. - * - * The expression LLSDArray(17)(34) constructs an LLSDArray with those two - * values. The reader assumes it should be converted to LLSD, as we always - * want with LLSDArray, before passing it to the @em outer LLSDArray - * constructor! This copy constructor makes that happen. - */ - LLSDArray(const LLSDArray& inner): - _data(LLSD::emptyArray()) - { - _data.append(inner); - } - - LLSDArray(const LLSD& value): - _data(LLSD::emptyArray()) - { - _data.append(value); - } - - LLSDArray& operator()(const LLSD& value) - { - _data.append(value); - return *this; - } - - operator LLSD() const { return _data; } - LLSD get() const { return _data; } - -private: - LLSD _data; -}; - namespace llsd { diff --git a/indra/llcommon/llstreamtools.cpp b/indra/llcommon/llstreamtools.cpp index 1ff15fcf89..bc32b6fd9e 100644 --- a/indra/llcommon/llstreamtools.cpp +++ b/indra/llcommon/llstreamtools.cpp @@ -513,3 +513,29 @@ std::istream& operator>>(std::istream& str, const char *tocheck) } return str; } + +int cat_streambuf::underflow() +{ + if (gptr() == egptr()) + { + // here because our buffer is empty + std::streamsize size = 0; + // Until we've run out of mInputs, try reading the first of them + // into mBuffer. If that fetches some characters, break the loop. + while (! mInputs.empty() + && ! (size = mInputs.front()->sgetn(mBuffer.data(), mBuffer.size()))) + { + // We tried to read mInputs.front() but got zero characters. + // Discard the first streambuf and try the next one. + mInputs.pop_front(); + } + // Either we ran out of mInputs or we succeeded in reading some + // characters, that is, size != 0. Tell base class what we have. + setg(mBuffer.data(), mBuffer.data(), mBuffer.data() + size); + } + // If we fell out of the above loop with mBuffer still empty, return + // eof(), otherwise return the next character. + return (gptr() == egptr()) + ? std::char_traits<char>::eof() + : std::char_traits<char>::to_int_type(*gptr()); +} diff --git a/indra/llcommon/llstreamtools.h b/indra/llcommon/llstreamtools.h index 1b04bf91d7..bb7bc20327 100644 --- a/indra/llcommon/llstreamtools.h +++ b/indra/llcommon/llstreamtools.h @@ -27,8 +27,10 @@ #ifndef LL_STREAM_TOOLS_H #define LL_STREAM_TOOLS_H +#include <deque> #include <iostream> #include <string> +#include <vector> // unless specifed otherwise these all return input_stream.good() @@ -113,6 +115,27 @@ LL_COMMON_API std::streamsize fullread( LL_COMMON_API std::istream& operator>>(std::istream& str, const char *tocheck); -#endif - +/** + * cat_streambuf is a std::streambuf subclass that accepts a variadic number + * of std::streambuf* (e.g. some_istream.rdbuf()) and virtually concatenates + * their contents. + */ +// derived from https://stackoverflow.com/a/49441066/5533635 +class cat_streambuf: public std::streambuf +{ +private: + std::deque<std::streambuf*> mInputs; + std::vector<char> mBuffer; + +public: + // only valid for std::streambuf* arguments + template <typename... Inputs> + cat_streambuf(Inputs... inputs): + mInputs{inputs...}, + mBuffer(1024) + {} + + int underflow() override; +}; +#endif diff --git a/indra/llcommon/llstringtable.cpp b/indra/llcommon/llstringtable.cpp index f288999964..92a5e777a6 100644 --- a/indra/llcommon/llstringtable.cpp +++ b/indra/llcommon/llstringtable.cpp @@ -89,9 +89,8 @@ LLStringTable::~LLStringTable() { if (mStringList[i]) { - string_list_t::iterator iter; - for (iter = mStringList[i]->begin(); iter != mStringList[i]->end(); iter++) - delete *iter; // *iter = (LLStringTableEntry*) + for (LLStringTableEntry* entry : *mStringList[i]) + delete entry; } delete mStringList[i]; } @@ -156,9 +155,9 @@ LLStringTableEntry* LLStringTable::checkStringEntry(const char *str) if (str) { char *ret_val; - LLStringTableEntry *entry; U32 hash_value = hash_my_string(str, mMaxEntries); #if STRING_TABLE_HASH_MAP + LLStringTableEntry *entry; #if 1 // Microsoft string_hash_t::iterator lower = mStringHash.lower_bound(hash_value); string_hash_t::iterator upper = mStringHash.upper_bound(hash_value); @@ -180,10 +179,8 @@ LLStringTableEntry* LLStringTable::checkStringEntry(const char *str) string_list_t *strlist = mStringList[hash_value]; if (strlist) { - string_list_t::iterator iter; - for (iter = strlist->begin(); iter != strlist->end(); iter++) + for (LLStringTableEntry* entry : *strlist) { - entry = *iter; ret_val = entry->mString; if (!strncmp(ret_val, str, MAX_STRINGS_LENGTH)) { @@ -226,9 +223,9 @@ LLStringTableEntry* LLStringTable::addStringEntry(const char *str) if (str) { char *ret_val = NULL; - LLStringTableEntry *entry; U32 hash_value = hash_my_string(str, mMaxEntries); #if STRING_TABLE_HASH_MAP + LLStringTableEntry *entry; #if 1 // Microsoft string_hash_t::iterator lower = mStringHash.lower_bound(hash_value); string_hash_t::iterator upper = mStringHash.upper_bound(hash_value); @@ -257,10 +254,8 @@ LLStringTableEntry* LLStringTable::addStringEntry(const char *str) if (strlist) { - string_list_t::iterator iter; - for (iter = strlist->begin(); iter != strlist->end(); iter++) + for (LLStringTableEntry* entry : *strlist) { - entry = *iter; ret_val = entry->mString; if (!strncmp(ret_val, str, MAX_STRINGS_LENGTH)) { @@ -294,10 +289,10 @@ void LLStringTable::removeString(const char *str) if (str) { char *ret_val; - LLStringTableEntry *entry; U32 hash_value = hash_my_string(str, mMaxEntries); #if STRING_TABLE_HASH_MAP { + LLStringTableEntry *entry; #if 1 // Microsoft string_hash_t::iterator lower = mStringHash.lower_bound(hash_value); string_hash_t::iterator upper = mStringHash.upper_bound(hash_value); @@ -331,10 +326,8 @@ void LLStringTable::removeString(const char *str) if (strlist) { - string_list_t::iterator iter; - for (iter = strlist->begin(); iter != strlist->end(); iter++) + for (LLStringTableEntry* entry : *strlist) { - entry = *iter; ret_val = entry->mString; if (!strncmp(ret_val, str, MAX_STRINGS_LENGTH)) { diff --git a/indra/llcommon/llstringtable.h b/indra/llcommon/llstringtable.h index ff09e71677..0a292c8bac 100644 --- a/indra/llcommon/llstringtable.h +++ b/indra/llcommon/llstringtable.h @@ -136,9 +136,9 @@ public: for (S32 i = 0; i<mTableSize; i++) { string_set_t& stringset = mStringList[i]; - for (string_set_t::iterator iter = stringset.begin(); iter != stringset.end(); iter++) + for (LLStdStringHandle str : stringset) { - delete *iter; + delete str; } stringset.clear(); } diff --git a/indra/llcommon/lltracerecording.cpp b/indra/llcommon/lltracerecording.cpp index dd148dd08a..bb3d667a42 100644 --- a/indra/llcommon/lltracerecording.cpp +++ b/indra/llcommon/lltracerecording.cpp @@ -577,10 +577,12 @@ S32 Recording::getSampleCount( const StatType<EventAccumulator>& stat ) // PeriodicRecording /////////////////////////////////////////////////////////////////////// -PeriodicRecording::PeriodicRecording( S32 num_periods, EPlayState state) +PeriodicRecording::PeriodicRecording( size_t num_periods, EPlayState state) : mAutoResize(num_periods == 0), mCurPeriod(0), mNumRecordedPeriods(0), + // This guarantee that mRecordingPeriods cannot be empty is essential for + // code in several methods. mRecordingPeriods(num_periods ? num_periods : 1) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; @@ -596,18 +598,19 @@ PeriodicRecording::~PeriodicRecording() void PeriodicRecording::nextPeriod() { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; if (mAutoResize) { mRecordingPeriods.push_back(Recording()); } Recording& old_recording = getCurRecording(); - mCurPeriod = (mCurPeriod + 1) % mRecordingPeriods.size(); + inci(mCurPeriod); old_recording.splitTo(getCurRecording()); - mNumRecordedPeriods = mRecordingPeriods.empty()? 0 : - llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + 1); + // Since mRecordingPeriods always has at least one entry, we can always + // safely subtract 1 from its size(). + mNumRecordedPeriods = llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + 1); } void PeriodicRecording::appendRecording(Recording& recording) @@ -620,31 +623,29 @@ void PeriodicRecording::appendRecording(Recording& recording) void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; if (other.mRecordingPeriods.empty()) return; getCurRecording().update(); other.getCurRecording().update(); - - const auto other_recording_slots = other.mRecordingPeriods.size(); + const auto other_num_recordings = other.getNumRecordedPeriods(); const auto other_current_recording_index = other.mCurPeriod; - const auto other_oldest_recording_index = (other_current_recording_index + other_recording_slots - other_num_recordings) % other_recording_slots; + const auto other_oldest_recording_index = other.previ(other_current_recording_index, other_num_recordings); // append first recording into our current slot getCurRecording().appendRecording(other.mRecordingPeriods[other_oldest_recording_index]); // from now on, add new recordings for everything after the first - auto other_index = (other_oldest_recording_index + 1) % other_recording_slots; + auto other_index = other.nexti(other_oldest_recording_index); if (mAutoResize) { // push back recordings for everything in the middle - auto other_index = (other_oldest_recording_index + 1) % other_recording_slots; while (other_index != other_current_recording_index) { mRecordingPeriods.push_back(other.mRecordingPeriods[other_index]); - other_index = (other_index + 1) % other_recording_slots; + other.inci(other_index); } // add final recording, if it wasn't already added as the first @@ -653,36 +654,25 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) mRecordingPeriods.push_back(other.mRecordingPeriods[other_current_recording_index]); } - mCurPeriod = mRecordingPeriods.empty()? 0 : mRecordingPeriods.size() - 1; + // mRecordingPeriods is never empty() + mCurPeriod = mRecordingPeriods.size() - 1; mNumRecordedPeriods = mCurPeriod; } else { - S32 num_to_copy = llmin((S32)mRecordingPeriods.size(), (S32)other_num_recordings); - - std::vector<Recording>::iterator src_it = other.mRecordingPeriods.begin() + other_index ; - std::vector<Recording>::iterator dest_it = mRecordingPeriods.begin() + mCurPeriod; - + auto num_to_copy = llmin(mRecordingPeriods.size(), other_num_recordings); // already consumed the first recording from other, so start counting at 1 - for(S32 i = 1; i < num_to_copy; i++) + for (size_t n = 1, srci = other_index, dsti = mCurPeriod; + n < num_to_copy; + ++n, other.inci(srci), inci(dsti)) { - *dest_it = *src_it; - - if (++src_it == other.mRecordingPeriods.end()) - { - src_it = other.mRecordingPeriods.begin(); - } - - if (++dest_it == mRecordingPeriods.end()) - { - dest_it = mRecordingPeriods.begin(); - } + mRecordingPeriods[dsti] = other.mRecordingPeriods[srci]; } - + // want argument to % to be positive, otherwise result could be negative and thus out of bounds llassert(num_to_copy >= 1); // advance to last recording period copied, and make that our current period - mCurPeriod = (mCurPeriod + num_to_copy - 1) % mRecordingPeriods.size(); + inci(mCurPeriod, num_to_copy - 1); mNumRecordedPeriods = llmin(mRecordingPeriods.size() - 1, mNumRecordedPeriods + num_to_copy - 1); } @@ -694,13 +684,11 @@ void PeriodicRecording::appendPeriodicRecording( PeriodicRecording& other ) F64Seconds PeriodicRecording::getDuration() const { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; F64Seconds duration; - auto num_periods = mRecordingPeriods.size(); - for (size_t i = 1; i <= num_periods; i++) + for (size_t n = 0; n < mRecordingPeriods.size(); ++n) { - auto index = (mCurPeriod + num_periods - i) % num_periods; - duration += mRecordingPeriods[index].getDuration(); + duration += mRecordingPeriods[nexti(mCurPeriod, n)].getDuration(); } return duration; } @@ -737,16 +725,14 @@ const Recording& PeriodicRecording::getCurRecording() const Recording& PeriodicRecording::getPrevRecording( size_t offset ) { - auto num_periods = mRecordingPeriods.size(); - offset = llclamp(offset, 0, num_periods - 1); - return mRecordingPeriods[(mCurPeriod + num_periods - offset) % num_periods]; + // reuse const implementation, but return non-const reference + return const_cast<Recording&>( + const_cast<const PeriodicRecording*>(this)->getPrevRecording(offset)); } const Recording& PeriodicRecording::getPrevRecording( size_t offset ) const { - auto num_periods = mRecordingPeriods.size(); - offset = llclamp(offset, 0, num_periods - 1); - return mRecordingPeriods[(mCurPeriod + num_periods - offset) % num_periods]; + return mRecordingPeriods[previ(mCurPeriod, offset)]; } void PeriodicRecording::handleStart() @@ -773,11 +759,9 @@ void PeriodicRecording::handleReset() } else { - for (std::vector<Recording>::iterator it = mRecordingPeriods.begin(), end_it = mRecordingPeriods.end(); - it != end_it; - ++it) + for (Recording& rec : mRecordingPeriods) { - it->reset(); + rec.reset(); } } mCurPeriod = 0; @@ -791,14 +775,14 @@ void PeriodicRecording::handleSplitTo(PeriodicRecording& other) getCurRecording().splitTo(other.getCurRecording()); } -F64 PeriodicRecording::getPeriodMin( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMin( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 min_val = std::numeric_limits<F64>::max(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -813,14 +797,14 @@ F64 PeriodicRecording::getPeriodMin( const StatType<EventAccumulator>& stat, siz : NaN; } -F64 PeriodicRecording::getPeriodMax( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMax( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 max_val = std::numeric_limits<F64>::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -836,7 +820,7 @@ F64 PeriodicRecording::getPeriodMax( const StatType<EventAccumulator>& stat, siz } // calculates means using aggregates per period -F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -844,7 +828,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, si F64 mean = 0; S32 valid_period_count = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -859,7 +843,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType<EventAccumulator>& stat, si : NaN; } -F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -868,7 +852,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulat F64 sum_of_squares = 0; S32 valid_period_count = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -884,14 +868,14 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<EventAccumulat : NaN; } -F64 PeriodicRecording::getPeriodMin( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMin( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 min_val = std::numeric_limits<F64>::max(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -906,14 +890,14 @@ F64 PeriodicRecording::getPeriodMin( const StatType<SampleAccumulator>& stat, si : NaN; } -F64 PeriodicRecording::getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/) +F64 PeriodicRecording::getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; F64 max_val = std::numeric_limits<F64>::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -929,7 +913,7 @@ F64 PeriodicRecording::getPeriodMax(const StatType<SampleAccumulator>& stat, siz } -F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -937,7 +921,7 @@ F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, s S32 valid_period_count = 0; F64 mean = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -952,13 +936,13 @@ F64 PeriodicRecording::getPeriodMean( const StatType<SampleAccumulator>& stat, s : NaN; } -F64 PeriodicRecording::getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); std::vector<F64> buf; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -978,7 +962,7 @@ F64 PeriodicRecording::getPeriodMedian( const StatType<SampleAccumulator>& stat, return F64((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); } -F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -987,7 +971,7 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumula S32 valid_period_count = 0; F64 sum_of_squares = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -1004,13 +988,13 @@ F64 PeriodicRecording::getPeriodStandardDeviation( const StatType<SampleAccumula } -F64Kilobytes PeriodicRecording::getPeriodMin( const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodMin( const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes min_val(std::numeric_limits<F64>::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); min_val = llmin(min_val, recording.getMin(stat)); @@ -1024,13 +1008,13 @@ F64Kilobytes PeriodicRecording::getPeriodMin(const MemStatHandle& stat, size_t n return getPeriodMin(static_cast<const StatType<MemAccumulator>&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/) +F64Kilobytes PeriodicRecording::getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes max_val(0.0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); max_val = llmax(max_val, recording.getMax(stat)); @@ -1044,14 +1028,14 @@ F64Kilobytes PeriodicRecording::getPeriodMax(const MemStatHandle& stat, size_t n return getPeriodMax(static_cast<const StatType<MemAccumulator>&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodMean( const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodMean( const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64Kilobytes mean(0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); mean += recording.getMean(stat); @@ -1065,7 +1049,7 @@ F64Kilobytes PeriodicRecording::getPeriodMean(const MemStatHandle& stat, size_t return getPeriodMean(static_cast<const StatType<MemAccumulator>&>(stat), num_periods); } -F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType<MemAccumulator>& stat, size_t num_periods /*= S32_MAX*/ ) +F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType<MemAccumulator>& stat, size_t num_periods /*= std::numeric_limits<size_t>::max()*/ ) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -1074,7 +1058,7 @@ F64Kilobytes PeriodicRecording::getPeriodStandardDeviation( const StatType<MemAc S32 valid_period_count = 0; F64 sum_of_squares = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) diff --git a/indra/llcommon/lltracerecording.h b/indra/llcommon/lltracerecording.h index 8b56721f42..a6b1a67d02 100644 --- a/indra/llcommon/lltracerecording.h +++ b/indra/llcommon/lltracerecording.h @@ -33,6 +33,7 @@ #include "lltimer.h" #include "lltraceaccumulators.h" #include "llpointer.h" +#include <limits> class LLStopWatchControlsMixinCommon { @@ -330,7 +331,7 @@ namespace LLTrace : public LLStopWatchControlsMixin<PeriodicRecording> { public: - PeriodicRecording(S32 num_periods, EPlayState state = STOPPED); + PeriodicRecording(size_t num_periods, EPlayState state = STOPPED); ~PeriodicRecording(); void nextPeriod(); @@ -353,7 +354,7 @@ namespace LLTrace Recording snapshotCurRecording() const; template <typename T> - auto getSampleCount(const StatType<T>& stat, size_t num_periods = S32_MAX) + auto getSampleCount(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); @@ -373,14 +374,14 @@ namespace LLTrace // catch all for stats that have a defined sum template <typename T> - typename T::value_t getPeriodMin(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename T::value_t getPeriodMin(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; typename T::value_t min_val(std::numeric_limits<typename T::value_t>::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -396,39 +397,39 @@ namespace LLTrace } template<typename T> - T getPeriodMin(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMin(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMin(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMin(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodMin(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMin(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMin(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMin(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMin(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodMin(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMin(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMin(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodMin(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMinPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMinPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes<typename T::value_t>::fractional_t min_val(std::numeric_limits<F64>::max()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); min_val = llmin(min_val, recording.getPerSec(stat)); @@ -437,7 +438,7 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMinPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMinPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMinPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); @@ -449,14 +450,14 @@ namespace LLTrace // catch all for stats that have a defined sum template <typename T> - typename T::value_t getPeriodMax(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename T::value_t getPeriodMax(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); bool has_value = false; typename T::value_t max_val(std::numeric_limits<typename T::value_t>::min()); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.hasValue(stat)) @@ -472,39 +473,39 @@ namespace LLTrace } template<typename T> - T getPeriodMax(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMax(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMax(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodMax(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMax(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - T getPeriodMax(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + T getPeriodMax(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return T(getPeriodMax(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMax(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMax(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodMax(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMaxPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMaxPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); F64 max_val = std::numeric_limits<F64>::min(); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); max_val = llmax(max_val, recording.getPerSec(stat)); @@ -513,7 +514,7 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMaxPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMaxPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMaxPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); @@ -525,14 +526,14 @@ namespace LLTrace // catch all for stats that have a defined sum template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMean(const StatType<T >& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMean(const StatType<T >& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes<typename T::value_t>::fractional_t mean(0); - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -546,39 +547,39 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMean(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMean(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMean(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMean(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMean(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMean(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMean(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMean(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodMean(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMean(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMean(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMean(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMean(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodMean(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodMean(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodMean(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodMean(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMeanPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMeanPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; num_periods = llmin(num_periods, getNumRecordedPeriods()); typename RelatedTypes<typename T::value_t>::fractional_t mean = 0; - for (S32 i = 1; i <= num_periods; i++) + for (size_t i = 1; i <= num_periods; i++) { Recording& recording = getPrevRecording(i); if (recording.getDuration() > (F32Seconds)0.f) @@ -593,64 +594,64 @@ namespace LLTrace } template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMeanPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodMeanPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodMeanPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); } - F64 getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodMedian( const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); - template <typename T> - typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMedianPerSec(const StatType<T>& stat, size_t num_periods = S32_MAX) - { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; - num_periods = llmin(num_periods, getNumRecordedPeriods()); - - std::vector <typename RelatedTypes<typename T::value_t>::fractional_t> buf; - for (S32 i = 1; i <= num_periods; i++) - { - Recording& recording = getPrevRecording(i); - if (recording.getDuration() > (F32Seconds)0.f) - { - buf.push_back(recording.getPerSec(stat)); - } - } - std::sort(buf.begin(), buf.end()); - - return typename RelatedTypes<T>::fractional_t((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); - } - - template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodMedianPerSec(const CountStatHandle<T>& stat, size_t num_periods = S32_MAX) - { - LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; - return typename RelatedTypes<T>::fractional_t(getPeriodMedianPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); - } + template <typename T> + typename RelatedTypes<typename T::value_t>::fractional_t getPeriodMedianPerSec(const StatType<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) + { + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + num_periods = llmin(num_periods, getNumRecordedPeriods()); + + std::vector <typename RelatedTypes<typename T::value_t>::fractional_t> buf; + for (size_t i = 1; i <= num_periods; i++) + { + Recording& recording = getPrevRecording(i); + if (recording.getDuration() > (F32Seconds)0.f) + { + buf.push_back(recording.getPerSec(stat)); + } + } + std::sort(buf.begin(), buf.end()); + + return typename RelatedTypes<T>::fractional_t((buf.size() % 2 == 0) ? (buf[buf.size() / 2 - 1] + buf[buf.size() / 2]) / 2 : buf[buf.size() / 2]); + } + + template<typename T> + typename RelatedTypes<T>::fractional_t getPeriodMedianPerSec(const CountStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) + { + LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; + return typename RelatedTypes<T>::fractional_t(getPeriodMedianPerSec(static_cast<const StatType<CountAccumulator>&>(stat), num_periods)); + } // // PERIODIC STANDARD DEVIATION // - F64 getPeriodStandardDeviation(const StatType<SampleAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodStandardDeviation(const StatType<SampleAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const SampleStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const SampleStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodStandardDeviation(static_cast<const StatType<SampleAccumulator>&>(stat), num_periods)); } - F64 getPeriodStandardDeviation(const StatType<EventAccumulator>& stat, size_t num_periods = S32_MAX); + F64 getPeriodStandardDeviation(const StatType<EventAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); template<typename T> - typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const EventStatHandle<T>& stat, size_t num_periods = S32_MAX) + typename RelatedTypes<T>::fractional_t getPeriodStandardDeviation(const EventStatHandle<T>& stat, size_t num_periods = std::numeric_limits<size_t>::max()) { LL_PROFILE_ZONE_SCOPED_CATEGORY_STATS; return typename RelatedTypes<T>::fractional_t(getPeriodStandardDeviation(static_cast<const StatType<EventAccumulator>&>(stat), num_periods)); } - F64Kilobytes getPeriodStandardDeviation(const StatType<MemAccumulator>& stat, size_t num_periods = S32_MAX); - F64Kilobytes getPeriodStandardDeviation(const MemStatHandle& stat, size_t num_periods = S32_MAX); + F64Kilobytes getPeriodStandardDeviation(const StatType<MemAccumulator>& stat, size_t num_periods = std::numeric_limits<size_t>::max()); + F64Kilobytes getPeriodStandardDeviation(const MemStatHandle& stat, size_t num_periods = std::numeric_limits<size_t>::max()); private: // implementation for LLStopWatchControlsMixin @@ -659,6 +660,35 @@ namespace LLTrace /*virtual*/ void handleReset(); /*virtual*/ void handleSplitTo(PeriodicRecording& other); + // helper methods for wraparound ring-buffer arithmetic + inline + size_t wrapi(size_t i) const + { + return i % mRecordingPeriods.size(); + } + + inline + size_t nexti(size_t i, size_t offset=1) const + { + return wrapi(i + offset); + } + + inline + size_t previ(size_t i, size_t offset=1) const + { + auto num_periods = mRecordingPeriods.size(); + // constrain offset + offset = llclamp(offset, 0, num_periods - 1); + // add size() so expression can't go (unsigned) "negative" + return wrapi(i + num_periods - offset); + } + + inline + void inci(size_t& i, size_t offset=1) const + { + i = nexti(i, offset); + } + private: std::vector<Recording> mRecordingPeriods; const bool mAutoResize; diff --git a/indra/llcommon/lltracethreadrecorder.cpp b/indra/llcommon/lltracethreadrecorder.cpp index 4028a5ce97..282c454a2a 100644 --- a/indra/llcommon/lltracethreadrecorder.cpp +++ b/indra/llcommon/lltracethreadrecorder.cpp @@ -284,13 +284,11 @@ void ThreadRecorder::pullFromChildren() AccumulatorBufferGroup& target_recording_buffers = mActiveRecordings.back()->mPartialRecording; target_recording_buffers.sync(); - for (child_thread_recorder_list_t::iterator it = mChildThreadRecorders.begin(), end_it = mChildThreadRecorders.end(); - it != end_it; - ++it) - { LLMutexLock lock(&(*it)->mSharedRecordingMutex); + for (LLTrace::ThreadRecorder* rec : mChildThreadRecorders) + { LLMutexLock lock(&(rec->mSharedRecordingMutex)); - target_recording_buffers.merge((*it)->mSharedRecordingBuffers); - (*it)->mSharedRecordingBuffers.reset(); + target_recording_buffers.merge(rec->mSharedRecordingBuffers); + rec->mSharedRecordingBuffers.reset(); } } #endif diff --git a/indra/llcommon/lluri.cpp b/indra/llcommon/lluri.cpp index 22711a83d2..4fb92a8f3e 100644 --- a/indra/llcommon/lluri.cpp +++ b/indra/llcommon/lluri.cpp @@ -663,9 +663,9 @@ LLSD LLURI::pathArray() const tokenizer::iterator end = tokens.end(); LLSD params; - for ( ; it != end; ++it) + for (const std::string& str : tokens) { - params.append(*it); + params.append(str); } return params; } diff --git a/indra/llcommon/lluuid.cpp b/indra/llcommon/lluuid.cpp index 6f9e09a587..5655e8e2f2 100644 --- a/indra/llcommon/lluuid.cpp +++ b/indra/llcommon/lluuid.cpp @@ -40,11 +40,12 @@ #include "lluuid.h" #include "llerror.h" #include "llrand.h" -#include "llmd5.h" #include "llstring.h" #include "lltimer.h" #include "llthread.h" #include "llmutex.h" +#include "llmd5.h" +#include "hbxxh.h" const LLUUID LLUUID::null; const LLTransactionID LLTransactionID::tnull; @@ -400,6 +401,9 @@ LLUUID LLUUID::operator^(const LLUUID& rhs) const return id; } +// WARNING: this algorithm SHALL NOT be changed. It is also used by the server +// and plays a role in some assets validation (e.g. clothing items). Changing +// it would cause invalid assets. void LLUUID::combine(const LLUUID& other, LLUUID& result) const { LLMD5 md5_uuid; @@ -857,17 +861,12 @@ void LLUUID::generate() tmp >>= 8; mData[8] = (unsigned char) tmp; - LLMD5 md5_uuid; - - md5_uuid.update(mData,16); - md5_uuid.finalize(); - md5_uuid.raw_digest(mData); + HBXXH128::digest(*this, (const void*)mData, 16); } void LLUUID::generate(const std::string& hash_string) { - LLMD5 md5_uuid((U8*)hash_string.c_str()); - md5_uuid.raw_digest(mData); + HBXXH128::digest(*this, hash_string); } U32 LLUUID::getRandomSeed() @@ -885,13 +884,8 @@ U32 LLUUID::getRandomSeed() seed[7]=(unsigned char)(pid); getSystemTime((uuid_time_t *)(&seed[8])); - LLMD5 md5_seed; - - md5_seed.update(seed,16); - md5_seed.finalize(); - md5_seed.raw_digest(seed); - - return(*(U32 *)seed); + U64 seed64 = HBXXH64::digest((const void*)seed, 16); + return U32(seed64) ^ U32(seed64 >> 32); } BOOL LLUUID::parseUUID(const std::string& buf, LLUUID* value) diff --git a/indra/llcommon/lluuid.h b/indra/llcommon/lluuid.h index c139c4eb4e..80597fa186 100644 --- a/indra/llcommon/lluuid.h +++ b/indra/llcommon/lluuid.h @@ -116,6 +116,14 @@ public: U16 getCRC16() const; U32 getCRC32() const; + // Returns a 64 bits digest of the UUID, by XORing its two 64 bits long + // words. HB + inline U64 getDigest64() const + { + U64* tmp = (U64*)mData; + return tmp[0] ^ tmp[1]; + } + static BOOL validate(const std::string& in_string); // Validate that the UUID string is legal. static const LLUUID null; @@ -165,36 +173,22 @@ public: LLAssetID makeAssetID(const LLUUID& session) const; }; -// Generate a hash of an LLUUID object using the boost hash templates. -template <> -struct boost::hash<LLUUID> -{ - typedef LLUUID argument_type; - typedef std::size_t result_type; - result_type operator()(argument_type const& s) const - { - result_type seed(0); - - for (S32 i = 0; i < UUID_BYTES; ++i) - { - boost::hash_combine(seed, s.mData[i]); - } - - return seed; - } -}; - -// Adapt boost hash to std hash +// std::hash implementation for LLUUID namespace std { - template<> struct hash<LLUUID> - { - std::size_t operator()(LLUUID const& s) const noexcept - { - return boost::hash<LLUUID>()(s); - } - }; + template<> struct hash<LLUUID> + { + inline size_t operator()(const LLUUID& id) const noexcept + { + return (size_t)id.getDigest64(); + } + }; } -#endif +// For use with boost containers. +inline size_t hash_value(const LLUUID& id) noexcept +{ + return (size_t)id.getDigest64(); +} +#endif // LL_LLUUID_H diff --git a/indra/llcommon/llworkerthread.cpp b/indra/llcommon/llworkerthread.cpp index 97838e296e..e5eda1eac7 100644 --- a/indra/llcommon/llworkerthread.cpp +++ b/indra/llcommon/llworkerthread.cpp @@ -69,11 +69,11 @@ void LLWorkerThread::clearDeleteList() << " entries in delete list." << LL_ENDL; mDeleteMutex->lock(); - for (delete_list_t::iterator iter = mDeleteList.begin(); iter != mDeleteList.end(); ++iter) + for (LLWorkerClass* worker : mDeleteList) { - (*iter)->mRequestHandle = LLWorkerThread::nullHandle(); - (*iter)->clearFlags(LLWorkerClass::WCF_HAVE_WORK); - delete *iter ; + worker->mRequestHandle = LLWorkerThread::nullHandle(); + worker->clearFlags(LLWorkerClass::WCF_HAVE_WORK); + delete worker; } mDeleteList.clear() ; mDeleteMutex->unlock() ; @@ -108,15 +108,12 @@ size_t LLWorkerThread::update(F32 max_time_ms) } mDeleteMutex->unlock(); // abort and delete after releasing mutex - for (std::vector<LLWorkerClass*>::iterator iter = abort_list.begin(); - iter != abort_list.end(); ++iter) + for (LLWorkerClass* worker : abort_list) { - (*iter)->abortWork(false); + worker->abortWork(false); } - for (std::vector<LLWorkerClass*>::iterator iter = delete_list.begin(); - iter != delete_list.end(); ++iter) + for (LLWorkerClass* worker : delete_list) { - LLWorkerClass* worker = *iter; if (worker->mRequestHandle) { // Finished but not completed @@ -124,7 +121,7 @@ size_t LLWorkerThread::update(F32 max_time_ms) worker->mRequestHandle = LLWorkerThread::nullHandle(); worker->clearFlags(LLWorkerClass::WCF_HAVE_WORK); } - delete *iter; + delete worker; } // delete and aborted entries mean there's still work to do res += delete_list.size() + abort_list.size(); diff --git a/indra/llcommon/tests/lleventdispatcher_test.cpp b/indra/llcommon/tests/lleventdispatcher_test.cpp index 966dc2c5aa..466f11f52a 100644 --- a/indra/llcommon/tests/lleventdispatcher_test.cpp +++ b/indra/llcommon/tests/lleventdispatcher_test.cpp @@ -345,7 +345,7 @@ namespace tut lleventdispatcher_data(): work("test dispatcher", "op"), // map {d=double, array=[3 elements]} - required(LLSDMap("d", LLSD::Real(0))("array", LLSDArray(LLSD())(LLSD())(LLSD()))), + required(LLSDMap("d", LLSD::Real(0))("array", llsd::array(LLSD(), LLSD(), LLSD()))), // first several params are required, last couple optional partial_offset(3) { @@ -434,8 +434,8 @@ namespace tut // freena(), methodna(), cmethodna(), smethodna() all take same param list. // Same for freenb() et al. - params = LLSDMap("a", LLSDArray("b")("i")("f")("d")("cp")) - ("b", LLSDArray("s")("uuid")("date")("uri")("bin")); + params = LLSDMap("a", llsd::array("b", "i", "f", "d", "cp")) + ("b", llsd::array("s", "uuid", "date", "uri", "bin")); debug("params:\n", params, "\n" "params[\"a\"]:\n", @@ -452,12 +452,12 @@ namespace tut // LLDate values are, as long as they're different from the // LLUUID() and LLDate() default values so inspect() will report // them. - dft_array_full = LLSDMap("a", LLSDArray(true)(17)(3.14)(123456.78)("classic")) - ("b", LLSDArray("string") - (LLUUID::generateNewID()) - (LLDate::now()) - (LLURI("http://www.ietf.org/rfc/rfc3986.txt")) - (binary)); + dft_array_full = LLSDMap("a", llsd::array(true, 17, 3.14, 123456.78, "classic")) + ("b", llsd::array("string", + LLUUID::generateNewID(), + LLDate::now(), + LLURI("http://www.ietf.org/rfc/rfc3986.txt"), + binary)); debug("dft_array_full:\n", dft_array_full); // Partial defaults arrays. @@ -723,7 +723,7 @@ namespace tut { set_test_name("map-style registration with non-array params"); // Pass "param names" as scalar or as map - LLSD attempts(LLSDArray(17)(LLSDMap("pi", 3.14)("two", 2))); + LLSD attempts(llsd::array(17, LLSDMap("pi", 3.14)("two", 2))); foreach(LLSD ae, inArray(attempts)) { std::string threw = catch_what<std::exception>([this, &ae](){ @@ -738,7 +738,7 @@ namespace tut { set_test_name("map-style registration with badly-formed defaults"); std::string threw = catch_what<std::exception>([this](){ - work.add("freena_err", "freena", freena, LLSDArray("a")("b"), 17); + work.add("freena_err", "freena", freena, llsd::array("a", "b"), 17); }); ensure_has(threw, "must be a map or an array"); } @@ -749,8 +749,8 @@ namespace tut set_test_name("map-style registration with too many array defaults"); std::string threw = catch_what<std::exception>([this](){ work.add("freena_err", "freena", freena, - LLSDArray("a")("b"), - LLSDArray(17)(0.9)("gack")); + llsd::array("a", "b"), + llsd::array(17, 0.9, "gack")); }); ensure_has(threw, "shorter than"); } @@ -761,7 +761,7 @@ namespace tut set_test_name("map-style registration with too many map defaults"); std::string threw = catch_what<std::exception>([this](){ work.add("freena_err", "freena", freena, - LLSDArray("a")("b"), + llsd::array("a", "b"), LLSDMap("b", 17)("foo", 3.14)("bar", "sinister")); }); ensure_has(threw, "nonexistent params"); @@ -798,7 +798,7 @@ namespace tut void object::test<8>() { set_test_name("query Callables with/out required params"); - LLSD names(LLSDArray("free1")("Dmethod1")("Dcmethod1")("method1")); + LLSD names(llsd::array("free1", "Dmethod1", "Dcmethod1", "method1")); foreach(LLSD nm, inArray(names)) { LLSD metadata(getMetadata(nm)); @@ -821,13 +821,13 @@ namespace tut { set_test_name("query array-style functions/methods"); // Associate each registered name with expected arity. - LLSD expected(LLSDArray - (LLSDArray - (0)(LLSDArray("free0_array")("smethod0_array")("method0_array"))) - (LLSDArray - (5)(LLSDArray("freena_array")("smethodna_array")("methodna_array"))) - (LLSDArray - (5)(LLSDArray("freenb_array")("smethodnb_array")("methodnb_array")))); + LLSD expected(llsd::array + (llsd::array + (0, llsd::array("free0_array", "smethod0_array", "method0_array")), + llsd::array + (5, llsd::array("freena_array", "smethodna_array", "methodna_array")), + llsd::array + (5, llsd::array("freenb_array", "smethodnb_array", "methodnb_array")))); foreach(LLSD ae, inArray(expected)) { LLSD::Integer arity(ae[0].asInteger()); @@ -853,7 +853,7 @@ namespace tut set_test_name("query map-style no-params functions/methods"); // - (Free function | non-static method), map style, no params (ergo // no defaults) - LLSD names(LLSDArray("free0_map")("smethod0_map")("method0_map")); + LLSD names(llsd::array("free0_map", "smethod0_map", "method0_map")); foreach(LLSD nm, inArray(names)) { LLSD metadata(getMetadata(nm)); @@ -877,13 +877,13 @@ namespace tut // there should (!) be no difference beween array defaults and map // defaults. Verify, so we can ignore the distinction for all other // tests. - LLSD equivalences(LLSDArray - (LLSDArray("freena_map_adft")("freena_map_mdft")) - (LLSDArray("freenb_map_adft")("freenb_map_mdft")) - (LLSDArray("smethodna_map_adft")("smethodna_map_mdft")) - (LLSDArray("smethodnb_map_adft")("smethodnb_map_mdft")) - (LLSDArray("methodna_map_adft")("methodna_map_mdft")) - (LLSDArray("methodnb_map_adft")("methodnb_map_mdft"))); + LLSD equivalences(llsd::array + (llsd::array("freena_map_adft", "freena_map_mdft"), + llsd::array("freenb_map_adft", "freenb_map_mdft"), + llsd::array("smethodna_map_adft", "smethodna_map_mdft"), + llsd::array("smethodnb_map_adft", "smethodnb_map_mdft"), + llsd::array("methodna_map_adft", "methodna_map_mdft"), + llsd::array("methodnb_map_adft", "methodnb_map_mdft"))); foreach(LLSD eq, inArray(equivalences)) { LLSD adft(eq[0]); @@ -953,42 +953,42 @@ namespace tut debug("skipreq:\n", skipreq); - LLSD groups(LLSDArray // array of groups + LLSD groups(llsd::array // array of groups - (LLSDArray // group - (LLSDArray("freena_map_allreq")("smethodna_map_allreq")("methodna_map_allreq")) - (LLSDArray(allreq["a"])(LLSD()))) // required, optional + (llsd::array // group + (llsd::array("freena_map_allreq", "smethodna_map_allreq", "methodna_map_allreq"), + llsd::array(allreq["a"], LLSD())), // required, optional - (LLSDArray // group - (LLSDArray("freenb_map_allreq")("smethodnb_map_allreq")("methodnb_map_allreq")) - (LLSDArray(allreq["b"])(LLSD()))) // required, optional + llsd::array // group + (llsd::array("freenb_map_allreq", "smethodnb_map_allreq", "methodnb_map_allreq"), + llsd::array(allreq["b"], LLSD())), // required, optional - (LLSDArray // group - (LLSDArray("freena_map_leftreq")("smethodna_map_leftreq")("methodna_map_leftreq")) - (LLSDArray(leftreq["a"])(rightdft["a"]))) // required, optional + llsd::array // group + (llsd::array("freena_map_leftreq", "smethodna_map_leftreq", "methodna_map_leftreq"), + llsd::array(leftreq["a"], rightdft["a"])), // required, optional - (LLSDArray // group - (LLSDArray("freenb_map_leftreq")("smethodnb_map_leftreq")("methodnb_map_leftreq")) - (LLSDArray(leftreq["b"])(rightdft["b"]))) // required, optional + llsd::array // group + (llsd::array("freenb_map_leftreq", "smethodnb_map_leftreq", "methodnb_map_leftreq"), + llsd::array(leftreq["b"], rightdft["b"])), // required, optional - (LLSDArray // group - (LLSDArray("freena_map_skipreq")("smethodna_map_skipreq")("methodna_map_skipreq")) - (LLSDArray(skipreq["a"])(dft_map_partial["a"]))) // required, optional + llsd::array // group + (llsd::array("freena_map_skipreq", "smethodna_map_skipreq", "methodna_map_skipreq"), + llsd::array(skipreq["a"], dft_map_partial["a"])), // required, optional - (LLSDArray // group - (LLSDArray("freenb_map_skipreq")("smethodnb_map_skipreq")("methodnb_map_skipreq")) - (LLSDArray(skipreq["b"])(dft_map_partial["b"]))) // required, optional + llsd::array // group + (llsd::array("freenb_map_skipreq", "smethodnb_map_skipreq", "methodnb_map_skipreq"), + llsd::array(skipreq["b"], dft_map_partial["b"])), // required, optional - // We only need mention the full-map-defaults ("_mdft" suffix) - // registrations, having established their equivalence with the - // full-array-defaults ("_adft" suffix) registrations in another test. - (LLSDArray // group - (LLSDArray("freena_map_mdft")("smethodna_map_mdft")("methodna_map_mdft")) - (LLSDArray(LLSD::emptyMap())(dft_map_full["a"]))) // required, optional + // We only need mention the full-map-defaults ("_mdft" suffix) + // registrations, having established their equivalence with the + // full-array-defaults ("_adft" suffix) registrations in another test. + llsd::array // group + (llsd::array("freena_map_mdft", "smethodna_map_mdft", "methodna_map_mdft"), + llsd::array(LLSD::emptyMap(), dft_map_full["a"])), // required, optional - (LLSDArray // group - (LLSDArray("freenb_map_mdft")("smethodnb_map_mdft")("methodnb_map_mdft")) - (LLSDArray(LLSD::emptyMap())(dft_map_full["b"])))); // required, optional + llsd::array // group + (llsd::array("freenb_map_mdft", "smethodnb_map_mdft", "methodnb_map_mdft"), + llsd::array(LLSD::emptyMap(), dft_map_full["b"])))); // required, optional foreach(LLSD grp, inArray(groups)) { @@ -1077,7 +1077,7 @@ namespace tut // with 'required'. LLSD answer(42); // LLSD value matching 'required' according to llsd_matches() rules. - LLSD matching(LLSDMap("d", 3.14)("array", LLSDArray("answer")(true)(answer))); + LLSD matching(LLSDMap("d", 3.14)("array", llsd::array("answer", true, answer))); // Okay, walk through 'tests'. foreach(const CallablesTriple& tr, tests) { @@ -1114,17 +1114,17 @@ namespace tut call_exc("free0_map", 17, map_exc); // Passing an array to a map-style function works now! No longer an // error case! -// call_exc("free0_map", LLSDArray("a")("b"), map_exc); +// call_exc("free0_map", llsd::array("a", "b"), map_exc); } template<> template<> void object::test<18>() { set_test_name("call no-args functions"); - LLSD names(LLSDArray - ("free0_array")("free0_map") - ("smethod0_array")("smethod0_map") - ("method0_array")("method0_map")); + LLSD names(llsd::array + ("free0_array", "free0_map", + "smethod0_array", "smethod0_map", + "method0_array", "method0_map")); foreach(LLSD name, inArray(names)) { // Look up the Vars instance for this function. @@ -1142,10 +1142,10 @@ namespace tut } // Break out this data because we use it in a couple different tests. - LLSD array_funcs(LLSDArray - (LLSDMap("a", "freena_array") ("b", "freenb_array")) - (LLSDMap("a", "smethodna_array")("b", "smethodnb_array")) - (LLSDMap("a", "methodna_array") ("b", "methodnb_array"))); + LLSD array_funcs(llsd::array + (LLSDMap("a", "freena_array") ("b", "freenb_array"), + LLSDMap("a", "smethodna_array")("b", "smethodnb_array"), + LLSDMap("a", "methodna_array") ("b", "methodnb_array"))); template<> template<> void object::test<19>() @@ -1153,7 +1153,7 @@ namespace tut set_test_name("call array-style functions with too-short arrays"); // Could have two different too-short arrays, one for *na and one for // *nb, but since they both take 5 params... - LLSD tooshort(LLSDArray("this")("array")("too")("short")); + LLSD tooshort(llsd::array("this", "array", "too", "short")); foreach(const LLSD& funcsab, inArray(array_funcs)) { foreach(const llsd::MapEntry& e, inMap(funcsab)) @@ -1172,12 +1172,12 @@ namespace tut { binary.push_back((U8)h); } - LLSD args(LLSDMap("a", LLSDArray(true)(17)(3.14)(123.456)("char*")) - ("b", LLSDArray("string") - (LLUUID("01234567-89ab-cdef-0123-456789abcdef")) - (LLDate("2011-02-03T15:07:00Z")) - (LLURI("http://secondlife.com")) - (binary))); + LLSD args(LLSDMap("a", llsd::array(true, 17, 3.14, 123.456, "char*")) + ("b", llsd::array("string", + LLUUID("01234567-89ab-cdef-0123-456789abcdef"), + LLDate("2011-02-03T15:07:00Z"), + LLURI("http://secondlife.com"), + binary))); LLSD argsplus(args); argsplus["a"].append("bogus"); argsplus["b"].append("bogus"); @@ -1191,7 +1191,7 @@ namespace tut debug("expect: ", expect); // Use substantially the same logic for args and argsplus - LLSD argsarrays(LLSDArray(args)(argsplus)); + LLSD argsarrays(llsd::array(args, argsplus)); // So i==0 selects 'args', i==1 selects argsplus for (LLSD::Integer i(0), iend(argsarrays.size()); i < iend; ++i) { @@ -1236,8 +1236,8 @@ namespace tut set_test_name("call map-style functions with (full | oversized) (arrays | maps)"); const char binary[] = "\x99\x88\x77\x66\x55"; LLSD array_full(LLSDMap - ("a", LLSDArray(false)(255)(98.6)(1024.5)("pointer")) - ("b", LLSDArray("object")(LLUUID::generateNewID())(LLDate::now())(LLURI("http://wiki.lindenlab.com/wiki"))(LLSD::Binary(boost::begin(binary), boost::end(binary))))); + ("a", llsd::array(false, 255, 98.6, 1024.5, "pointer")) + ("b", llsd::array("object", LLUUID::generateNewID(), LLDate::now(), LLURI("http://wiki.lindenlab.com/wiki"), LLSD::Binary(boost::begin(binary), boost::end(binary))))); LLSD array_overfull(array_full); foreach(LLSD::String a, ab) { @@ -1280,20 +1280,20 @@ namespace tut // parameter defaults should make NO DIFFERENCE WHATSOEVER. Every call // should pass all params. LLSD names(LLSDMap - ("a", LLSDArray - ("freena_map_allreq") ("smethodna_map_allreq") ("methodna_map_allreq") - ("freena_map_leftreq")("smethodna_map_leftreq")("methodna_map_leftreq") - ("freena_map_skipreq")("smethodna_map_skipreq")("methodna_map_skipreq") - ("freena_map_adft") ("smethodna_map_adft") ("methodna_map_adft") - ("freena_map_mdft") ("smethodna_map_mdft") ("methodna_map_mdft")) - ("b", LLSDArray - ("freenb_map_allreq") ("smethodnb_map_allreq") ("methodnb_map_allreq") - ("freenb_map_leftreq")("smethodnb_map_leftreq")("methodnb_map_leftreq") - ("freenb_map_skipreq")("smethodnb_map_skipreq")("methodnb_map_skipreq") - ("freenb_map_adft") ("smethodnb_map_adft") ("methodnb_map_adft") - ("freenb_map_mdft") ("smethodnb_map_mdft") ("methodnb_map_mdft"))); + ("a", llsd::array + ("freena_map_allreq", "smethodna_map_allreq", "methodna_map_allreq", + "freena_map_leftreq", "smethodna_map_leftreq", "methodna_map_leftreq", + "freena_map_skipreq", "smethodna_map_skipreq", "methodna_map_skipreq", + "freena_map_adft", "smethodna_map_adft", "methodna_map_adft", + "freena_map_mdft", "smethodna_map_mdft", "methodna_map_mdft")) + ("b", llsd::array + ("freenb_map_allreq", "smethodnb_map_allreq", "methodnb_map_allreq", + "freenb_map_leftreq", "smethodnb_map_leftreq", "methodnb_map_leftreq", + "freenb_map_skipreq", "smethodnb_map_skipreq", "methodnb_map_skipreq", + "freenb_map_adft", "smethodnb_map_adft", "methodnb_map_adft", + "freenb_map_mdft", "smethodnb_map_mdft", "methodnb_map_mdft"))); // Treat (full | overfull) (array | map) the same. - LLSD argssets(LLSDArray(array_full)(array_overfull)(map_full)(map_overfull)); + LLSD argssets(llsd::array(array_full, array_overfull, map_full, map_overfull)); foreach(const LLSD& args, inArray(argssets)) { foreach(LLSD::String a, ab) diff --git a/indra/llcommon/tests/llleap_test.cpp b/indra/llcommon/tests/llleap_test.cpp index 7ee36a9ea6..3ae48a2532 100644 --- a/indra/llcommon/tests/llleap_test.cpp +++ b/indra/llcommon/tests/llleap_test.cpp @@ -109,7 +109,12 @@ namespace tut "import os\n" "import sys\n" "\n" - "from llbase import llsd\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n" "\n" "class ProtocolError(Exception):\n" " def __init__(self, msg, data):\n" @@ -120,26 +125,26 @@ namespace tut " pass\n" "\n" "def get():\n" - " hdr = ''\n" - " while ':' not in hdr and len(hdr) < 20:\n" - " hdr += sys.stdin.read(1)\n" + " hdr = []\n" + " while b':' not in hdr and len(hdr) < 20:\n" + " hdr.append(sys.stdin.buffer.read(1))\n" " if not hdr:\n" " sys.exit(0)\n" - " if not hdr.endswith(':'):\n" + " if not hdr[-1] == b':':\n" " raise ProtocolError('Expected len:data, got %r' % hdr, hdr)\n" " try:\n" - " length = int(hdr[:-1])\n" + " length = int(b''.join(hdr[:-1]))\n" " except ValueError:\n" " raise ProtocolError('Non-numeric len %r' % hdr[:-1], hdr[:-1])\n" " parts = []\n" " received = 0\n" " while received < length:\n" - " parts.append(sys.stdin.read(length - received))\n" + " parts.append(sys.stdin.buffer.read(length - received))\n" " received += len(parts[-1])\n" - " data = ''.join(parts)\n" + " data = b''.join(parts)\n" " assert len(data) == length\n" " try:\n" - " return llsd.parse(data.encode())\n" + " return llsd.parse(data)\n" // Seems the old indra.base.llsd module didn't properly // convert IndexError (from running off end of string) to // LLSDParseError. @@ -179,11 +184,11 @@ namespace tut " return _reply\n" "\n" "def put(req):\n" - " sys.stdout.write(':'.join((str(len(req)), req)))\n" + " sys.stdout.buffer.write(b'%d:%b' % (len(req), req))\n" " sys.stdout.flush()\n" "\n" "def send(pump, data):\n" - " put(llsd.format_notation(dict(pump=pump, data=data)).decode())\n" + " put(llsd.format_notation(dict(pump=pump, data=data)))\n" "\n" "def request(pump, data):\n" " # we expect 'data' is a dict\n" diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index 999d432079..81449b4a42 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -356,7 +356,7 @@ namespace tut // Create a script file in a temporary place. NamedTempFile script("py", - "from __future__ import print_function" EOL + "from __future__ import print_function" EOL "import sys" EOL "import time" EOL EOL @@ -366,7 +366,7 @@ namespace tut "time.sleep(2)" EOL "print('stderr after wait',file=sys.stderr)" EOL "sys.stderr.flush()" EOL - ); + ); // Arrange to track the history of our interaction with child: what we // fetched, which pipe it came from, how many tries it took before we @@ -862,8 +862,8 @@ namespace tut set_test_name("'bogus' test"); CaptureLog recorder; PythonProcessLauncher py(get_test_name(), - "from __future__ import print_function\n" - "print('Hello world')\n"); + "from __future__ import print_function\n" + "print('Hello world')\n"); py.mParams.files.add(LLProcess::FileParam("bogus")); py.mPy = LLProcess::create(py.mParams); ensure("should have rejected 'bogus'", ! py.mPy); @@ -878,8 +878,8 @@ namespace tut // Replace this test with one or more real 'file' tests when we // implement 'file' support PythonProcessLauncher py(get_test_name(), - "from __future__ import print_function\n" - "print('Hello world')\n"); + "from __future__ import print_function\n" + "print('Hello world')\n"); py.mParams.files.add(LLProcess::FileParam()); py.mParams.files.add(LLProcess::FileParam("file")); py.mPy = LLProcess::create(py.mParams); @@ -894,8 +894,8 @@ namespace tut // implement 'tpipe' support CaptureLog recorder; PythonProcessLauncher py(get_test_name(), - "from __future__ import print_function\n" - "print('Hello world')\n"); + "from __future__ import print_function\n" + "print('Hello world')\n"); py.mParams.files.add(LLProcess::FileParam()); py.mParams.files.add(LLProcess::FileParam("tpipe")); py.mPy = LLProcess::create(py.mParams); @@ -912,8 +912,8 @@ namespace tut // implement 'npipe' support CaptureLog recorder; PythonProcessLauncher py(get_test_name(), - "from __future__ import print_function\n" - "print('Hello world')\n"); + "from __future__ import print_function\n" + "print('Hello world')\n"); py.mParams.files.add(LLProcess::FileParam()); py.mParams.files.add(LLProcess::FileParam()); py.mParams.files.add(LLProcess::FileParam("npipe")); @@ -989,20 +989,20 @@ namespace tut { set_test_name("get*Pipe() validation"); PythonProcessLauncher py(get_test_name(), - "from __future__ import print_function\n" - "print('this output is expected')\n"); + "from __future__ import print_function\n" + "print('this output is expected')\n"); py.mParams.files.add(LLProcess::FileParam("pipe")); // pipe for stdin py.mParams.files.add(LLProcess::FileParam()); // inherit stdout py.mParams.files.add(LLProcess::FileParam("pipe")); // pipe for stderr py.run(); TEST_getPipe(*py.mPy, getWritePipe, getOptWritePipe, - LLProcess::STDIN, // VALID - LLProcess::STDOUT, // NOPIPE - LLProcess::STDERR); // BADPIPE + LLProcess::STDIN, // VALID + LLProcess::STDOUT, // NOPIPE + LLProcess::STDERR); // BADPIPE TEST_getPipe(*py.mPy, getReadPipe, getOptReadPipe, - LLProcess::STDERR, // VALID - LLProcess::STDOUT, // NOPIPE - LLProcess::STDIN); // BADPIPE + LLProcess::STDERR, // VALID + LLProcess::STDOUT, // NOPIPE + LLProcess::STDIN); // BADPIPE } template<> template<> @@ -1129,8 +1129,8 @@ namespace tut { set_test_name("ReadPipe \"eof\" event"); PythonProcessLauncher py(get_test_name(), - "from __future__ import print_function\n" - "print('Hello from Python!')\n"); + "from __future__ import print_function\n" + "print('Hello from Python!')\n"); py.mParams.files.add(LLProcess::FileParam()); // stdin py.mParams.files.add(LLProcess::FileParam("pipe")); // stdout py.launch(); diff --git a/indra/llcommon/tests/llsdserialize_test.cpp b/indra/llcommon/tests/llsdserialize_test.cpp index c246f5ee56..acb2953b5b 100644 --- a/indra/llcommon/tests/llsdserialize_test.cpp +++ b/indra/llcommon/tests/llsdserialize_test.cpp @@ -46,20 +46,24 @@ typedef U32 uint32_t; #include "boost/range.hpp" #include "boost/foreach.hpp" -#include "boost/function.hpp" #include "boost/bind.hpp" #include "boost/phoenix/bind/bind_function.hpp" #include "boost/phoenix/core/argument.hpp" using namespace boost::phoenix; -#include "../llsd.h" -#include "../llsdserialize.h" +#include "llsd.h" +#include "llsdserialize.h" #include "llsdutil.h" -#include "../llformat.h" +#include "llformat.h" +#include "llmemorystream.h" #include "../test/lltut.h" #include "../test/namedtempfile.h" #include "stringize.h" +#include <functional> + +typedef std::function<void(const LLSD& data, std::ostream& str)> FormatterFunction; +typedef std::function<bool(std::istream& istr, LLSD& data, llssize max_bytes)> ParserFunction; std::vector<U8> string_to_vector(const std::string& str) { @@ -112,7 +116,7 @@ namespace tut mSD = LLUUID::null; expected = "<llsd><uuid /></llsd>\n"; xml_test("null uuid", expected); - + mSD = LLUUID("c96f9b1e-f589-4100-9774-d98643ce0bed"); expected = "<llsd><uuid>c96f9b1e-f589-4100-9774-d98643ce0bed</uuid></llsd>\n"; xml_test("uuid", expected); @@ -136,7 +140,7 @@ namespace tut expected = "<llsd><binary encoding=\"base64\">aGVsbG8=</binary></llsd>\n"; xml_test("binary", expected); } - + template<> template<> void sd_xml_object::test<2>() { @@ -225,7 +229,7 @@ namespace tut expected = "<llsd><map><key>baz</key><undef /><key>foo</key><string>bar</string></map></llsd>\n"; xml_test("2 element map", expected); } - + template<> template<> void sd_xml_object::test<6>() { @@ -241,7 +245,7 @@ namespace tut expected = "<llsd><binary encoding=\"base64\">Nnw2fGFzZGZoYXBweWJveHw2MGU0NGVjNS0zMDVjLTQzYzItOWExOS1iNGI4OWIxYWUyYTZ8NjBlNDRlYzUtMzA1Yy00M2MyLTlhMTktYjRiODliMWFlMmE2fDYwZTQ0ZWM1LTMwNWMtNDNjMi05YTE5LWI0Yjg5YjFhZTJhNnwwMDAwMDAwMC0wMDAwLTAwMDAtMDAwMC0wMDAwMDAwMDAwMDB8N2ZmZmZmZmZ8N2ZmZmZmZmZ8MHwwfDgyMDAwfDQ1MGZlMzk0LTI5MDQtYzlhZC0yMTRjLWEwN2ViN2ZlZWMyOXwoTm8gRGVzY3JpcHRpb24pfDB8MTB8MA==</binary></llsd>\n"; xml_test("binary", expected); } - + class TestLLSDSerializeData { public: @@ -250,9 +254,34 @@ namespace tut void doRoundTripTests(const std::string&); void checkRoundTrip(const std::string&, const LLSD& v); - - LLPointer<LLSDFormatter> mFormatter; - LLPointer<LLSDParser> mParser; + + void setFormatterParser(LLPointer<LLSDFormatter> formatter, LLPointer<LLSDParser> parser) + { + mFormatter = [formatter](const LLSD& data, std::ostream& str) + { + formatter->format(data, str); + }; + // this lambda must be mutable since otherwise the bound 'parser' + // is assumed to point to a const LLSDParser + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) mutable + { + // reset() call is needed since test code re-uses parser object + parser->reset(); + return (parser->parse(istr, data, max_bytes) > 0); + }; + } + + void setParser(bool (*parser)(LLSD&, std::istream&, llssize)) + { + // why does LLSDSerialize::deserialize() reverse the parse() params?? + mParser = [parser](std::istream& istr, LLSD& data, llssize max_bytes) + { + return parser(data, istr, max_bytes); + }; + } + + FormatterFunction mFormatter; + ParserFunction mParser; }; TestLLSDSerializeData::TestLLSDSerializeData() @@ -265,12 +294,11 @@ namespace tut void TestLLSDSerializeData::checkRoundTrip(const std::string& msg, const LLSD& v) { - std::stringstream stream; - mFormatter->format(v, stream); + std::stringstream stream; + mFormatter(v, stream); //LL_INFOS() << "checkRoundTrip: length " << stream.str().length() << LL_ENDL; LLSD w; - mParser->reset(); // reset() call is needed since test code re-uses mParser - mParser->parse(stream, w, stream.str().size()); + mParser(stream, w, stream.str().size()); try { @@ -299,52 +327,52 @@ namespace tut fillmap(root[key], width, depth - 1); } } - + void TestLLSDSerializeData::doRoundTripTests(const std::string& msg) { LLSD v; checkRoundTrip(msg + " undefined", v); - + v = true; checkRoundTrip(msg + " true bool", v); - + v = false; checkRoundTrip(msg + " false bool", v); - + v = 1; checkRoundTrip(msg + " positive int", v); - + v = 0; checkRoundTrip(msg + " zero int", v); - + v = -1; checkRoundTrip(msg + " negative int", v); - + v = 1234.5f; checkRoundTrip(msg + " positive float", v); - + v = 0.0f; checkRoundTrip(msg + " zero float", v); - + v = -1234.5f; checkRoundTrip(msg + " negative float", v); - + // FIXME: need a NaN test - + v = LLUUID::null; checkRoundTrip(msg + " null uuid", v); - + LLUUID newUUID; newUUID.generate(); v = newUUID; checkRoundTrip(msg + " new uuid", v); - + v = ""; checkRoundTrip(msg + " empty string", v); - + v = "some string"; checkRoundTrip(msg + " non-empty string", v); - + v = "Second Life is a 3-D virtual world entirely built and owned by its residents. " "Since opening to the public in 2003, it has grown explosively and today is " @@ -372,7 +400,7 @@ namespace tut for (U32 block = 0x000000; block <= 0x10ffff; block += block_size) { std::ostringstream out; - + for (U32 c = block; c < block + block_size; ++c) { if (c <= 0x000001f @@ -386,7 +414,7 @@ namespace tut if (0x00fdd0 <= c && c <= 0x00fdef) { continue; } if ((c & 0x00fffe) == 0x00fffe) { continue; } // see Unicode standard, section 15.8 - + if (c <= 0x00007f) { out << (char)(c & 0x7f); @@ -410,55 +438,55 @@ namespace tut out << (char)(0x80 | ((c >> 0) & 0x3f)); } } - + v = out.str(); std::ostringstream blockmsg; blockmsg << msg << " unicode string block 0x" << std::hex << block; checkRoundTrip(blockmsg.str(), v); } - + LLDate epoch; v = epoch; checkRoundTrip(msg + " epoch date", v); - + LLDate aDay("2002-12-07T05:07:15.00Z"); v = aDay; checkRoundTrip(msg + " date", v); - + LLURI path("http://slurl.com/secondlife/Ambleside/57/104/26/"); v = path; checkRoundTrip(msg + " url", v); - + const char source[] = "it must be a blue moon again"; std::vector<U8> data; // note, includes terminating '\0' copy(&source[0], &source[sizeof(source)], back_inserter(data)); - + v = data; checkRoundTrip(msg + " binary", v); - + v = LLSD::emptyMap(); checkRoundTrip(msg + " empty map", v); - + v = LLSD::emptyMap(); v["name"] = "luke"; //v.insert("name", "luke"); v["age"] = 3; //v.insert("age", 3); checkRoundTrip(msg + " map", v); - + v.clear(); v["a"]["1"] = true; v["b"]["0"] = false; checkRoundTrip(msg + " nested maps", v); - + v = LLSD::emptyArray(); checkRoundTrip(msg + " empty array", v); - + v = LLSD::emptyArray(); v.append("ali"); v.append(28); checkRoundTrip(msg + " array", v); - + v.clear(); v[0][0] = true; v[1][0] = false; @@ -468,7 +496,7 @@ namespace tut fillmap(v, 10, 3); // 10^6 maps checkRoundTrip(msg + " many nested maps", v); } - + typedef tut::test_group<TestLLSDSerializeData> TestLLSDSerializeGroup; typedef TestLLSDSerializeGroup::object TestLLSDSerializeObject; TestLLSDSerializeGroup gTestLLSDSerializeGroup("llsd serialization"); @@ -476,35 +504,106 @@ namespace tut template<> template<> void TestLLSDSerializeObject::test<1>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_PRETTY_BINARY), + new LLSDNotationParser()); doRoundTripTests("pretty binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<2>() { - mFormatter = new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE); - mParser = new LLSDNotationParser(); + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); doRoundTripTests("raw binary notation serialization"); } template<> template<> void TestLLSDSerializeObject::test<3>() { - mFormatter = new LLSDXMLFormatter(); - mParser = new LLSDXMLParser(); + setFormatterParser(new LLSDXMLFormatter(), new LLSDXMLParser()); doRoundTripTests("xml serialization"); } template<> template<> void TestLLSDSerializeObject::test<4>() { - mFormatter = new LLSDBinaryFormatter(); - mParser = new LLSDBinaryParser(); + setFormatterParser(new LLSDBinaryFormatter(), new LLSDBinaryParser()); doRoundTripTests("binary serialization"); } + template<> template<> + void TestLLSDSerializeObject::test<5>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_BINARY); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_BINARY)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<6>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_XML); + }; + setParser(LLSDSerialize::deserialize); + doRoundTripTests("serialize(LLSD_XML)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<7>() + { + mFormatter = [](const LLSD& sd, std::ostream& str) + { + LLSDSerialize::serialize(sd, str, LLSDSerialize::LLSD_NOTATION); + }; + setParser(LLSDSerialize::deserialize); + // In this test, serialize(LLSD_NOTATION) emits a header recognized by + // deserialize(). + doRoundTripTests("serialize(LLSD_NOTATION)"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<8>() + { + setFormatterParser(new LLSDNotationFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDNotationParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDNotationFormatter does not + // emit an llsd/notation header. + doRoundTripTests("LLSDNotationFormatter -> deserialize"); + }; + + template<> template<> + void TestLLSDSerializeObject::test<9>() + { + setFormatterParser(new LLSDXMLFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDXMLParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDXMLFormatter does not + // emit an LLSD/XML header. + doRoundTripTests("LLSDXMLFormatter -> deserialize"); + }; + +/*==========================================================================*| + // We do not expect this test to succeed. Without a header, neither + // notation LLSD nor binary LLSD reliably start with a distinct character, + // the way XML LLSD starts with '<'. By convention, we default to notation + // rather than binary. + template<> template<> + void TestLLSDSerializeObject::test<10>() + { + setFormatterParser(new LLSDBinaryFormatter(false, "", LLSDFormatter::OPTIONS_NONE), + new LLSDBinaryParser()); + setParser(LLSDSerialize::deserialize); + // This is an interesting test because LLSDBinaryFormatter does not + // emit an LLSD/Binary header. + doRoundTripTests("LLSDBinaryFormatter -> deserialize"); + }; +|*==========================================================================*/ /** * @class TestLLSDParsing @@ -555,7 +654,7 @@ namespace tut public: TestLLSDXMLParsing() {} }; - + typedef tut::test_group<TestLLSDXMLParsing> TestLLSDXMLParsingGroup; typedef TestLLSDXMLParsingGroup::object TestLLSDXMLParsingObject; TestLLSDXMLParsingGroup gTestLLSDXMLParsingGroup("llsd XML parsing"); @@ -586,8 +685,8 @@ namespace tut LLSD(), LLSDParser::PARSE_FAILURE); } - - + + template<> template<> void TestLLSDXMLParsingObject::test<2>() { @@ -596,7 +695,7 @@ namespace tut v["amy"] = 23; v["bob"] = LLSD(); v["cam"] = 1.23; - + ensureParse( "unknown data type", "<llsd><map>" @@ -607,16 +706,16 @@ namespace tut v, v.size() + 1); } - + template<> template<> void TestLLSDXMLParsingObject::test<3>() { // test handling of nested bad data - + LLSD v; v["amy"] = 23; v["cam"] = 1.23; - + ensureParse( "map with html", "<llsd><map>" @@ -626,7 +725,7 @@ namespace tut "</map></llsd>", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["cam"] = 1.23; @@ -639,7 +738,7 @@ namespace tut "</map></llsd>", v, v.size() + 1); - + v.clear(); v["amy"] = 23; v["bob"] = LLSD::emptyMap(); @@ -661,7 +760,7 @@ namespace tut v[0] = 23; v[1] = LLSD(); v[2] = 1.23; - + ensureParse( "array value of html", "<llsd><array>" @@ -671,7 +770,7 @@ namespace tut "</array></llsd>", v, v.size() + 1); - + v.clear(); v[0] = 23; v[1] = LLSD::emptyMap(); @@ -1225,7 +1324,7 @@ namespace tut vec[0] = 'a'; vec[1] = 'b'; vec[2] = 'c'; vec[3] = '3'; vec[4] = '2'; vec[5] = '1'; LLSD value = vec; - + vec.resize(11); vec[0] = 'b'; // for binary vec[5] = 'a'; vec[6] = 'b'; vec[7] = 'c'; @@ -1694,85 +1793,83 @@ namespace tut ensureBinaryAndXML("map", test); } - struct TestPythonCompatible + // helper for TestPythonCompatible + static std::string import_llsd("import os.path\n" + "import sys\n" + "try:\n" + // new freestanding llsd package + " import llsd\n" + "except ImportError:\n" + // older llbase.llsd module + " from llbase import llsd\n"); + + // helper for TestPythonCompatible + template <typename CONTENT> + void python(const std::string& desc, const CONTENT& script, int expect=0) { - TestPythonCompatible(): - // Note the peculiar insertion of __FILE__ into this string. Since - // this script is being written into a platform-dependent temp - // directory, we can't locate indra/lib/python relative to - // Python's __file__. Use __FILE__ instead, navigating relative - // to this C++ source file. Use Python raw-string syntax so - // Windows pathname backslashes won't mislead Python's string - // scanner. - import_llsd("import os.path\n" - "import sys\n" - "from llbase import llsd\n") - {} - ~TestPythonCompatible() {} + auto PYTHON(LLStringUtil::getenv("PYTHON")); + ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - std::string import_llsd; + NamedTempFile scriptfile("py", script); - template <typename CONTENT> - void python(const std::string& desc, const CONTENT& script, int expect=0) +#if LL_WINDOWS + std::string q("\""); + std::string qPYTHON(q + PYTHON + q); + std::string qscript(q + scriptfile.getName() + q); + int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); + if (rc == -1) { - auto PYTHON(LLStringUtil::getenv("PYTHON")); - ensure("Set $PYTHON to the Python interpreter", !PYTHON.empty()); - - NamedTempFile scriptfile("py", script); + char buffer[256]; + strerror_s(buffer, errno); // C++ can infer the buffer size! :-O + ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); + } + else + { + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + } -#if LL_WINDOWS - std::string q("\""); - std::string qPYTHON(q + PYTHON + q); - std::string qscript(q + scriptfile.getName() + q); - int rc = _spawnl(_P_WAIT, PYTHON.c_str(), qPYTHON.c_str(), qscript.c_str(), NULL); - if (rc == -1) - { - char buffer[256]; - strerror_s(buffer, errno); // C++ can infer the buffer size! :-O - ensure(STRINGIZE("Couldn't run Python " << desc << "script: " << buffer), false); - } - else +#else // LL_DARWIN, LL_LINUX + LLProcess::Params params; + params.executable = PYTHON; + params.args.add(scriptfile.getName()); + LLProcessPtr py(LLProcess::create(params)); + ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); + // Implementing timeout would mean messing with alarm() and + // catching SIGALRM... later maybe... + int status(0); + if (waitpid(py->getProcessID(), &status, 0) == -1) + { + int waitpid_errno(errno); + ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " + "waitpid() errno " << waitpid_errno), + waitpid_errno, ECHILD); + } + else + { + if (WIFEXITED(status)) { - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), rc, expect); + int rc(WEXITSTATUS(status)); + ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), + rc, expect); } - -#else // LL_DARWIN, LL_LINUX - LLProcess::Params params; - params.executable = PYTHON; - params.args.add(scriptfile.getName()); - LLProcessPtr py(LLProcess::create(params)); - ensure(STRINGIZE("Couldn't launch " << desc << " script"), bool(py)); - // Implementing timeout would mean messing with alarm() and - // catching SIGALRM... later maybe... - int status(0); - if (waitpid(py->getProcessID(), &status, 0) == -1) + else if (WIFSIGNALED(status)) { - int waitpid_errno(errno); - ensure_equals(STRINGIZE("Couldn't retrieve rc from " << desc << " script: " - "waitpid() errno " << waitpid_errno), - waitpid_errno, ECHILD); + ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), + false); } else { - if (WIFEXITED(status)) - { - int rc(WEXITSTATUS(status)); - ensure_equals(STRINGIZE(desc << " script terminated with rc " << rc), - rc, expect); - } - else if (WIFSIGNALED(status)) - { - ensure(STRINGIZE(desc << " script terminated by signal " << WTERMSIG(status)), - false); - } - else - { - ensure(STRINGIZE(desc << " script produced impossible status " << status), - false); - } + ensure(STRINGIZE(desc << " script produced impossible status " << status), + false); } -#endif } +#endif + } + + struct TestPythonCompatible + { + TestPythonCompatible() {} + ~TestPythonCompatible() {} }; typedef tut::test_group<TestPythonCompatible> TestPythonCompatibleGroup; @@ -1798,29 +1895,37 @@ namespace tut "print('Running on', sys.platform)\n"); } - // helper for test<3> - static void writeLLSDArray(std::ostream& out, const LLSD& array) + // helper for test<3> - test<7> + static void writeLLSDArray(const FormatterFunction& serialize, + std::ostream& out, const LLSD& array) { - BOOST_FOREACH(LLSD item, llsd::inArray(array)) + for (const LLSD& item : llsd::inArray(array)) { - LLSDSerialize::toNotation(item, out); - // It's important to separate with newlines because Python's llsd - // module doesn't support parsing from a file stream, only from a - // string, so we have to know how much of the file to read into a - // string. - out << '\n'; + // It's important to delimit the entries in this file somehow + // because, although Python's llsd.parse() can accept a file + // stream, the XML parser expects EOF after a single outer element + // -- it doesn't just stop. So we must extract a sequence of bytes + // strings from the file. But since one of the serialization + // formats we want to test is binary, we can't pick any single + // byte value as a delimiter! Use a binary integer length prefix + // instead. + std::ostringstream buffer; + serialize(item, buffer); + auto buffstr{ buffer.str() }; + int bufflen{ static_cast<int>(buffstr.length()) }; + out.write(reinterpret_cast<const char*>(&bufflen), sizeof(bufflen)); + out.write(buffstr.c_str(), buffstr.length()); } } - template<> template<> - void TestPythonCompatibleObject::test<3>() + // helper for test<3> - test<7> + static void toPythonUsing(const std::string& desc, + const FormatterFunction& serialize) { - set_test_name("verify sequence to Python"); - - LLSD cdata(LLSDArray(17)(3.14) - ("This string\n" - "has several\n" - "lines.")); + LLSD cdata(llsd::array(17, 3.14, + "This string\n" + "has several\n" + "lines.")); const char pydata[] = "def verify(iterable):\n" @@ -1836,7 +1941,7 @@ namespace tut " except StopIteration:\n" " pass\n" " else:\n" - " assert False, 'Too many data items'\n"; + " raise AssertionError('Too many data items')\n"; // Create an llsdXXXXXX file containing 'data' serialized to // notation. @@ -1845,32 +1950,128 @@ namespace tut // takes a callable. To this callable it passes the // std::ostream with which it's writing the // NamedTempFile. - boost::bind(writeLLSDArray, _1, cdata)); + [serialize, cdata] + (std::ostream& out) + { writeLLSDArray(serialize, out, cdata); }); - python("read C++ notation", + python("read C++ " + desc, placeholders::arg1 << import_llsd << - "def parse_each(iterable):\n" - " for item in iterable:\n" - " yield llsd.parse(item)\n" << - pydata << + "from functools import partial\n" + "import io\n" + "import struct\n" + "lenformat = struct.Struct('i')\n" + "def parse_each(inf):\n" + " for rawlen in iter(partial(inf.read, lenformat.size), b''):\n" + " len = lenformat.unpack(rawlen)[0]\n" + // Since llsd.parse() has no max_bytes argument, instead of + // passing the input stream directly to parse(), read the item + // into a distinct bytes object and parse that. + " data = inf.read(len)\n" + " try:\n" + " frombytes = llsd.parse(data)\n" + " except llsd.LLSDParseError as err:\n" + " print(f'*** {err}')\n" + " print(f'Bad content:\\n{data!r}')\n" + " raise\n" + // Also try parsing from a distinct stream. + " stream = io.BytesIO(data)\n" + " fromstream = llsd.parse(stream)\n" + " assert frombytes == fromstream\n" + " yield frombytes\n" + << pydata << // Don't forget raw-string syntax for Windows pathnames. "verify(parse_each(open(r'" << file.getName() << "', 'rb')))\n"); } template<> template<> + void TestPythonCompatibleObject::test<3>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_XML)"); + toPythonUsing("LLSD_XML", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_XML); }); + } + + template<> template<> void TestPythonCompatibleObject::test<4>() { - set_test_name("verify sequence from Python"); + set_test_name("to Python using LLSDSerialize::serialize(LLSD_NOTATION)"); + toPythonUsing("LLSD_NOTATION", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_NOTATION); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<5>() + { + set_test_name("to Python using LLSDSerialize::serialize(LLSD_BINARY)"); + toPythonUsing("LLSD_BINARY", + [](const LLSD& sd, std::ostream& out) + { LLSDSerialize::serialize(sd, out, LLSDSerialize::LLSD_BINARY); }); + } + + template<> template<> + void TestPythonCompatibleObject::test<6>() + { + set_test_name("to Python using LLSDSerialize::toXML()"); + toPythonUsing("toXML()", LLSDSerialize::toXML); + } + + template<> template<> + void TestPythonCompatibleObject::test<7>() + { + set_test_name("to Python using LLSDSerialize::toNotation()"); + toPythonUsing("toNotation()", LLSDSerialize::toNotation); + } +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("to Python using LLSDSerialize::toBinary()"); + // We don't expect this to work because, without a header, + // llsd.parse() will assume notation rather than binary. + toPythonUsing("toBinary()", LLSDSerialize::toBinary); + } +|*==========================================================================*/ + + // helper for test<8> - test<12> + bool itemFromStream(std::istream& istr, LLSD& item, const ParserFunction& parse) + { + // reset the output value for debugging clarity + item.clear(); + // We use an int length prefix as a foolproof delimiter even for + // binary serialized streams. + int length{ 0 }; + istr.read(reinterpret_cast<char*>(&length), sizeof(length)); +// return parse(istr, item, length); + // Sadly, as of 2022-12-01 it seems we can't really trust our LLSD + // parsers to honor max_bytes: this test works better when we read + // each item into its own distinct LLMemoryStream, instead of passing + // the original istr with a max_bytes constraint. + std::vector<U8> buffer(length); + istr.read(reinterpret_cast<char*>(buffer.data()), length); + LLMemoryStream stream(buffer.data(), length); + return parse(stream, item, length); + } + + // helper for test<8> - test<12> + void fromPythonUsing(const std::string& pyformatter, + const ParserFunction& parse= + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::deserialize(data, istr, max_bytes); }) + { // Create an empty data file. This is just a placeholder for our // script to write into. Create it to establish a unique name that // we know. NamedTempFile file("llsd", ""); - python("write Python notation", + python("Python " + pyformatter, placeholders::arg1 << import_llsd << + "import struct\n" + "lenformat = struct.Struct('i')\n" "DATA = [\n" " 17,\n" " 3.14,\n" @@ -1881,34 +2082,87 @@ namespace tut "]\n" // Don't forget raw-string syntax for Windows pathnames. // N.B. Using 'print' implicitly adds newlines. - "with open(r'" << file.getName() << "', 'w') as f:\n" + "with open(r'" << file.getName() << "', 'wb') as f:\n" " for item in DATA:\n" - " print(llsd.format_notation(item).decode(), file=f)\n"); + " serialized = llsd." << pyformatter << "(item)\n" + " f.write(lenformat.pack(len(serialized)))\n" + " f.write(serialized)\n"); std::ifstream inf(file.getName().c_str()); LLSD item; - // Notice that we're not doing anything special to parse out the - // newlines: LLSDSerialize::fromNotation ignores them. While it would - // seem they're not strictly necessary, going in this direction, we - // want to ensure that notation-separated-by-newlines works in both - // directions -- since in practice, a given file might be read by - // either language. - ensure_equals("Failed to read LLSD::Integer from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_equals(item.asInteger(), 17); - ensure_equals("Failed to read LLSD::Real from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_approximately_equals("Bad LLSD::Real value from Python", - item.asReal(), 3.14, 7); // 7 bits ~= 0.01 - ensure_equals("Failed to read LLSD::String from Python", - LLSDSerialize::fromNotation(item, inf, LLSDSerialize::SIZE_UNLIMITED), - 1); - ensure_equals(item.asString(), - "This string\n" - "has several\n" - "lines."); - + try + { + ensure("Failed to read LLSD::Integer from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asInteger(), 17); + ensure("Failed to read LLSD::Real from Python", + itemFromStream(inf, item, parse)); + ensure_approximately_equals("Bad LLSD::Real value from Python", + item.asReal(), 3.14, 7); // 7 bits ~= 0.01 + ensure("Failed to read LLSD::String from Python", + itemFromStream(inf, item, parse)); + ensure_equals(item.asString(), + "This string\n" + "has several\n" + "lines."); + } + catch (const tut::failure& err) + { + std::cout << "for " << err.what() << ", item = " << item << std::endl; + throw; + } + } + + template<> template<> + void TestPythonCompatibleObject::test<8>() + { + set_test_name("from Python XML using LLSDSerialize::deserialize()"); + fromPythonUsing("format_xml"); + } + + template<> template<> + void TestPythonCompatibleObject::test<9>() + { + set_test_name("from Python notation using LLSDSerialize::deserialize()"); + fromPythonUsing("format_notation"); + } + + template<> template<> + void TestPythonCompatibleObject::test<10>() + { + set_test_name("from Python binary using LLSDSerialize::deserialize()"); + fromPythonUsing("format_binary"); + } + + template<> template<> + void TestPythonCompatibleObject::test<11>() + { + set_test_name("from Python XML using fromXML()"); + // fromXML()'s optional 3rd param isn't max_bytes, it's emit_errors + fromPythonUsing("format_xml", + [](std::istream& istr, LLSD& data, llssize) + { return LLSDSerialize::fromXML(data, istr) > 0; }); + } + + template<> template<> + void TestPythonCompatibleObject::test<12>() + { + set_test_name("from Python notation using fromNotation()"); + fromPythonUsing("format_notation", + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::fromNotation(data, istr, max_bytes) > 0; }); + } + +/*==========================================================================*| + template<> template<> + void TestPythonCompatibleObject::test<13>() + { + set_test_name("from Python binary using fromBinary()"); + // We don't expect this to work because format_binary() emits a + // header, but fromBinary() won't recognize a header. + fromPythonUsing("format_binary", + [](std::istream& istr, LLSD& data, llssize max_bytes) + { return LLSDSerialize::fromBinary(data, istr, max_bytes) > 0; }); } +|*==========================================================================*/ } |