diff options
Diffstat (limited to 'indra/llcommon')
31 files changed, 914 insertions, 444 deletions
diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 670eb0fd0b..55dde5dc42 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -11,10 +11,6 @@ include(LLSharedLibs) include(ZLIBNG) include(Tracy) -if ($ENV{MSYSTEM_CARCH} MATCHES aarch64) - include(cpuinfo) -endif () - set(llcommon_SOURCE_FILES apply.cpp commoncontrol.cpp @@ -285,14 +281,10 @@ target_link_libraries( ll::tracy ) -if ($ENV{MSYSTEM_CARCH} MATCHES aarch64) - target_link_libraries(llcommon ll::cpuinfo) -endif () - target_include_directories(llcommon INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(llcommon PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -if (CMAKE_OSX_ARCHITECTURES MATCHES arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES aarch64 OR ($ENV{MSYSTEM_CARCH} MATCHES aarch64)) +if (CMAKE_OSX_ARCHITECTURES MATCHES arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES aarch64) if (${PREBUILD_TRACKING_DIR}/sentinel_installed IS_NEWER_THAN ${PREBUILD_TRACKING_DIR}/sse2neon_installed OR NOT ${sse2neon_installed} EQUAL 0) file(MAKE_DIRECTORY ${LIBS_PREBUILT_DIR}/include/sse2neon) if (NOT EXISTS ${LIBS_PREBUILT_DIR}/include/sse2neon/sse2neon.h) @@ -304,6 +296,8 @@ if (CMAKE_OSX_ARCHITECTURES MATCHES arm64 OR CMAKE_SYSTEM_PROCESSOR MATCHES aarc file(WRITE ${PREBUILD_TRACKING_DIR}/sse2neon_installed "0") endif () target_include_directories(llcommon PUBLIC ${LIBS_PREBUILT_DIR}/include/sse2neon) +elseif ($ENV{MSYSTEM_CARCH} MATCHES aarch64) + target_include_directories(llcommon PUBLIC ${prefix_result}/../include/sse2neon) endif () if (${LINUX_DISTRO} MATCHES debian OR (${LINUX_DISTRO} MATCHES ubuntu) OR (${LINUX_DISTRO} MATCHES opensuse-tumbleweed)) diff --git a/indra/llcommon/StackWalker.cpp b/indra/llcommon/StackWalker.cpp index e9ae1723fb..027df80df5 100644 --- a/indra/llcommon/StackWalker.cpp +++ b/indra/llcommon/StackWalker.cpp @@ -1100,6 +1100,14 @@ bool StackWalker::ShowCallstack(bool verbose, HANDLE hThread, const CONTEXT *con s.AddrBStore.Mode = AddrModeFlat; s.AddrStack.Offset = c.IntSp; s.AddrStack.Mode = AddrModeFlat; +#elif _M_ARM64 + imageType = IMAGE_FILE_MACHINE_ARM64; + s.AddrPC.Offset = c.Pc; + s.AddrPC.Mode = AddrModeFlat; + s.AddrFrame.Offset = c.Fp; + s.AddrFrame.Mode = AddrModeFlat; + s.AddrStack.Offset = c.Sp; + s.AddrStack.Mode = AddrModeFlat; #else #error "Platform not supported!" #endif diff --git a/indra/llcommon/fsyspath.h b/indra/llcommon/fsyspath.h index e9c96edce3..2c900c02a7 100644 --- a/indra/llcommon/fsyspath.h +++ b/indra/llcommon/fsyspath.h @@ -12,7 +12,10 @@ #if ! defined(LL_FSYSPATH_H) #define LL_FSYSPATH_H +#include <boost/iterator/transform_iterator.hpp> #include <filesystem> +#include <string> +#include <string_view> // While std::filesystem::path can be directly constructed from std::string on // both Posix and Windows, that's not what we want on Windows. Per @@ -33,37 +36,43 @@ // char"), the "native narrow encoding" isn't UTF-8, so file paths containing // non-ASCII characters get mangled. // -// Once we're building with C++20, we could pass a UTF-8 std::string through a -// vector<char8_t> to engage std::filesystem::path's own UTF-8 conversion. But -// sigh, as of 2024-04-03 we're not yet there. -// -// Anyway, encapsulating the important UTF-8 conversions in our own subclass -// allows us to migrate forward to C++20 conventions without changing -// referencing code. +// Encapsulating the important UTF-8 conversions in our own subclass allows us +// to migrate forward to C++20 conventions without changing referencing code. class fsyspath: public std::filesystem::path { using super = std::filesystem::path; + // In C++20 (__cpp_lib_char8_t), std::filesystem::u8path() is deprecated. + // std::filesystem::path(iter, iter) performs UTF-8 conversions when the + // value_type of the iterators is char8_t. While we could copy into a + // temporary std::u8string and from there into std::filesystem::path, to + // minimize string copying we'll define a transform_iterator that accepts + // a std::string_view::iterator and dereferences to char8_t. + struct u8ify + { + char8_t operator()(char c) const { return char8_t(c); } + }; + using u8iter = boost::transform_iterator<u8ify, std::string_view::iterator>; + public: // default fsyspath() {} - // construct from UTF-8 encoded std::string - fsyspath(const std::string& path): super(std::filesystem::u8path(path)) {} - // construct from UTF-8 encoded const char* - fsyspath(const char* path): super(std::filesystem::u8path(path)) {} + // construct from UTF-8 encoded string + fsyspath(const std::string& path): fsyspath(std::string_view(path)) {} + fsyspath(const char* path): fsyspath(std::string_view(path)) {} + fsyspath(std::string_view path): + super(u8iter(path.begin(), u8ify()), u8iter(path.end(), u8ify())) + {} // construct from existing path fsyspath(const super& path): super(path) {} - fsyspath& operator=(const super& p) { super::operator=(p); return *this; } - fsyspath& operator=(const std::string& p) - { - super::operator=(std::filesystem::u8path(p)); - return *this; - } - fsyspath& operator=(const char* p) + fsyspath& operator=(const super& p) { super::operator=(p); return *this; } + fsyspath& operator=(const std::string& p) { return (*this) = std::string_view(p); } + fsyspath& operator=(const char* p) { return (*this) = std::string_view(p); } + fsyspath& operator=(std::string_view p) { - super::operator=(std::filesystem::u8path(p)); + assign(u8iter(p.begin(), u8ify()), u8iter(p.end(), u8ify())); return *this; } diff --git a/indra/llcommon/linden_common.h b/indra/llcommon/linden_common.h index a918caa2e8..a41af153fe 100644 --- a/indra/llcommon/linden_common.h +++ b/indra/llcommon/linden_common.h @@ -28,12 +28,6 @@ #define LL_LINDEN_COMMON_H #include "llprofiler.h" -#if TRACY_ENABLE && !defined(LL_PROFILER_ENABLE_TRACY_OPENGL) // hooks for memory profiling -void *tracy_aligned_malloc(size_t size, size_t alignment); -void tracy_aligned_free(void *memblock); -#define _aligned_malloc(X, Y) tracy_aligned_malloc((X), (Y)) -#define _aligned_free(X) tracy_aligned_free((X)) -#endif // *NOTE: Please keep includes here to a minimum! // diff --git a/indra/llcommon/llapp.cpp b/indra/llcommon/llapp.cpp index 08a43983d3..c532620daa 100644 --- a/indra/llcommon/llapp.cpp +++ b/indra/llcommon/llapp.cpp @@ -229,7 +229,7 @@ bool LLApp::parseCommandOptions(int argc, wchar_t** wargv) if(wargv[ii][1] == '-') ++offset; #if LL_WINDOWS - name.assign(utf16str_to_utf8str(&wargv[ii][offset])); + name.assign(ll_convert_wide_to_string(&wargv[ii][offset])); #else name.assign(wstring_to_utf8str(&wargv[ii][offset])); #endif @@ -253,7 +253,7 @@ bool LLApp::parseCommandOptions(int argc, wchar_t** wargv) ++ii; #if LL_WINDOWS - value.assign(utf16str_to_utf8str((wargv[ii]))); + value.assign(ll_convert_wide_to_string((wargv[ii]))); #else value.assign(wstring_to_utf8str((wargv[ii]))); #endif diff --git a/indra/llcommon/llassettype.cpp b/indra/llcommon/llassettype.cpp index c09cf7abd2..9672a3262b 100644 --- a/indra/llcommon/llassettype.cpp +++ b/indra/llcommon/llassettype.cpp @@ -29,6 +29,7 @@ #include "llassettype.h" #include "lldictionary.h" #include "llmemory.h" +#include "llsd.h" #include "llsingleton.h" ///---------------------------------------------------------------------------- @@ -246,3 +247,19 @@ bool LLAssetType::lookupIsAssetIDKnowable(EType asset_type) } return false; } + +LLSD LLAssetType::getTypeNames() +{ + LLSD type_names; + const LLAssetDictionary *dict = LLAssetDictionary::getInstance(); + for (S32 type = AT_TEXTURE; type < AT_COUNT; ++type) + { + const AssetEntry *entry = dict->lookup((LLAssetType::EType) type); + // skip llassettype_bad_lookup + if (entry) + { + type_names.append(entry->mTypeName); + } + } + return type_names; +} diff --git a/indra/llcommon/llassettype.h b/indra/llcommon/llassettype.h index 547c3f4329..17177d81c3 100644 --- a/indra/llcommon/llassettype.h +++ b/indra/llcommon/llassettype.h @@ -165,6 +165,8 @@ public: static bool lookupIsAssetFetchByIDAllowed(EType asset_type); // the asset allows direct download static bool lookupIsAssetIDKnowable(EType asset_type); // asset data can be known by the viewer + static LLSD getTypeNames(); + static const std::string BADLOOKUP; protected: diff --git a/indra/llcommon/llcommon.cpp b/indra/llcommon/llcommon.cpp index 84b35749cc..7a22eaf203 100644 --- a/indra/llcommon/llcommon.cpp +++ b/indra/llcommon/llcommon.cpp @@ -33,23 +33,23 @@ #include "lltracethreadrecorder.h" #include "llcleanup.h" -thread_local bool gProfilerEnabled = false; - -#if (TRACY_ENABLE) +#if LL_PROFILER_CONFIGURATION >= LL_PROFILER_CONFIG_TRACY && TRACY_ENABLE // Override new/delete for tracy memory profiling void* ll_tracy_new(size_t size) { - void* ptr; - if (gProfilerEnabled) - { - //LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; - ptr = (malloc)(size); - } - else + void* ptr = (malloc)(size); + if (!ptr) { - ptr = (malloc)(size); + throw std::bad_alloc(); } + LL_PROFILE_ALLOC(ptr, size); + return ptr; +} + +void* ll_tracy_aligned_new(size_t size, size_t alignment) +{ + void* ptr = ll_aligned_malloc_fallback(size, alignment); if (!ptr) { throw std::bad_alloc(); @@ -58,6 +58,18 @@ void* ll_tracy_new(size_t size) return ptr; } +void ll_tracy_delete(void* ptr) +{ + LL_PROFILE_FREE(ptr); + (free)(ptr); +} + +void ll_tracy_aligned_delete(void* ptr) +{ + LL_PROFILE_FREE(ptr); + ll_aligned_free_fallback(ptr); +} + void* operator new(size_t size) { return ll_tracy_new(size); @@ -68,18 +80,14 @@ void* operator new[](std::size_t count) return ll_tracy_new(count); } -void ll_tracy_delete(void* ptr) +void* operator new(size_t size, std::align_val_t align) { - LL_PROFILE_FREE(ptr); - if (gProfilerEnabled) - { - //LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; - (free)(ptr); - } - else - { - (free)(ptr); - } + return ll_tracy_aligned_new(size, (size_t)align); +} + +void* operator new[](std::size_t count, std::align_val_t align) +{ + return ll_tracy_aligned_new(count, (size_t)align); } void operator delete(void *ptr) noexcept @@ -92,27 +100,17 @@ void operator delete[](void* ptr) noexcept ll_tracy_delete(ptr); } -// C-style malloc/free can't be so easily overridden, so we define tracy versions and use -// a pre-processor #define in linden_common.h to redirect to them. The parens around the native -// functions below prevents recursive substitution by the preprocessor. -// -// Unaligned mallocs are rare in LL code but hooking them causes problems in 3p lib code (looking at -// you, Havok), so we'll only capture the aligned version. - -void *tracy_aligned_malloc(size_t size, size_t alignment) +void operator delete(void *ptr, std::align_val_t align) noexcept { - auto ptr = ll_aligned_malloc_fallback(size, alignment); - if (ptr) LL_PROFILE_ALLOC(ptr, size); - return ptr; + ll_tracy_aligned_delete(ptr); } -void tracy_aligned_free(void *memblock) +void operator delete[](void* ptr, std::align_val_t align) noexcept { - LL_PROFILE_FREE(memblock); - ll_aligned_free_fallback(memblock); + ll_tracy_aligned_delete(ptr); } -#endif +#endif // TRACY_ENABLE && !LL_PROFILER_ENABLE_TRACY_OPENGL //static bool LLCommon::sAprInitialized = false; diff --git a/indra/llcommon/llerror.cpp b/indra/llcommon/llerror.cpp index d834098994..3411e9c6bb 100644 --- a/indra/llcommon/llerror.cpp +++ b/indra/llcommon/llerror.cpp @@ -79,7 +79,7 @@ namespace { // if (s.size()) { - OutputDebugString(utf8str_to_utf16str(s).c_str()); + OutputDebugString(ll_convert<std::wstring>(s).c_str()); OutputDebugString(TEXT("\n")); } } diff --git a/indra/llcommon/llfasttimer.cpp b/indra/llcommon/llfasttimer.cpp index 7ceb57336e..be0cab36e3 100644 --- a/indra/llcommon/llfasttimer.cpp +++ b/indra/llcommon/llfasttimer.cpp @@ -64,7 +64,7 @@ bool BlockTimer::sLog = false; std::string BlockTimer::sLogName = ""; bool BlockTimer::sMetricLog = false; -#if LL_LINUX +#if LL_LINUX || (LL_DARWIN && LL_ARM64) U64 BlockTimer::sClockResolution = 1000000000; // Nanosecond resolution #else U64 BlockTimer::sClockResolution = 1000000; // Microsecond resolution diff --git a/indra/llcommon/llfasttimer.h b/indra/llcommon/llfasttimer.h index 8499655bfa..fde168b022 100644 --- a/indra/llcommon/llfasttimer.h +++ b/indra/llcommon/llfasttimer.h @@ -30,9 +30,14 @@ #include "llinstancetracker.h" #include "lltrace.h" #include "lltreeiterators.h" +#include "llprocessor.h" +#if LL_X86 || LL_X86_64 #if LL_WINDOWS #include <intrin.h> +#else +#include <x86intrin.h> +#endif #endif #define LL_FAST_TIMER_ON 1 @@ -68,35 +73,10 @@ public: // // Windows implementation of CPU clock // - - // - // NOTE: put back in when we aren't using platform sdk anymore - // - // because MS has different signatures for these functions in winnt.h - // need to rename them to avoid conflicts - //#define _interlockedbittestandset _renamed_interlockedbittestandset - //#define _interlockedbittestandreset _renamed_interlockedbittestandreset - //#include <intrin.h> - //#undef _interlockedbittestandset - //#undef _interlockedbittestandreset - - //inline U32 getCPUClockCount32() - //{ - // U64 time_stamp = __rdtsc(); - // return (U32)(time_stamp >> 8); - //} - // - //// return full timer value, *not* shifted by 8 bits - //inline U64 getCPUClockCount64() - //{ - // return __rdtsc(); - //} - - +#if LL_FASTTIMER_USE_RDTSC // shift off lower 8 bits for lower resolution but longer term timing // on 1Ghz machine, a 32-bit word will hold ~1000 seconds of timing -#if LL_FASTTIMER_USE_RDTSC static U32 getCPUClockCount32() { #if _M_ARM64 @@ -181,23 +161,39 @@ public: #endif // (LL_LINUX) && !(defined(__i386__) || defined(__amd64__)) +/* +#if LL_DARWIN && LL_ARM64 + // + // Mac implementation of CPU clock - non-x86. + // + static U64 getCPUClockCount64() + { + return clock_gettime_nsec_np(CLOCK_UPTIME_RAW); + } + + static U32 getCPUClockCount32() + { + return (U32)(getCPUClockCount64() >> 8); + } +#endif // LL_DARWIN && LL_ARM64 +*/ + #if (LL_LINUX || LL_DARWIN || __FreeBSD__) && (defined(__i386__) || defined(__amd64__)) // // Mac+Linux FAST x86 implementation of CPU clock + // +#if LL_FASTTIMER_USE_RDTSC static U32 getCPUClockCount32() { - U32 low(0),high(0); - __asm__ volatile (".byte 0x0f, 0x31": "=a"(low), "=d"(high) ); - return (low>>8) | (high<<24); + U64 time_stamp = __rdtsc() >> 8U; + return static_cast<U32>(time_stamp); } static U64 getCPUClockCount64() { - U32 low(0),high(0); - __asm__ volatile (".byte 0x0f, 0x31": "=a"(low), "=d"(high) ); - return (U64)low | ( ((U64)high) << 32); + return static_cast<U64>(__rdtsc()); } - +#endif #endif static BlockTimerStatHandle& getRootTimeBlock(); diff --git a/indra/llcommon/llfile.cpp b/indra/llcommon/llfile.cpp index d0bc8f7652..a539e4fe28 100644 --- a/indra/llcommon/llfile.cpp +++ b/indra/llcommon/llfile.cpp @@ -35,7 +35,6 @@ #if LL_WINDOWS #include "llwin32headers.h" -#include <stdlib.h> // Windows errno #include <vector> #else #include <errno.h> @@ -49,6 +48,86 @@ static std::string empty; // variants of strerror() to report errors. #if LL_WINDOWS +// For the situations where we directly call into Windows API functions we need to translate +// the Windows error codes into errno values +namespace +{ + struct errentry + { + unsigned long oserr; // Windows OS error value + int errcode; // System V error code + }; +} + +// translation table between Windows OS error value and System V errno code +static errentry const errtable[] +{ + { ERROR_INVALID_FUNCTION, EINVAL }, // 1 + { ERROR_FILE_NOT_FOUND, ENOENT }, // 2 + { ERROR_PATH_NOT_FOUND, ENOENT }, // 3 + { ERROR_TOO_MANY_OPEN_FILES, EMFILE }, // 4 + { ERROR_ACCESS_DENIED, EACCES }, // 5 + { ERROR_INVALID_HANDLE, EBADF }, // 6 + { ERROR_ARENA_TRASHED, ENOMEM }, // 7 + { ERROR_NOT_ENOUGH_MEMORY, ENOMEM }, // 8 + { ERROR_INVALID_BLOCK, ENOMEM }, // 9 + { ERROR_BAD_ENVIRONMENT, E2BIG }, // 10 + { ERROR_BAD_FORMAT, ENOEXEC }, // 11 + { ERROR_INVALID_ACCESS, EINVAL }, // 12 + { ERROR_INVALID_DATA, EINVAL }, // 13 + { ERROR_INVALID_DRIVE, ENOENT }, // 15 + { ERROR_CURRENT_DIRECTORY, EACCES }, // 16 + { ERROR_NOT_SAME_DEVICE, EXDEV }, // 17 + { ERROR_NO_MORE_FILES, ENOENT }, // 18 + { ERROR_LOCK_VIOLATION, EACCES }, // 33 + { ERROR_BAD_NETPATH, ENOENT }, // 53 + { ERROR_NETWORK_ACCESS_DENIED, EACCES }, // 65 + { ERROR_BAD_NET_NAME, ENOENT }, // 67 + { ERROR_FILE_EXISTS, EEXIST }, // 80 + { ERROR_CANNOT_MAKE, EACCES }, // 82 + { ERROR_FAIL_I24, EACCES }, // 83 + { ERROR_INVALID_PARAMETER, EINVAL }, // 87 + { ERROR_NO_PROC_SLOTS, EAGAIN }, // 89 + { ERROR_DRIVE_LOCKED, EACCES }, // 108 + { ERROR_BROKEN_PIPE, EPIPE }, // 109 + { ERROR_DISK_FULL, ENOSPC }, // 112 + { ERROR_INVALID_TARGET_HANDLE, EBADF }, // 114 + { ERROR_WAIT_NO_CHILDREN, ECHILD }, // 128 + { ERROR_CHILD_NOT_COMPLETE, ECHILD }, // 129 + { ERROR_DIRECT_ACCESS_HANDLE, EBADF }, // 130 + { ERROR_NEGATIVE_SEEK, EINVAL }, // 131 + { ERROR_SEEK_ON_DEVICE, EACCES }, // 132 + { ERROR_DIR_NOT_EMPTY, ENOTEMPTY }, // 145 + { ERROR_NOT_LOCKED, EACCES }, // 158 + { ERROR_BAD_PATHNAME, ENOENT }, // 161 + { ERROR_MAX_THRDS_REACHED, EAGAIN }, // 164 + { ERROR_LOCK_FAILED, EACCES }, // 167 + { ERROR_ALREADY_EXISTS, EEXIST }, // 183 + { ERROR_FILENAME_EXCED_RANGE, ENOENT }, // 206 + { ERROR_NESTING_NOT_ALLOWED, EAGAIN }, // 215 + { ERROR_NO_UNICODE_TRANSLATION, EILSEQ }, // 1113 + { ERROR_NOT_ENOUGH_QUOTA, ENOMEM } // 1816 +}; + +static int set_errno_from_oserror(unsigned long oserr) +{ + if (!oserr) + return 0; + + // Check the table for the Windows OS error code + for (const struct errentry &entry : errtable) + { + if (oserr == entry.oserr) + { + _set_errno(entry.errcode); + return -1; + } + } + + _set_errno(EINVAL); + return -1; +} + // On Windows, use strerror_s(). std::string strerr(int errn) { @@ -57,7 +136,67 @@ std::string strerr(int errn) return buffer; } -typedef std::basic_ios<char,std::char_traits < char > > _Myios; +inline bool is_slash(wchar_t const c) +{ + return c == L'\\' || c == L'/'; +} + +static std::wstring utf8path_to_wstring(const std::string& utf8path) +{ + if (utf8path.size() >= MAX_PATH) + { + // By prepending "\\?\" to a path, Windows widechar file APIs will not fail on long path names + std::wstring utf16path = L"\\\\?\\" + ll_convert<std::wstring>(utf8path); + // We need to make sure that the path does not contain forward slashes as above + // prefix does bypass the path normalization that replaces slashes with backslashes + // before passing the path to kernel mode APIs + std::replace(utf16path.begin(), utf16path.end(), L'/', L'\\'); + return utf16path; + } + return ll_convert<std::wstring>(utf8path); +} + +static unsigned short get_fileattr(const std::wstring& utf16path, bool dontFollowSymLink = false) +{ + unsigned long flags = FILE_FLAG_BACKUP_SEMANTICS; + if (dontFollowSymLink) + { + flags |= FILE_FLAG_OPEN_REPARSE_POINT; + } + HANDLE file_handle = CreateFileW(utf16path.c_str(), FILE_READ_ATTRIBUTES, + FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, OPEN_EXISTING, flags, nullptr); + if (file_handle != INVALID_HANDLE_VALUE) + { + FILE_ATTRIBUTE_TAG_INFO attribute_info; + if (GetFileInformationByHandleEx(file_handle, FileAttributeTagInfo, &attribute_info, sizeof(attribute_info))) + { + // A volume path alone (only drive letter) is not recognized as directory while it technically is + bool is_directory = (attribute_info.FileAttributes & FILE_ATTRIBUTE_DIRECTORY) || + (iswalpha(utf16path[0]) && utf16path[1] == ':' && + (!utf16path[2] || (is_slash(utf16path[2]) && !utf16path[3]))); + unsigned short st_mode = is_directory ? S_IFDIR : + (attribute_info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT ? S_IFLNK : S_IFREG); + st_mode |= (attribute_info.FileAttributes & FILE_ATTRIBUTE_READONLY) ? S_IREAD : S_IREAD | S_IWRITE; + // we do not try to guess executable flag + + // propagate user bits to group/other fields: + st_mode |= (st_mode & 0700) >> 3; + st_mode |= (st_mode & 0700) >> 6; + + CloseHandle(file_handle); + return st_mode; + } + } + // Retrieve last error and set errno before calling CloseHandle() + set_errno_from_oserror(GetLastError()); + + if (file_handle != INVALID_HANDLE_VALUE) + { + CloseHandle(file_handle); + } + return 0; +} #else // On Posix we want to call strerror_r(), but alarmingly, there are two @@ -108,18 +247,13 @@ std::string strerr(int errn) } #endif // ! LL_WINDOWS -// On either system, shorthand call just infers global 'errno'. -std::string strerr() -{ - return strerr(errno); -} - -int warnif(const std::string& desc, const std::string& filename, int rc, int accept=0) +static int warnif(const std::string& desc, const std::string& filename, int rc, int accept = 0) { if (rc < 0) { // Capture errno before we start emitting output int errn = errno; + // For certain operations, a particular errno value might be // acceptable -- e.g. stat() could permit ENOENT, mkdir() could permit // EEXIST. Don't warn if caller explicitly says this errno is okay. @@ -176,68 +310,59 @@ int warnif(const std::string& desc, const std::string& filename, int rc, int acc // static int LLFile::mkdir(const std::string& dirname, int perms) { + // We often use mkdir() to ensure the existence of a directory that might + // already exist. There is no known case in which we want to call out as + // an error the requested directory already existing. #if LL_WINDOWS // permissions are ignored on Windows - std::string utf8dirname = dirname; - llutf16string utf16dirname = utf8str_to_utf16str(utf8dirname); - int rc = _wmkdir(utf16dirname.c_str()); + int rc = 0; + std::wstring utf16dirname = utf8path_to_wstring(dirname); + if (!CreateDirectoryW(utf16dirname.c_str(), nullptr)) + { + // Only treat other errors than an already existing file as a real error + unsigned long oserr = GetLastError(); + if (oserr != ERROR_ALREADY_EXISTS) + { + rc = set_errno_from_oserror(oserr); + } + } #else int rc = ::mkdir(dirname.c_str(), (mode_t)perms); -#endif - // We often use mkdir() to ensure the existence of a directory that might - // already exist. There is no known case in which we want to call out as - // an error the requested directory already existing. if (rc < 0 && errno == EEXIST) { // this is not the error you want, move along return 0; } +#endif // anything else might be a problem - return warnif("mkdir", dirname, rc, EEXIST); + return warnif("mkdir", dirname, rc); } // static -int LLFile::rmdir(const std::string& dirname) +int LLFile::rmdir(const std::string& dirname, int suppress_error) { #if LL_WINDOWS - // permissions are ignored on Windows - std::string utf8dirname = dirname; - llutf16string utf16dirname = utf8str_to_utf16str(utf8dirname); + std::wstring utf16dirname = utf8path_to_wstring(dirname); int rc = _wrmdir(utf16dirname.c_str()); #else int rc = ::rmdir(dirname.c_str()); #endif - return warnif("rmdir", dirname, rc); + return warnif("rmdir", dirname, rc, suppress_error); } // static -LLFILE* LLFile::fopen(const std::string& filename, const char* mode) /* Flawfinder: ignore */ +LLFILE* LLFile::fopen(const std::string& filename, const char* mode) { #if LL_WINDOWS - std::string utf8filename = filename; - std::string utf8mode = std::string(mode); - llutf16string utf16filename = utf8str_to_utf16str(utf8filename); - llutf16string utf16mode = utf8str_to_utf16str(utf8mode); - return _wfopen(utf16filename.c_str(),utf16mode.c_str()); + std::wstring utf16filename = utf8path_to_wstring(filename); + std::wstring utf16mode = ll_convert<std::wstring>(std::string(mode)); + return _wfopen(utf16filename.c_str(), utf16mode.c_str()); #else - return ::fopen(filename.c_str(),mode); /* Flawfinder: ignore */ -#endif -} - -LLFILE* LLFile::_fsopen(const std::string& filename, const char* mode, int sharingFlag) -{ -#if LL_WINDOWS - std::string utf8filename = filename; - std::string utf8mode = std::string(mode); - llutf16string utf16filename = utf8str_to_utf16str(utf8filename); - llutf16string utf16mode = utf8str_to_utf16str(utf8mode); - return _wfsopen(utf16filename.c_str(),utf16mode.c_str(),sharingFlag); -#else - llassert(0);//No corresponding function on non-windows - return NULL; + return ::fopen(filename.c_str(),mode); #endif } +// static int LLFile::close(LLFILE * file) { int ret_value = 0; @@ -248,9 +373,10 @@ int LLFile::close(LLFILE * file) return ret_value; } +// static std::string LLFile::getContents(const std::string& filename) { - LLFILE* fp = fopen(filename, "rb"); /* Flawfinder: ignore */ + LLFILE* fp = LLFile::fopen(filename, "rb"); if (fp) { fseek(fp, 0, SEEK_END); @@ -267,45 +393,80 @@ std::string LLFile::getContents(const std::string& filename) return LLStringUtil::null; } -int LLFile::remove(const std::string& filename, int supress_error) +// static +int LLFile::remove(const std::string& filename, int suppress_error) { #if LL_WINDOWS - std::string utf8filename = filename; - llutf16string utf16filename = utf8str_to_utf16str(utf8filename); - int rc = _wremove(utf16filename.c_str()); + // Posix remove() works on both files and directories although on Windows + // remove() and its wide char variant _wremove() only removes files just + // as its siblings unlink() and _wunlink(). + // If we really only want to support files we should instead use + // unlink() in the non-Windows part below too + int rc = -1; + std::wstring utf16filename = utf8path_to_wstring(filename); + unsigned short st_mode = get_fileattr(utf16filename); + if (S_ISDIR(st_mode)) + { + rc = _wrmdir(utf16filename.c_str()); + } + else if (S_ISREG(st_mode)) + { + rc = _wunlink(utf16filename.c_str()); + } + else if (st_mode) + { + // it is something else than a file or directory + // this should not really happen as long as we do not allow for symlink + // detection in the optional parameter to get_fileattr() + rc = set_errno_from_oserror(ERROR_INVALID_PARAMETER); + } + else + { + // get_fileattr() failed and already set errno, preserve it for correct error reporting + } #else int rc = ::remove(filename.c_str()); #endif - return warnif("remove", filename, rc, supress_error); + return warnif("remove", filename, rc, suppress_error); } -int LLFile::rename(const std::string& filename, const std::string& newname, int supress_error) +// static +int LLFile::rename(const std::string& filename, const std::string& newname, int suppress_error) { #if LL_WINDOWS - std::string utf8filename = filename; - std::string utf8newname = newname; - llutf16string utf16filename = utf8str_to_utf16str(utf8filename); - llutf16string utf16newname = utf8str_to_utf16str(utf8newname); - int rc = _wrename(utf16filename.c_str(),utf16newname.c_str()); + // Posix rename() will gladly overwrite a file at newname if it exists, the Windows + // rename(), respectively _wrename(), will bark on that. Instead call directly the Windows + // API MoveFileEx() and use its flags to specify that overwrite is allowed. + std::wstring utf16filename = utf8path_to_wstring(filename); + std::wstring utf16newname = utf8path_to_wstring(newname); + int rc = 0; + if (!MoveFileExW(utf16filename.c_str(), utf16newname.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) + { + rc = set_errno_from_oserror(GetLastError()); + } #else int rc = ::rename(filename.c_str(),newname.c_str()); #endif - return warnif(STRINGIZE("rename to '" << newname << "' from"), filename, rc, supress_error); + return warnif(STRINGIZE("rename to '" << newname << "' from"), filename, rc, suppress_error); } +// Make this a define rather than using magic numbers multiple times in the code +#define LLFILE_COPY_BUFFER_SIZE 16384 + +// static bool LLFile::copy(const std::string& from, const std::string& to) { bool copied = false; - LLFILE* in = LLFile::fopen(from, "rb"); /* Flawfinder: ignore */ + LLFILE* in = LLFile::fopen(from, "rb"); if (in) { - LLFILE* out = LLFile::fopen(to, "wb"); /* Flawfinder: ignore */ + LLFILE* out = LLFile::fopen(to, "wb"); if (out) { - char buf[16384]; /* Flawfinder: ignore */ + char buf[LLFILE_COPY_BUFFER_SIZE]; size_t readbytes; bool write_ok = true; - while(write_ok && (readbytes = fread(buf, 1, 16384, in))) /* Flawfinder: ignore */ + while (write_ok && (readbytes = fread(buf, 1, LLFILE_COPY_BUFFER_SIZE, in))) { if (fwrite(buf, 1, readbytes, out) != readbytes) { @@ -324,34 +485,62 @@ bool LLFile::copy(const std::string& from, const std::string& to) return copied; } -int LLFile::stat(const std::string& filename, llstat* filestatus) +// static +int LLFile::stat(const std::string& filename, llstat* filestatus, int suppress_error) { #if LL_WINDOWS - std::string utf8filename = filename; - llutf16string utf16filename = utf8str_to_utf16str(utf8filename); - int rc = _wstat(utf16filename.c_str(),filestatus); + std::wstring utf16filename = utf8path_to_wstring(filename); + int rc = _wstat64(utf16filename.c_str(), filestatus); #else - int rc = ::stat(filename.c_str(),filestatus); + int rc = ::stat(filename.c_str(), filestatus); #endif - // We use stat() to determine existence (see isfile(), isdir()). - // Don't spam the log if the subject pathname doesn't exist. - return warnif("stat", filename, rc, ENOENT); + return warnif("stat", filename, rc, suppress_error); } -bool LLFile::isdir(const std::string& filename) +// static +unsigned short LLFile::getattr(const std::string& filename, bool dontFollowSymLink, int suppress_error) { - llstat st; +#if LL_WINDOWS + // _wstat64() is a bit heavyweight on Windows, use a more lightweight API + // to just get the attributes + int rc = -1; + std::wstring utf16filename = utf8path_to_wstring(filename); + unsigned short st_mode = get_fileattr(utf16filename, dontFollowSymLink); + if (st_mode) + { + return st_mode; + } +#else + llstat filestatus; + int rc = dontFollowSymLink ? ::lstat(filename.c_str(), &filestatus) : ::stat(filename.c_str(), &filestatus); + if (rc == 0) + { + return filestatus.st_mode; + } +#endif + warnif("getattr", filename, rc, suppress_error); + return 0; +} - return stat(filename, &st) == 0 && S_ISDIR(st.st_mode); +// static +bool LLFile::isdir(const std::string& filename) +{ + return S_ISDIR(getattr(filename)); } +// static bool LLFile::isfile(const std::string& filename) { - llstat st; + return S_ISREG(getattr(filename)); +} - return stat(filename, &st) == 0 && S_ISREG(st.st_mode); +// static +bool LLFile::islink(const std::string& filename) +{ + return S_ISLNK(getattr(filename, true)); } +// static const char *LLFile::tmpdir() { static std::string utf8path; @@ -378,89 +567,22 @@ const char *LLFile::tmpdir() return utf8path.c_str(); } - -/***************** Modified file stream created to overcome the incorrect behaviour of posix fopen in windows *******************/ - #if LL_WINDOWS -LLFILE * LLFile::_Fiopen(const std::string& filename, - std::ios::openmode mode) -{ // open a file - static const char *mods[] = - { // fopen mode strings corresponding to valid[i] - "r", "w", "w", "a", "rb", "wb", "wb", "ab", - "r+", "w+", "a+", "r+b", "w+b", "a+b", - 0}; - static const int valid[] = - { // valid combinations of open flags - ios_base::in, - ios_base::out, - ios_base::out | ios_base::trunc, - ios_base::out | ios_base::app, - ios_base::in | ios_base::binary, - ios_base::out | ios_base::binary, - ios_base::out | ios_base::trunc | ios_base::binary, - ios_base::out | ios_base::app | ios_base::binary, - ios_base::in | ios_base::out, - ios_base::in | ios_base::out | ios_base::trunc, - ios_base::in | ios_base::out | ios_base::app, - ios_base::in | ios_base::out | ios_base::binary, - ios_base::in | ios_base::out | ios_base::trunc - | ios_base::binary, - ios_base::in | ios_base::out | ios_base::app - | ios_base::binary, - 0}; - - LLFILE *fp = 0; - int n; - ios_base::openmode atendflag = mode & ios_base::ate; - ios_base::openmode norepflag = mode & ios_base::_Noreplace; - - if (mode & ios_base::_Nocreate) - mode |= ios_base::in; // file must exist - mode &= ~(ios_base::ate | ios_base::_Nocreate | ios_base::_Noreplace); - for (n = 0; valid[n] != 0 && valid[n] != mode; ++n) - ; // look for a valid mode - - if (valid[n] == 0) - return (0); // no valid mode - else if (norepflag && mode & (ios_base::out | ios_base::app) - && (fp = LLFile::fopen(filename, "r")) != 0) /* Flawfinder: ignore */ - { // file must not exist, close and fail - fclose(fp); - return (0); - } - else if (fp != 0 && fclose(fp) != 0) - return (0); // can't close after test open -// should open with protection here, if other than default - else if ((fp = LLFile::fopen(filename, mods[n])) == 0) /* Flawfinder: ignore */ - return (0); // open failed - - if (!atendflag || fseek(fp, 0, SEEK_END) == 0) - return (fp); // no need to seek to end, or seek succeeded - - fclose(fp); // can't position at end - return (0); -} - -#endif /* LL_WINDOWS */ - - -#if LL_WINDOWS /************** input file stream ********************************/ llifstream::llifstream() {} // explicit llifstream::llifstream(const std::string& _Filename, ios_base::openmode _Mode): - std::ifstream(utf8str_to_utf16str( _Filename ).c_str(), + std::ifstream(ll_convert<std::wstring>( _Filename ).c_str(), _Mode | ios_base::in) { } void llifstream::open(const std::string& _Filename, ios_base::openmode _Mode) { - std::ifstream::open(utf8str_to_utf16str(_Filename).c_str(), + std::ifstream::open(ll_convert<std::wstring>(_Filename).c_str(), _Mode | ios_base::in); } @@ -472,14 +594,14 @@ llofstream::llofstream() {} // explicit llofstream::llofstream(const std::string& _Filename, ios_base::openmode _Mode): - std::ofstream(utf8str_to_utf16str( _Filename ).c_str(), + std::ofstream(ll_convert<std::wstring>( _Filename ).c_str(), _Mode | ios_base::out) { } void llofstream::open(const std::string& _Filename, ios_base::openmode _Mode) { - std::ofstream::open(utf8str_to_utf16str( _Filename ).c_str(), + std::ofstream::open(ll_convert<std::wstring>( _Filename ).c_str(), _Mode | ios_base::out); } diff --git a/indra/llcommon/llfile.h b/indra/llcommon/llfile.h index 1661cbeb55..04a2946ac4 100644 --- a/indra/llcommon/llfile.h +++ b/indra/llcommon/llfile.h @@ -41,8 +41,9 @@ typedef FILE LLFILE; #include <sys/stat.h> #if LL_WINDOWS -// windows version of stat function and stat data structure are called _stat -typedef struct _stat llstat; +// The Windows version of stat function and stat data structure are called _stat64 +// We use _stat64 here to support 64-bit st_size and time_t values +typedef struct _stat64 llstat; #else typedef struct stat llstat; #include <sys/types.h> @@ -56,35 +57,110 @@ typedef struct stat llstat; # define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR) #endif +// Windows C runtime library does not define this and does not support symlink detection in the +// stat functions but we do in our getattr() function +#ifndef S_IFLNK +#define S_IFLNK 0xA000 /* symlink */ +#endif + +#ifndef S_ISLNK +#define S_ISLNK(x) (((x) & S_IFMT) == S_IFLNK) +#endif + #include "llstring.h" // safe char* -> std::string conversion +/// LLFile is a class of static functions operating on paths +/// All the functions with a path string input take UTF8 path/filenames class LL_COMMON_API LLFile { public: - // All these functions take UTF8 path/filenames. - static LLFILE* fopen(const std::string& filename,const char* accessmode); /* Flawfinder: ignore */ - static LLFILE* _fsopen(const std::string& filename,const char* accessmode,int sharingFlag); + /// open a file with the specified access mode + static LLFILE* fopen(const std::string& filename, const char* accessmode); /* Flawfinder: ignore */ + ///< 'accessmode' follows the rules of the Posix fopen() mode parameter + /// "r" open the file for reading only and positions the stream at the beginning + /// "r+" open the file for reading and writing and positions the stream at the beginning + /// "w" open the file for reading and writing and truncate it to zero length + /// "w+" open or create the file for reading and writing and truncate to zero length if it existed + /// "a" open the file for reading and writing and position the stream at the end of the file + /// "a+" open or create the file for reading and writing and position the stream at the end of the file + /// + /// in addition to these values, "b" can be appended to indicate binary stream access, but on Linux and Mac + /// this is strictly for compatibility and has no effect. On Windows this makes the file functions not + /// try to translate line endings. Windows also allows to append "t" to indicate text mode. If neither + /// "b" or "t" is defined, Windows uses the value set by _fmode which by default is _O_TEXT. + /// This means that it is always a good idea to append "b" specifically for binary file access to + /// avoid corruption of the binary consistency of the data stream when reading or writing + /// Other characters in 'accessmode' will usually cause an error as fopen will verify this parameter + /// @returns a valid LLFILE* pointer on success or NULL on failure static int close(LLFILE * file); + /// retrieve the content of a file into a string static std::string getContents(const std::string& filename); + ///< @returns the content of the file or an empty string on failure - // perms is a permissions mask like 0777 or 0700. In most cases it will - // be overridden by the user's umask. It is ignored on Windows. - // mkdir() considers "directory already exists" to be SUCCESS. + /// create a directory static int mkdir(const std::string& filename, int perms = 0700); - - static int rmdir(const std::string& filename); - static int remove(const std::string& filename, int supress_error = 0); - static int rename(const std::string& filename,const std::string& newname, int supress_error = 0); + ///< perms is a permissions mask like 0777 or 0700. In most cases it will be + /// overridden by the user's umask. It is ignored on Windows. + /// mkdir() considers "directory already exists" to be not an error. + /// @returns 0 on success and -1 on failure. + + //// remove a directory + static int rmdir(const std::string& filename, int suppress_error = 0); + ///< pass ENOENT in the optional 'suppress_error' parameter + /// if you don't want a warning in the log when the directory does not exist + /// @returns 0 on success and -1 on failure. + + /// remove a file or directory + static int remove(const std::string& filename, int suppress_error = 0); + ///< pass ENOENT in the optional 'suppress_error' parameter + /// if you don't want a warning in the log when the directory does not exist + /// @returns 0 on success and -1 on failure. + + /// rename a file + static int rename(const std::string& filename, const std::string& newname, int suppress_error = 0); + ///< it will silently overwrite newname if it exists without returning an error + /// Posix guarantees that if newname already exists, then there will be no moment + /// in which for other processes newname does not exist. There is no such guarantee + /// under Windows at this time. It may do it in the same way but the used Windows API + /// does not make such guarantees. + /// @returns 0 on success and -1 on failure. + + + /// copy the contents of file from 'from' to 'to' filename static bool copy(const std::string& from, const std::string& to); - - static int stat(const std::string& filename,llstat* file_status); - static bool isdir(const std::string& filename); - static bool isfile(const std::string& filename); - static LLFILE * _Fiopen(const std::string& filename, - std::ios::openmode mode); - + ///< @returns true on success and false on failure. + + /// return the file stat structure for filename + static int stat(const std::string& filename, llstat* file_status, int suppress_error = ENOENT); + ///< for compatibility with existing uses of LL_File::stat() we use ENOENT as default in the + /// optional 'suppress_error' parameter to avoid spamming the log with warnings when the API + /// is used to detect if a file exists + /// @returns 0 on success and -1 on failure. + + /// get the file or directory attributes for filename + static unsigned short getattr(const std::string& filename, bool dontFollowSymLink = false, int suppress_error = ENOENT); + ///< a more lightweight function on Windows to stat, that just returns the file attribute flags + /// dontFollowSymLinks set to true returns the attributes of the symlink if it is one, rather than resolving it + /// we pass by default ENOENT in the optional 'suppress_error' parameter to not spam the log with + /// warnings when the file or directory does not exist + /// @returns 0 on failure and a st_mode value with either S_IFDIR or S_IFREG set otherwise + /// together with the three access bits which under Windows only the write bit is relevant. + + /// check if filename is an existing directory + static bool isdir(const std::string& filename); + ///< @returns true if the path is for an existing directory + + /// check if filename is an existing file + static bool isfile(const std::string& filename); + ///< @returns true if the path is for an existing file + + /// check if filename is a symlink + static bool islink(const std::string& filename); + ///< @returns true if the path is pointing at a symlink + + /// return a path to the temporary directory on the system static const char * tmpdir(); }; diff --git a/indra/llcommon/llleap.cpp b/indra/llcommon/llleap.cpp index 662a2511cd..ada6b9519e 100644 --- a/indra/llcommon/llleap.cpp +++ b/indra/llcommon/llleap.cpp @@ -188,6 +188,17 @@ public: << childout.peek(0, peeklen) << "..." << LL_ENDL; } + // Handle any remaining stderr data (partial lines) the same way as we do + // for stdout: log it. + LLProcess::ReadPipe& childerr(mChild->getReadPipe(LLProcess::STDERR)); + if (childerr.size()) + { + LLProcess::ReadPipe::size_type + peeklen((std::min)(LLProcess::ReadPipe::size_type(50), childerr.size())); + LL_WARNS("LLLeap") << "Final stderr " << childerr.size() << " bytes: " + << childerr.peek(0, peeklen) << "..." << LL_ENDL; + } + // Kill this instance. MUST BE LAST before return! delete this; return false; diff --git a/indra/llcommon/llmd5.cpp b/indra/llcommon/llmd5.cpp index e999b8f597..c8ca586e7f 100644 --- a/indra/llcommon/llmd5.cpp +++ b/indra/llcommon/llmd5.cpp @@ -255,6 +255,11 @@ void LLMD5::raw_digest(unsigned char* s) const memcpy(s, digest, 16); /* Flawfinder: ignore */ } +#if LL_DARWIN +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + void LLMD5::hex_digest(char* s) const { if (!finalized) @@ -273,6 +278,10 @@ void LLMD5::hex_digest(char* s) const s[32] = '\0'; } +#if LL_DARWIN +#pragma clang diagnostic pop +#endif + std::ostream& operator<<(std::ostream& stream, const LLMD5& context) { char s[33]; /* Flawfinder: ignore */ diff --git a/indra/llcommon/llmemory.h b/indra/llcommon/llmemory.h index d5802b9d95..c06ae93766 100644 --- a/indra/llcommon/llmemory.h +++ b/indra/llcommon/llmemory.h @@ -235,8 +235,6 @@ inline void* ll_aligned_malloc_32(size_t size) // returned hunk MUST be freed wi LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; #if defined(LL_WINDOWS) void* ret = _aligned_malloc(size, 32); -#elif defined(LL_DARWIN) - void* ret = ll_aligned_malloc_fallback( size, 32 ); #else void *ret; if (0 != posix_memalign(&ret, 32, size)) @@ -252,8 +250,31 @@ inline void ll_aligned_free_32(void *p) LL_PROFILE_FREE(p); #if defined(LL_WINDOWS) _aligned_free(p); -#elif defined(LL_DARWIN) - ll_aligned_free_fallback( p ); +#else + free(p); // posix_memalign() is compatible with heap deallocator +#endif +} + +inline void* ll_aligned_malloc_64(size_t size) // returned hunk MUST be freed with ll_aligned_free_32(). +{ + LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; +#if defined(LL_WINDOWS) + void* ret = _aligned_malloc(size, 64); +#else + void *ret; + if (0 != posix_memalign(&ret, 64, size)) + return nullptr; +#endif + LL_PROFILE_ALLOC(ret, size); + return ret; +} + +inline void ll_aligned_free_64(void *p) +{ + LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; + LL_PROFILE_FREE(p); +#if defined(LL_WINDOWS) + _aligned_free(p); #else free(p); // posix_memalign() is compatible with heap deallocator #endif @@ -265,19 +286,23 @@ LL_FORCE_INLINE void* ll_aligned_malloc(size_t size) { LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; void* ret; - if (LL_DEFAULT_HEAP_ALIGN % ALIGNMENT == 0) + if constexpr (LL_DEFAULT_HEAP_ALIGN % ALIGNMENT == 0) { ret = malloc(size); LL_PROFILE_ALLOC(ret, size); } - else if (ALIGNMENT == 16) + else if constexpr (ALIGNMENT == 16) { ret = ll_aligned_malloc_16(size); } - else if (ALIGNMENT == 32) + else if constexpr (ALIGNMENT == 32) { ret = ll_aligned_malloc_32(size); } + else if constexpr (ALIGNMENT == 64) + { + ret = ll_aligned_malloc_64(size); + } else { ret = ll_aligned_malloc_fallback(size, ALIGNMENT); @@ -289,16 +314,20 @@ template<size_t ALIGNMENT> LL_FORCE_INLINE void ll_aligned_free(void* ptr) { LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; - if (ALIGNMENT == LL_DEFAULT_HEAP_ALIGN) + if constexpr (ALIGNMENT == LL_DEFAULT_HEAP_ALIGN) { LL_PROFILE_FREE(ptr); free(ptr); } - else if (ALIGNMENT == 16) + else if constexpr (ALIGNMENT == 16) { ll_aligned_free_16(ptr); } - else if (ALIGNMENT == 32) + else if constexpr (ALIGNMENT == 32) + { + return ll_aligned_free_32(ptr); + } + else if constexpr (ALIGNMENT == 64) { return ll_aligned_free_32(ptr); } @@ -314,6 +343,9 @@ LL_FORCE_INLINE void ll_aligned_free(void* ptr) inline void ll_memcpy_nonaliased_aligned_16(char* __restrict dst, const char* __restrict src, size_t bytes) { LL_PROFILE_ZONE_SCOPED_CATEGORY_MEMORY; +#if defined(LL_ARM64) + memcpy(dst, src, bytes); +#else assert(src != NULL); assert(dst != NULL); assert(bytes > 0); @@ -379,6 +411,7 @@ inline void ll_memcpy_nonaliased_aligned_16(char* __restrict dst, const char* __ dst += 16; src += 16; } +#endif } #ifndef __DEBUG_PRIVATE_MEM__ diff --git a/indra/llcommon/llpreprocessor.h b/indra/llcommon/llpreprocessor.h index 268109e8b7..7e2d0b47f7 100644 --- a/indra/llcommon/llpreprocessor.h +++ b/indra/llcommon/llpreprocessor.h @@ -78,6 +78,23 @@ #endif #endif +// Set up CPU architecture defines +#if LL_MSVC && defined(_M_ARM64) +# define LL_ARM64 1 +#elif LL_GNUC && (defined(__arm64__) || defined(__aarch64__)) +# define LL_ARM64 1 +#elif LL_MSVC && _M_X64 +# define LL_X86_64 1 +# define LL_X86 1 +#elif LL_MSVC && _M_IX86 +# define LL_X86 1 +#elif LL_GNUC && ( defined(__amd64__) || defined(__x86_64__) ) +# define LL_X86_64 1 +# define LL_X86 1 +#elif LL_GNUC && ( defined(__i386__) ) +# define LL_X86 1 +#endif + // Deal with minor differences on Unixy OSes. #if LL_DARWIN || LL_LINUX || __FreeBSD__ // Different name, same functionality. @@ -120,11 +137,8 @@ #if LL_WINDOWS #define LL_DLLEXPORT __declspec(dllexport) #define LL_DLLIMPORT __declspec(dllimport) -#elif LL_LINUX -#define LL_DLLEXPORT __attribute__ ((visibility("default"))) -#define LL_DLLIMPORT #else -#define LL_DLLEXPORT +#define LL_DLLEXPORT __attribute__ ((visibility("default"))) #define LL_DLLIMPORT #endif // LL_WINDOWS @@ -169,11 +183,7 @@ #define LL_TO_STRING_HELPER(x) #x #define LL_TO_STRING(x) LL_TO_STRING_HELPER(x) -#if _M_ARM64 #define LL_TO_WSTRING_HELPER(x) L## #x -#else -#define LL_TO_WSTRING_HELPER(x) L#x -#endif #define LL_TO_WSTRING(x) LL_TO_WSTRING_HELPER(x) #define LL_FILE_LINENO_MSG(msg) __FILE__ "(" LL_TO_STRING(__LINE__) ") : " msg #define LL_GLUE_IMPL(x, y) x##y @@ -193,4 +203,18 @@ #define LL_PRETTY_FUNCTION __PRETTY_FUNCTION__ #endif +#ifndef _M_ARM64 +#if LL_ARM64 +#define GLM_FORCE_NEON 1 +#else +#define GLM_FORCE_SSE2 1 +#endif +#endif + +#if LL_ARM64 +#define KDU_NEON_INTRINSICS 1 +#else +#define KDU_X86_INTRINSICS 1 +#endif + #endif // not LL_LINDEN_PREPROCESSOR_H diff --git a/indra/llcommon/llprocessor.cpp b/indra/llcommon/llprocessor.cpp index 434501af77..2b6e4be1c8 100644 --- a/indra/llcommon/llprocessor.cpp +++ b/indra/llcommon/llprocessor.cpp @@ -40,9 +40,6 @@ # include <intrin.h> # undef _interlockedbittestandset # undef _interlockedbittestandreset -#if _M_ARM64 -# include <cpuinfo.h> -#endif #endif #include "llsd.h" @@ -510,9 +507,7 @@ private: // the other three array elements. The CPU identification string is // not in linear order. The code below arranges the information // in a human readable form. -#if _M_ARM64 - cpuinfo_initialize(); -#else +#if !_M_ARM64 int cpu_info[4] = {-1}; __cpuid(cpu_info, 0); unsigned int ids = (unsigned int)cpu_info[0]; @@ -521,44 +516,7 @@ private: char cpu_vendor[0x20]; memset(cpu_vendor, 0, sizeof(cpu_vendor)); -#if _M_ARM64 - switch (cpuinfo_get_current_core()->vendor) - { - case cpuinfo_vendor_unknown: - setInfo(eVendor, "Unknown"); - break; - case cpuinfo_vendor_arm: - setInfo(eVendor, "ARM"); - break; - case cpuinfo_vendor_qualcomm: - setInfo(eVendor, "Qualcomm"); - break; - case cpuinfo_vendor_apple: - setInfo(eVendor, "Apple"); - break; - case cpuinfo_vendor_samsung: - setInfo(eVendor, "Samsung"); - break; - case cpuinfo_vendor_nvidia: - setInfo(eVendor, "Nvidia"); - break; - case cpuinfo_vendor_cavium: - setInfo(eVendor, "Cavium"); - break; - case cpuinfo_vendor_broadcom: - setInfo(eVendor, "Broadcom"); - break; - case cpuinfo_vendor_apm: - setInfo(eVendor, "APM"); - break; - case cpuinfo_vendor_huawei: - setInfo(eVendor, "Huawei"); - break; - default: - setInfo(eVendor, "Unknown"); - break; - } -#else +#if !_M_ARM64 *((int*)cpu_vendor) = cpu_info[1]; *((int*)(cpu_vendor+4)) = cpu_info[3]; *((int*)(cpu_vendor+8)) = cpu_info[2]; @@ -572,8 +530,18 @@ private: } #if _M_ARM64 - setInfo(eModel, cpuinfo_get_package(0)->name); - setInfo(eType, cpuinfo_get_uarch(0)->uarch); + HKEY hKey; + DWORD gotType; + char inBuffer[48] = ""; + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", 0, KEY_READ, &hKey) == ERROR_SUCCESS) + { + if (!RegQueryValueExA(hKey, "ProcessorNameString", nullptr, &gotType, (PBYTE)(inBuffer), nullptr)) + { + if ((gotType == REG_SZ) && strlen(inBuffer)) + setInfo(eModel, inBuffer); + } + RegCloseKey(hKey); + } #else // Get the information associated with each valid Id for(unsigned int i=0; i<=ids; ++i) @@ -701,7 +669,8 @@ public: { getCPUIDInfo(); uint64_t frequency = getSysctlInt64("hw.cpufrequency"); - if (!frequency) { + if (!frequency) + { auto tbfrequency = getSysctlInt64("hw.tbfrequency"); struct clockinfo clockrate; auto clockrate_len = sizeof(clockrate); @@ -711,14 +680,14 @@ public: setInfo(eFrequency, (F64)frequency / (F64)1000000); } - virtual ~LLProcessorInfoDarwinImpl() {} + virtual ~LLProcessorInfoDarwinImpl() = default; private: int getSysctlInt(const char* name) { int result = 0; size_t len = sizeof(int); - int error = sysctlbyname(name, (void*)&result, &len, NULL, 0); + int error = sysctlbyname(name, (void*)&result, &len, nullptr, 0); return error == -1 ? 0 : result; } @@ -726,7 +695,7 @@ private: { uint64_t value = 0; size_t size = sizeof(value); - int result = sysctlbyname(name, (void*)&value, &size, NULL, 0); + int result = sysctlbyname(name, (void*)&value, &size, nullptr, 0); if ( result == 0 ) { if ( size == sizeof( uint64_t ) ) diff --git a/indra/llcommon/llprocessor.h b/indra/llcommon/llprocessor.h index f8ccf686c8..b955f7f55c 100644 --- a/indra/llcommon/llprocessor.h +++ b/indra/llcommon/llprocessor.h @@ -28,20 +28,7 @@ #ifndef LLPROCESSOR_H #define LLPROCESSOR_H #include "llunits.h" - -#if LL_MSVC && _M_X64 -# define LL_X86_64 1 -# define LL_X86 1 -#elif LL_MSVC && _M_IX86 -# define LL_X86 1 -#elif LL_GNUC && ( defined(__amd64__) || defined(__x86_64__) ) -# define LL_X86_64 1 -# define LL_X86 1 -#elif LL_GNUC && ( defined(__i386__) ) -# define LL_X86 1 -#elif LL_GNUC && ( defined(__powerpc__) || defined(__ppc__) ) -# define LL_PPC 1 -#endif +#include "llpreprocessor.h" class LLProcessorInfoImpl; diff --git a/indra/llcommon/llprofiler.h b/indra/llcommon/llprofiler.h index f6a4d24747..fa94f78019 100644 --- a/indra/llcommon/llprofiler.h +++ b/indra/llcommon/llprofiler.h @@ -71,26 +71,21 @@ #define LL_PROFILER_CONFIG_TRACY_FAST_TIMER 3 // Profiling on: Fast Timers + Tracy #ifndef LL_PROFILER_CONFIGURATION -#define LL_PROFILER_CONFIGURATION LL_PROFILER_CONFIG_FAST_TIMER +#define LL_PROFILER_CONFIGURATION 1 #endif -extern thread_local bool gProfilerEnabled; - #if defined(LL_PROFILER_CONFIGURATION) && (LL_PROFILER_CONFIGURATION > LL_PROFILER_CONFIG_NONE) #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY || LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY_FAST_TIMER #include "tracy/Tracy.hpp" - // Enable OpenGL profiling - #define LL_PROFILER_ENABLE_TRACY_OPENGL 0 - // Enable RenderDoc labeling - #define LL_PROFILER_ENABLE_RENDER_DOC 0 + //#define LL_PROFILER_ENABLE_RENDER_DOC 0 #endif #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY #define LL_PROFILER_FRAME_END FrameMark - #define LL_PROFILER_SET_THREAD_NAME( name ) tracy::SetThreadName( name ); gProfilerEnabled = true; + #define LL_PROFILER_SET_THREAD_NAME( name ) tracy::SetThreadName( name ); #define LL_RECORD_BLOCK_TIME(name) ZoneScoped // Want descriptive names; was: ZoneNamedN( ___tracy_scoped_zone, #name, true ); #define LL_PROFILE_ZONE_NAMED(name) ZoneNamedN( ___tracy_scoped_zone, name, true ); #define LL_PROFILE_ZONE_NAMED_COLOR(name,color) ZoneNamedNC( ___tracy_scopped_zone, name, color, true ) // RGB @@ -133,7 +128,7 @@ extern thread_local bool gProfilerEnabled; #endif #if LL_PROFILER_CONFIGURATION == LL_PROFILER_CONFIG_TRACY_FAST_TIMER #define LL_PROFILER_FRAME_END FrameMark - #define LL_PROFILER_SET_THREAD_NAME( name ) tracy::SetThreadName( name ); gProfilerEnabled = true; + #define LL_PROFILER_SET_THREAD_NAME( name ) tracy::SetThreadName( name ); #define LL_RECORD_BLOCK_TIME(name) ZoneNamedN(___tracy_scoped_zone, #name, true); const LLTrace::BlockTimer& LL_GLUE_TOKENS(block_time_recorder, __LINE__)(LLTrace::timeThisBlock(name)); (void)LL_GLUE_TOKENS(block_time_recorder, __LINE__); #define LL_PROFILE_ZONE_NAMED(name) ZoneNamedN( ___tracy_scoped_zone, #name, true ); #define LL_PROFILE_ZONE_NAMED_COLOR(name,color) ZoneNamedNC( ___tracy_scopped_zone, name, color, true ) // RGB @@ -155,26 +150,45 @@ extern thread_local bool gProfilerEnabled; #else #define LL_PROFILER_FRAME_END #define LL_PROFILER_SET_THREAD_NAME( name ) (void)(name) + + #define LL_PROFILER_FRAME_END + #define LL_PROFILER_SET_THREAD_NAME( name ) (void)(name) + #define LL_RECORD_BLOCK_TIME(name) + #define LL_PROFILE_ZONE_NAMED(name) // LL_PROFILE_ZONE_NAMED is a no-op when Tracy is disabled + #define LL_PROFILE_ZONE_NAMED_COLOR(name,color) // LL_PROFILE_ZONE_NAMED_COLOR is a no-op when Tracy is disabled + #define LL_PROFILE_ZONE_SCOPED // LL_PROFILE_ZONE_SCOPED is a no-op when Tracy is disabled + #define LL_PROFILE_ZONE_COLOR(name,color) // LL_RECORD_BLOCK_TIME(name) + + #define LL_PROFILE_ZONE_NUM( val ) (void)( val ); // Not supported + #define LL_PROFILE_ZONE_TEXT( text, size ) (void)( text ); void( size ); // Not supported + + #define LL_PROFILE_ZONE_ERR(name) (void)(name); // Not supported + #define LL_PROFILE_ZONE_INFO(name) (void)(name); // Not supported + #define LL_PROFILE_ZONE_WARN(name) (void)(name); // Not supported + + #define LL_PROFILE_MUTEX(type, varname) type varname + #define LL_PROFILE_MUTEX_NAMED(type, varname, desc) type varname + #define LL_PROFILE_MUTEX_SHARED(type, varname) type varname + #define LL_PROFILE_MUTEX_SHARED_NAMED(type, varname, desc) type varname + #define LL_PROFILE_MUTEX_LOCK(varname) // LL_PROFILE_MUTEX_LOCK is a no-op when Tracy is disabled + #endif // LL_PROFILER #if LL_PROFILER_ENABLE_TRACY_OPENGL -#define LL_PROFILE_GPU_ZONE(name) TracyGpuZone(name) -#define LL_PROFILE_GPU_ZONEC(name,color) TracyGpuZoneC(name,color) +#define LL_PROFILE_GPU_ZONE(name) TracyGpuZone(name) +#define LL_PROFILE_GPU_ZONEC(name,color) TracyGpuZoneC(name,color) #define LL_PROFILER_GPU_COLLECT TracyGpuCollect #define LL_PROFILER_GPU_CONTEXT TracyGpuContext - -// disable memory tracking (incompatible with GPU tracing -#define LL_PROFILE_ALLOC(ptr, size) (void)(ptr); (void)(size); -#define LL_PROFILE_FREE(ptr) (void)(ptr); +#define LL_PROFILER_GPU_CONTEXT_NAMED TracyGpuContextName #else -#define LL_PROFILE_GPU_ZONE(name) (void)name; -#define LL_PROFILE_GPU_ZONEC(name,color) (void)name;(void)color; +#define LL_PROFILE_GPU_ZONE(name) (void)name; +#define LL_PROFILE_GPU_ZONEC(name,color) (void)name;(void)color; #define LL_PROFILER_GPU_COLLECT #define LL_PROFILER_GPU_CONTEXT +#define LL_PROFILER_GPU_CONTEXT_NAMED(name) (void)name; +#endif // LL_PROFILER_ENABLE_TRACY_OPENGL -#define LL_LABEL_OBJECT_GL(type, name, length, label) - -#if !LL_DARWIN && LL_PROFILER_CONFIGURATION > 1 +#if LL_PROFILER_CONFIGURATION >= LL_PROFILER_CONFIG_TRACY #define LL_PROFILE_ALLOC(ptr, size) TracyAlloc(ptr, size) #define LL_PROFILE_FREE(ptr) TracyFree(ptr) #else @@ -182,8 +196,6 @@ extern thread_local bool gProfilerEnabled; #define LL_PROFILE_FREE(ptr) (void)(ptr); #endif -#endif - #if LL_PROFILER_ENABLE_RENDER_DOC #define LL_LABEL_OBJECT_GL(type, name, length, label) glObjectLabel(type, name, length, label) #else diff --git a/indra/llcommon/llqueuedthread.cpp b/indra/llcommon/llqueuedthread.cpp index 0196a24b18..efeeb1340e 100644 --- a/indra/llcommon/llqueuedthread.cpp +++ b/indra/llcommon/llqueuedthread.cpp @@ -80,7 +80,7 @@ void LLQueuedThread::shutdown() mRequestQueue.close(); } - S32 timeout = 100; + S32 timeout = 50; for ( ; timeout>0; timeout--) { if (isStopped()) @@ -101,19 +101,34 @@ void LLQueuedThread::shutdown() } QueuedRequest* req; - S32 active_count = 0; + S32 queued_count = 0; + bool has_active = false; + lockData(); while ( (req = (QueuedRequest*)mRequestHash.pop_element()) ) { - if (req->getStatus() == STATUS_QUEUED || req->getStatus() == STATUS_INPROGRESS) + if (req->getStatus() == STATUS_INPROGRESS) + { + has_active = true; + req->setFlags(FLAG_ABORT | FLAG_AUTO_COMPLETE); + continue; + } + if (req->getStatus() == STATUS_QUEUED) { - ++active_count; + ++queued_count; req->setStatus(STATUS_ABORTED); // avoid assert in deleteRequest } req->deleteRequest(); } - if (active_count) + unlockData(); + if (queued_count) + { + LL_WARNS() << "~LLQueuedThread() called with unpocessed requests: " << queued_count << LL_ENDL; + } + if (has_active) { - LL_WARNS() << "~LLQueuedThread() called with active requests: " << active_count << LL_ENDL; + LL_WARNS() << "~LLQueuedThread() called with active requests!" << LL_ENDL; + ms_sleep(100); // last chance for request to finish + printQueueStats(); } mRequestQueue.close(); @@ -570,7 +585,12 @@ LLQueuedThread::QueuedRequest::QueuedRequest(LLQueuedThread::handle_t handle, U3 LLQueuedThread::QueuedRequest::~QueuedRequest() { - llassert_always(mStatus == STATUS_DELETE); + if (mStatus != STATUS_DELETE) + { + // The only method to delete a request is deleteRequest(), + // it should have set the status to STATUS_DELETE + LL_ERRS() << "LLQueuedThread::QueuedRequest deleted with status " << mStatus << LL_ENDL; + } } //virtual diff --git a/indra/llcommon/llsdutil.h b/indra/llcommon/llsdutil.h index c31030c5ea..497c0ad3eb 100644 --- a/indra/llcommon/llsdutil.h +++ b/indra/llcommon/llsdutil.h @@ -554,6 +554,45 @@ LLSD shallow(LLSD value, LLSD filter=LLSD()) { return llsd_shallow(value, filter } // namespace llsd /***************************************************************************** +* LLSDParam<std::vector<T>> +*****************************************************************************/ +// Given an LLSD array, return a const std::vector<T>&, where T is a type +// supported by LLSDParam. Bonus: if the LLSD value is actually a scalar, +// return a single-element vector containing the converted value. +template <typename T> +class LLSDParam<std::vector<T>>: public LLSDParamBase +{ +public: + LLSDParam(const LLSD& array) + { + // treat undefined "array" as empty vector + if (array.isDefined()) + { + // what if it's a scalar? + if (! array.isArray()) + { + v.push_back(LLSDParam<T>(array)); + } + else // really is an array + { + // reserve space for the array entries + v.reserve(array.size()); + for (const auto& item : llsd::inArray(array)) + { + v.push_back(LLSDParam<T>(item)); + } + } + } + } + + operator const std::vector<T>&() const { return v; } + +private: + std::vector<T> v; +}; + + +/***************************************************************************** * toArray(), toMap() *****************************************************************************/ namespace llsd diff --git a/indra/llcommon/llsys.cpp b/indra/llcommon/llsys.cpp index 5c35aeb476..edc891f5ec 100644 --- a/indra/llcommon/llsys.cpp +++ b/indra/llcommon/llsys.cpp @@ -197,6 +197,9 @@ LLOSInfo::LLOSInfo() : GetSystemInfo(&si); //if it fails get regular system info //(Warning: If GetSystemInfo it may result in incorrect information in a WOW64 machine, if the kernel fails to load) +#pragma warning(push) +#pragma warning(disable : 4996) // ignore 'deprecated.' GetVersionEx is deprecated + // Try calling GetVersionEx using the OSVERSIONINFOEX structure. OSVERSIONINFOEX osvi; ZeroMemory(&osvi, sizeof(OSVERSIONINFOEX)); @@ -215,6 +218,8 @@ LLOSInfo::LLOSInfo() : } } +#pragma warning(pop) + S32 ubr = 0; // Windows 10 Update Build Revision, can be retrieved from a registry if (mMajorVer == 10) { @@ -1332,7 +1337,7 @@ bool gunzip_file(const std::string& srcfile, const std::string& dstfile) S32 bytes = 0; tmpfile = dstfile + ".t"; #ifdef LL_WINDOWS - llutf16string utf16filename = utf8str_to_utf16str(srcfile); + std::wstring utf16filename = ll_convert<std::wstring>(srcfile); src = gzopen_w(utf16filename.c_str(), "rb"); #else src = gzopen(srcfile.c_str(), "rb"); @@ -1352,10 +1357,6 @@ bool gunzip_file(const std::string& srcfile, const std::string& dstfile) } while(gzeof(src) == 0); fclose(dst); dst = NULL; -#if LL_WINDOWS - // Rename in windows needs the dstfile to not exist. - LLFile::remove(dstfile, ENOENT); -#endif if (LLFile::rename(tmpfile, dstfile) == -1) goto err; /* Flawfinder: ignore */ retval = true; err: @@ -1376,7 +1377,7 @@ bool gzip_file(const std::string& srcfile, const std::string& dstfile) tmpfile = dstfile + ".t"; #ifdef LL_WINDOWS - llutf16string utf16filename = utf8str_to_utf16str(tmpfile); + std::wstring utf16filename = ll_convert<std::wstring>(tmpfile); dst = gzopen_w(utf16filename.c_str(), "wb"); #else dst = gzopen(tmpfile.c_str(), "wb"); @@ -1403,10 +1404,6 @@ bool gzip_file(const std::string& srcfile, const std::string& dstfile) gzclose(dst); dst = NULL; -#if LL_WINDOWS - // Rename in windows needs the dstfile to not exist. - LLFile::remove(dstfile); -#endif if (LLFile::rename(tmpfile, dstfile) == -1) goto err; /* Flawfinder: ignore */ retval = true; err: diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index e5d25b52f0..692941a892 100644 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -28,6 +28,7 @@ #include "apr_portable.h" +#include "llapp.h" #include "llthread.h" #include "llmutex.h" @@ -35,6 +36,7 @@ #include "lltrace.h" #include "lltracethreadrecorder.h" #include "llexception.h" +#include "workqueue.h" #if LL_LINUX #include <sched.h> @@ -106,6 +108,27 @@ namespace return s_thread_id; } +#if LL_WINDOWS + + static const U32 STATUS_MSC_EXCEPTION = 0xE06D7363; // compiler specific + + U32 exception_filter(U32 code, struct _EXCEPTION_POINTERS* exception_infop) + { + if (LLApp::instance()->reportCrashToBugsplat((void*)exception_infop)) + { + // Handled + return EXCEPTION_CONTINUE_SEARCH; + } + else if (code == STATUS_MSC_EXCEPTION) + { + // C++ exception, go on + return EXCEPTION_CONTINUE_SEARCH; + } + + // handle it, convert to std::exception + return EXCEPTION_EXECUTE_HANDLER; + } +#endif // LL_WINDOWS } // anonymous namespace LL_COMMON_API bool on_main_thread() @@ -157,20 +180,11 @@ void LLThread::threadRun() // Run the user supplied function do { - try - { - run(); - } - catch (const LLContinueError &e) - { - LL_WARNS("THREAD") << "ContinueException on thread '" << mName << - "' reentering run(). Error what is: '" << e.what() << "'" << LL_ENDL; - //output possible call stacks to log file. - LLError::LLCallStacks::print(); - - LOG_UNHANDLED_EXCEPTION("LLThread"); - continue; - } +#ifdef LL_WINDOWS + sehHandle(); // Structured Exception Handling +#else + tryRun(); +#endif break; } while (true); @@ -188,6 +202,69 @@ void LLThread::threadRun() mStatus = STOPPED; } +void LLThread::tryRun() +{ + try + { + run(); + } + catch (const LLContinueError& e) + { + LL_WARNS("THREAD") << "ContinueException on thread '" << mName << + "'. Error what is: '" << e.what() << "'" << LL_ENDL; + LLError::LLCallStacks::print(); + + LOG_UNHANDLED_EXCEPTION("LLThread"); + } + catch (std::bad_alloc&) + { + // Todo: improve this, this is going to have a different callstack + // instead of showing where it crashed + LL_WARNS("THREAD") << "Out of memory in a thread: " << mName << LL_ENDL; + + LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); + main_queue->post( + // Bind the current exception, rethrow it in main loop. + []() { + LLError::LLUserWarningMsg::showOutOfMemory(); + LL_ERRS("THREAD") << "Out of memory in a thread" << LL_ENDL; + }); + } +#ifndef LL_WINDOWS + catch (...) + { + // Stash any other kind of uncaught exception to be rethrown by main thread. + LL_WARNS("THREAD") << "Capturing and rethrowing uncaught exception in LLThread " + << mName << LL_ENDL; + + LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); + main_queue->post( + // Bind the current exception, rethrow it in main loop. + [exc = std::current_exception()]() { std::rethrow_exception(exc); }); + } +#endif // else LL_WINDOWS +} + +#ifdef LL_WINDOWS +void LLThread::sehHandle() +{ + __try + { + // handle stop and continue exceptions first + tryRun(); + } + __except (exception_filter(GetExceptionCode(), GetExceptionInformation())) + { + // convert to C++ styled exception + // Note: it might be better to use _se_set_translator + // if you want exception to inherit full callstack + char integer_string[512]; + sprintf(integer_string, "SEH, code: %lu\n", GetExceptionCode()); + throw std::exception(integer_string); + } +} +#endif + LLThread::LLThread(const std::string& name, apr_pool_t *poolp) : mPaused(false), mName(name), diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index 4194e0014d..8794ac93aa 100644 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -68,7 +68,7 @@ public: // Called from MAIN THREAD. void pause(); void unpause(); - bool isPaused() { return isStopped() || mPaused; } + bool isPaused() const { return isStopped() || mPaused; } // Cause the thread to wake up and check its condition void wake(); @@ -97,6 +97,11 @@ private: // static function passed to APR thread creation routine void threadRun(); + void tryRun(); + +#ifdef LL_WINDOWS + void sehHandle(); +#endif protected: std::string mName; diff --git a/indra/llcommon/llthreadsafequeue.h b/indra/llcommon/llthreadsafequeue.h index 85fcbbacac..6532439dd9 100644 --- a/indra/llcommon/llthreadsafequeue.h +++ b/indra/llcommon/llthreadsafequeue.h @@ -452,11 +452,9 @@ ElementT LLThreadSafeQueue<ElementT, QueueT>::pop(void) // so we can finish draining the queue. pop_result popped = pop_(lock1, value); if (popped == POPPED) -#if LL_LINUX return value; -#else - return std::move(value); -#endif + // don't use std::move when returning local value because + // it prevents the compiler from optimizing with copy elision // Once the queue is DONE, there will never be any more coming. if (popped == DONE) diff --git a/indra/llcommon/tests/llprocess_test.cpp b/indra/llcommon/tests/llprocess_test.cpp index b63cc52bec..13422612d6 100644 --- a/indra/llcommon/tests/llprocess_test.cpp +++ b/indra/llcommon/tests/llprocess_test.cpp @@ -54,14 +54,29 @@ std::string apr_strerror_helper(apr_status_t rv) *****************************************************************************/ #define ensure_equals_(left, right) \ - ensure_equals(STRINGIZE(#left << " != " << #right), (left), (right)) +do { \ + auto _left_val = (left); \ + auto _right_val = (right); \ + if (_left_val != _right_val) { \ + std::string _msg = std::string(#left) + " != " + std::string(#right); \ + tut::ensure_equals(_msg, _left_val, _right_val); \ + } else { \ + tut::ensure_equals("", _left_val, _right_val); \ + } \ +} while(0) #define aprchk(expr) aprchk_(#expr, (expr)) static void aprchk_(const char* call, apr_status_t rv, apr_status_t expected=APR_SUCCESS) { - tut::ensure_equals(STRINGIZE(call << " => " << rv << ": " << apr_strerror_helper - (rv)), - rv, expected); + if (rv != expected) + { + std::string msg = std::string(call) + " => " + std::to_string(rv) + ": " + apr_strerror_helper(rv); + tut::ensure_equals(msg, rv, expected); + } + else + { + tut::ensure_equals("", rv, expected); + } } /** @@ -78,11 +93,14 @@ static std::string readfile(const std::string& pathname, const std::string& desc std::string use_desc(desc); if (use_desc.empty()) { - use_desc = STRINGIZE("in " << pathname); + use_desc = "in " + pathname; } std::ifstream inf(pathname.c_str()); std::string output; - tut::ensure(STRINGIZE("No output " << use_desc), bool(std::getline(inf, output))); + if (!std::getline(inf, output)) + { + tut::ensure("No output " + use_desc, false); + } std::string more; while (std::getline(inf, more)) { @@ -108,8 +126,8 @@ void waitfor(LLProcess& proc, int timeout=60) { yield(); } - tut::ensure(STRINGIZE("process took longer than " << timeout << " seconds to terminate"), - i < timeout); + std::string msg = "process took longer than " + std::to_string(timeout) + " seconds to terminate"; + tut::ensure(msg, i < timeout); } void waitfor(LLProcess::handle h, const std::string& desc, int timeout=60) @@ -119,8 +137,8 @@ void waitfor(LLProcess::handle h, const std::string& desc, int timeout=60) { yield(); } - tut::ensure(STRINGIZE("process took longer than " << timeout << " seconds to terminate"), - i < timeout); + std::string msg = "process took longer than " + std::to_string(timeout) + " seconds to terminate"; + tut::ensure(msg, i < timeout); } /** @@ -153,7 +171,8 @@ struct PythonProcessLauncher try { mPy = LLProcess::create(mParams); - tut::ensure(STRINGIZE("Couldn't launch " << mDesc << " script"), bool(mPy)); + std::string msg = "Couldn't launch " + mDesc + " script"; + tut::ensure(msg, bool(mPy)); } catch (const tut::failure&) { @@ -214,7 +233,8 @@ struct PythonProcessLauncher mParams.args.add(out.getName()); run(); // assuming the script wrote to that file, read it - return readfile(out.getName(), STRINGIZE("from " << mDesc << " script")); + std::string desc = "from " + mDesc + " script"; + return readfile(out.getName(), desc); } LLProcess::Params mParams; diff --git a/indra/llcommon/tests/stringize_test.cpp b/indra/llcommon/tests/stringize_test.cpp index a3ce7f8e3d..3f99041592 100644 --- a/indra/llcommon/tests/stringize_test.cpp +++ b/indra/llcommon/tests/stringize_test.cpp @@ -110,6 +110,17 @@ namespace tut void stringize_object::test<3>() { //Tests rely on validity of wstring_to_utf8str() +#if LL_WINDOWS // Windows wstring is a 2byte UTF16 type + ensure_equals(ll_convert<std::string>(wstringize(c)), ll_convert<std::string>(std::wstring(L"c"))); + ensure_equals(ll_convert<std::string>(wstringize(s)), ll_convert<std::string>(std::wstring(L"17"))); + ensure_equals(ll_convert<std::string>(wstringize(i)), ll_convert<std::string>(std::wstring(L"34"))); + ensure_equals(ll_convert<std::string>(wstringize(l)), ll_convert<std::string>(std::wstring(L"68"))); + ensure_equals(ll_convert<std::string>(wstringize(f)), ll_convert<std::string>(std::wstring(L"3.14159"))); + ensure_equals(ll_convert<std::string>(wstringize(d)), ll_convert<std::string>(std::wstring(L"3.14159"))); + ensure_equals(ll_convert<std::string>(wstringize(abc)), ll_convert<std::string>(std::wstring(L"abc def"))); + ensure_equals(ll_convert<std::string>(wstringize(abc)), ll_convert<std::string>(wstringize(abc.c_str()))); + ensure_equals(ll_convert<std::string>(wstringize(def)), ll_convert<std::string>(std::wstring(L"def ghi"))); +#else ensure_equals(wstring_to_utf8str(wstringize(c)), wstring_to_utf8str(L"c")); ensure_equals(wstring_to_utf8str(wstringize(s)), wstring_to_utf8str(L"17")); ensure_equals(wstring_to_utf8str(wstringize(i)), wstring_to_utf8str(L"34")); @@ -119,6 +130,7 @@ namespace tut ensure_equals(wstring_to_utf8str(wstringize(abc)), wstring_to_utf8str(L"abc def")); ensure_equals(wstring_to_utf8str(wstringize(abc)), wstring_to_utf8str(wstringize(abc.c_str()))); ensure_equals(wstring_to_utf8str(wstringize(def)), wstring_to_utf8str(L"def ghi")); +#endif // ensure_equals(wstring_to_utf8str(wstringize(llsd)), wstring_to_utf8str(L"{'abc':'abc def','d':r3.14159,'i':i34}")); } } // namespace tut diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h index 0eb1891754..ac4f415f3e 100644 --- a/indra/llcommon/threadpool.h +++ b/indra/llcommon/threadpool.h @@ -55,7 +55,7 @@ namespace LL * ThreadPool listens for application shutdown messages on the "LLApp" * LLEventPump. Call close() to shut down this ThreadPool early. */ - virtual void close(); + void close(); std::string getName() const { return mName; } size_t getWidth() const { return mThreads.size(); } @@ -122,7 +122,7 @@ namespace LL size_t threads=1, size_t capacity=1024*1024, bool auto_shutdown = true): - ThreadPoolBase(name, threads, new queue_t(name, capacity), auto_shutdown) + ThreadPoolBase(name, threads, new queue_t(name, capacity, false), auto_shutdown) {} ~ThreadPoolUsing() override {} diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index c8ece616b2..7efaebd569 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -21,6 +21,7 @@ #include "llcoros.h" #include LLCOROS_MUTEX_HEADER #include "llerror.h" +#include "llevents.h" #include "llexception.h" #include "stringize.h" @@ -30,11 +31,38 @@ using Lock = LLCoros::LockType; /***************************************************************************** * WorkQueueBase *****************************************************************************/ -LL::WorkQueueBase::WorkQueueBase(const std::string& name): - super(makeName(name)) +LL::WorkQueueBase::WorkQueueBase(const std::string& name, bool auto_shutdown) + : super(makeName(name)) { - // TODO: register for "LLApp" events so we can implicitly close() on - // viewer shutdown. + if (auto_shutdown) + { + // Register for "LLApp" events so we can implicitly close() on viewer shutdown + std::string listener_name = "WorkQueue:" + getKey(); + LLEventPumps::instance().obtain("LLApp").listen( + listener_name, + [this](const LLSD& stat) + { + std::string status(stat["status"]); + if (status != "running") + { + // Viewer is shutting down, close this queue + LL_DEBUGS("WorkQueue") << getKey() << " closing on app shutdown" << LL_ENDL; + close(); + } + return false; + }); + + // Store the listener name so we can unregister in the destructor + mListenerName = listener_name; + } +} + +LL::WorkQueueBase::~WorkQueueBase() +{ + if (!mListenerName.empty() && !LLEventPumps::wasDeleted()) + { + LLEventPumps::instance().obtain("LLApp").stopListening(mListenerName); + } } void LL::WorkQueueBase::runUntilClose() @@ -182,14 +210,22 @@ void LL::WorkQueueBase::callWork(const Work& work) } catch (...) { - // Stash any other kind of uncaught exception to be rethrown by main thread. - LL_WARNS("LLCoros") << "Capturing and rethrowing uncaught exception in WorkQueueBase " - << getKey() << LL_ENDL; - - LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); - main_queue->post( - // Bind the current exception, rethrow it in main loop. - [exc = std::current_exception()]() { std::rethrow_exception(exc); }); + if (getKey() != "mainloop") + { + // Stash any other kind of uncaught exception to be rethrown by main thread. + LL_WARNS("LLCoros") << "Capturing and rethrowing uncaught exception in WorkQueueBase " + << getKey() << LL_ENDL; + + LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); + main_queue->post( + // Bind the current exception, rethrow it in main loop. + [exc = std::current_exception()]() { std::rethrow_exception(exc); }); + } + else + { + // let main loop crash + throw; + } } #endif // else LL_WINDOWS } @@ -212,8 +248,8 @@ void LL::WorkQueueBase::checkCoroutine(const std::string& method) /***************************************************************************** * WorkQueue *****************************************************************************/ -LL::WorkQueue::WorkQueue(const std::string& name, size_t capacity): - super(name), +LL::WorkQueue::WorkQueue(const std::string& name, size_t capacity, bool auto_shutdown): + super(name, auto_shutdown), mQueue(capacity) { } @@ -261,8 +297,8 @@ bool LL::WorkQueue::tryPop_(Work& work) /***************************************************************************** * WorkSchedule *****************************************************************************/ -LL::WorkSchedule::WorkSchedule(const std::string& name, size_t capacity): - super(name), +LL::WorkSchedule::WorkSchedule(const std::string& name, size_t capacity, bool auto_shutdown): + super(name, auto_shutdown), mQueue(capacity) { } diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 9d7bbfbf7a..735ad38a26 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -51,7 +51,9 @@ namespace LL * You may omit the WorkQueueBase name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkQueueBase(const std::string& name); + WorkQueueBase(const std::string& name, bool auto_shutdown); + + virtual ~WorkQueueBase(); /** * Since the point of WorkQueue is to pass work to some other worker @@ -197,6 +199,9 @@ namespace LL private: virtual Work pop_() = 0; virtual bool tryPop_(Work&) = 0; + + // Name used for the LLApp event listener (empty if not registered) + std::string mListenerName; }; /***************************************************************************** @@ -212,7 +217,7 @@ namespace LL * You may omit the WorkQueue name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkQueue(const std::string& name = std::string(), size_t capacity=1024); + WorkQueue(const std::string& name = std::string(), size_t capacity=1024, bool auto_shutdown = true); /** * Since the point of WorkQueue is to pass work to some other worker @@ -282,7 +287,7 @@ namespace LL * You may omit the WorkSchedule name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkSchedule(const std::string& name = std::string(), size_t capacity=1024); + WorkSchedule(const std::string& name = std::string(), size_t capacity=1024, bool auto_shutdown = true); /** * Since the point of WorkSchedule is to pass work to some other worker |