From 40f7501319087291c8b9881095b4b35f0dcf0554 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 22 Nov 2016 08:35:41 -0500 Subject: DRTVWR-418: Use uintptr_t when casting pointers to ints. LLPrivateMemoryPool and LLPrivateMemoryPoolManager have assumed that it's always valid to cast a pointer to U32. With 64-bit pointers, no longer true. --- indra/llcommon/llmemory.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'indra/llcommon/llmemory.cpp') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 3a8eabac09..1e04044269 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -591,7 +591,7 @@ char* LLPrivateMemoryPool::LLMemoryBlock::allocate() void LLPrivateMemoryPool::LLMemoryBlock::freeMem(void* addr) { //bit index - U32 idx = ((U32)addr - (U32)mBuffer - mDummySize) / mSlotSize ; + uintptr_t idx = ((uintptr_t)addr - (uintptr_t)mBuffer - mDummySize) / mSlotSize ; U32* bits = &mUsageBits ; if(idx >= 32) @@ -773,7 +773,7 @@ char* LLPrivateMemoryPool::LLMemoryChunk::allocate(U32 size) void LLPrivateMemoryPool::LLMemoryChunk::freeMem(void* addr) { - U32 blk_idx = getPageIndex((U32)addr) ; + U32 blk_idx = getPageIndex((uintptr_t)addr) ; LLMemoryBlock* blk = (LLMemoryBlock*)(mMetaBuffer + blk_idx * sizeof(LLMemoryBlock)) ; blk = blk->mSelf ; @@ -798,7 +798,7 @@ bool LLPrivateMemoryPool::LLMemoryChunk::empty() bool LLPrivateMemoryPool::LLMemoryChunk::containsAddress(const char* addr) const { - return (U32)mBuffer <= (U32)addr && (U32)mBuffer + mBufferSize > (U32)addr ; + return (uintptr_t)mBuffer <= (uintptr_t)addr && (uintptr_t)mBuffer + mBufferSize > (uintptr_t)addr ; } //debug use @@ -831,13 +831,13 @@ void LLPrivateMemoryPool::LLMemoryChunk::dump() for(U32 i = 1 ; i < blk_list.size(); i++) { total_size += blk_list[i]->getBufferSize() ; - if((U32)blk_list[i]->getBuffer() < (U32)blk_list[i-1]->getBuffer() + blk_list[i-1]->getBufferSize()) + if((uintptr_t)blk_list[i]->getBuffer() < (uintptr_t)blk_list[i-1]->getBuffer() + blk_list[i-1]->getBufferSize()) { LL_ERRS() << "buffer corrupted." << LL_ENDL ; } } - llassert_always(total_size + mMinBlockSize >= mBufferSize - ((U32)mDataBuffer - (U32)mBuffer)) ; + llassert_always(total_size + mMinBlockSize >= mBufferSize - ((uintptr_t)mDataBuffer - (uintptr_t)mBuffer)) ; U32 blk_num = (mBufferSize - (mDataBuffer - mBuffer)) / mMinBlockSize ; for(U32 i = 0 ; i < blk_num ; ) @@ -860,7 +860,7 @@ void LLPrivateMemoryPool::LLMemoryChunk::dump() #endif #if 0 LL_INFOS() << "---------------------------" << LL_ENDL ; - LL_INFOS() << "Chunk buffer: " << (U32)getBuffer() << " size: " << getBufferSize() << LL_ENDL ; + LL_INFOS() << "Chunk buffer: " << (uintptr_t)getBuffer() << " size: " << getBufferSize() << LL_ENDL ; LL_INFOS() << "available blocks ... " << LL_ENDL ; for(S32 i = 0 ; i < mBlockLevels ; i++) @@ -868,7 +868,7 @@ void LLPrivateMemoryPool::LLMemoryChunk::dump() LLMemoryBlock* blk = mAvailBlockList[i] ; while(blk) { - LL_INFOS() << "blk buffer " << (U32)blk->getBuffer() << " size: " << blk->getBufferSize() << LL_ENDL ; + LL_INFOS() << "blk buffer " << (uintptr_t)blk->getBuffer() << " size: " << blk->getBufferSize() << LL_ENDL ; blk = blk->mNext ; } } @@ -879,7 +879,7 @@ void LLPrivateMemoryPool::LLMemoryChunk::dump() LLMemoryBlock* blk = mFreeSpaceList[i] ; while(blk) { - LL_INFOS() << "blk buffer " << (U32)blk->getBuffer() << " size: " << blk->getBufferSize() << LL_ENDL ; + LL_INFOS() << "blk buffer " << (uintptr_t)blk->getBuffer() << " size: " << blk->getBufferSize() << LL_ENDL ; blk = blk->mNext ; } } @@ -1155,9 +1155,9 @@ void LLPrivateMemoryPool::LLMemoryChunk::addToAvailBlockList(LLMemoryBlock* blk) return ; } -U32 LLPrivateMemoryPool::LLMemoryChunk::getPageIndex(U32 addr) +U32 LLPrivateMemoryPool::LLMemoryChunk::getPageIndex(uintptr_t addr) { - return (addr - (U32)mDataBuffer) / mMinBlockSize ; + return (addr - (uintptr_t)mDataBuffer) / mMinBlockSize ; } //for mAvailBlockList @@ -1495,7 +1495,7 @@ void LLPrivateMemoryPool::removeChunk(LLMemoryChunk* chunk) U16 LLPrivateMemoryPool::findHashKey(const char* addr) { - return (((U32)addr) / CHUNK_SIZE) % mHashFactor ; + return (((uintptr_t)addr) / CHUNK_SIZE) % mHashFactor ; } LLPrivateMemoryPool::LLMemoryChunk* LLPrivateMemoryPool::findChunk(const char* addr) @@ -1720,7 +1720,7 @@ LLPrivateMemoryPoolManager::~LLPrivateMemoryPoolManager() S32 k = 0 ; for(mem_allocation_info_t::iterator iter = sMemAllocationTracker.begin() ; iter != sMemAllocationTracker.end() ; ++iter) { - LL_INFOS() << k++ << ", " << (U32)iter->first << " : " << iter->second << LL_ENDL ; + LL_INFOS() << k++ << ", " << (uintptr_t)iter->first << " : " << iter->second << LL_ENDL ; } sMemAllocationTracker.clear() ; } -- cgit v1.2.3 From 52899ed62a241d7875277b0f3412e2be78f7b3af Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 2 May 2017 10:51:18 -0400 Subject: DRTVWR-418, MAINT-6996: Rationalize LLMemory wrt 64-bit support. There were two distinct LLMemory methods getCurrentRSS() and getWorkingSetSize(). It was pointless to have both: on Windows they were completely redundant; on other platforms getWorkingSetSize() always returned 0. (Amusingly, though the Windows implementations both made exactly the same GetProcessMemoryInfo() call and used exactly the same logic, the code was different in the two -- as though the second was implemented without awareness of the first, even though they were adjacent in the source file.) One of the actual MAINT-6996 problems was due to the fact that getWorkingSetSize() returned U32, where getCurrentRSS() returns U64. In other words, getWorkingSetSize() was both useless *and* wrong. Remove it, and change its one call to getCurrentRSS() instead. The other culprit was that in several places, the 64-bit WorkingSetSize returned by the Windows GetProcessMemoryInfo() call (and by getCurrentRSS()) was explicitly cast to a 32-bit data type. That works only when explicitly or implicitly (using LLUnits type conversion) scaling the value to kilobytes or megabytes. When the size in bytes is desired, use 64-bit types instead. In addition to the symptoms, LLMemory was overdue for a bit of cleanup. There was a 16K block of memory called reserveMem, the comment on which read: "reserve 16K for out of memory error handling." Yet *nothing* was ever done with that block! If it were going to be useful, one would think someone would at some point explicitly free the block. In fact there was a method freeReserve(), apparently for just that purpose -- which was never called. As things stood, reserveMem served only to *prevent* the viewer from ever using that chunk of memory. Remove reserveMem and the unused freeReserve(). The only function of initClass() and cleanupClass() was to allocate and free reserveMem. Remove initClass(), cleanupClass() and the LLCommon calls to them. In a similar vein, there was an LLMemoryInfo::getPhysicalMemoryClamped() method that returned U32Bytes. Its job was simply to return a size in bytes that could fit into a U32 data type, returning U32_MAX if the 64-bit value exceeded 4GB. Eliminate that; change all its calls to getPhysicalMemoryKB() (which getPhysicalMemoryClamped() used internally anyway). We no longer care about any platform that cannot handle 64-bit data types. --- indra/llcommon/llmemory.cpp | 89 ++++++++------------------------------------- 1 file changed, 15 insertions(+), 74 deletions(-) (limited to 'indra/llcommon/llmemory.cpp') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 1e04044269..6a7e1362f0 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -44,10 +44,10 @@ #include "llsys.h" #include "llframetimer.h" #include "lltrace.h" +#include "llerror.h" //---------------------------------------------------------------------------- //static -char* LLMemory::reserveMem = 0; U32Kilobytes LLMemory::sAvailPhysicalMemInKB(U32_MAX); U32Kilobytes LLMemory::sMaxPhysicalMemInKB(0); static LLTrace::SampleStatHandle sAllocatedMem("allocated_mem", "active memory in use by application"); @@ -78,29 +78,6 @@ void ll_assert_aligned_func(uintptr_t ptr,U32 alignment) #endif } -//static -void LLMemory::initClass() -{ - if (!reserveMem) - { - reserveMem = new char[16*1024]; // reserve 16K for out of memory error handling - } -} - -//static -void LLMemory::cleanupClass() -{ - delete [] reserveMem; - reserveMem = NULL; -} - -//static -void LLMemory::freeReserve() -{ - delete [] reserveMem; - reserveMem = NULL; -} - //static void LLMemory::initMaxHeapSizeGB(F32Gigabytes max_heap_size, BOOL prevent_heap_failure) { @@ -111,19 +88,18 @@ void LLMemory::initMaxHeapSizeGB(F32Gigabytes max_heap_size, BOOL prevent_heap_f //static void LLMemory::updateMemoryInfo() { -#if LL_WINDOWS - HANDLE self = GetCurrentProcess(); +#if LL_WINDOWS PROCESS_MEMORY_COUNTERS counters; - - if (!GetProcessMemoryInfo(self, &counters, sizeof(counters))) + + if (!GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters))) { LL_WARNS() << "GetProcessMemoryInfo failed" << LL_ENDL; return ; } - sAllocatedMemInKB = (U32Bytes)(counters.WorkingSetSize) ; + sAllocatedMemInKB = U64Bytes(counters.WorkingSetSize) ; sample(sAllocatedMem, sAllocatedMemInKB); - sAllocatedPageSizeInKB = (U32Bytes)(counters.PagefileUsage) ; + sAllocatedPageSizeInKB = U64Bytes(counters.PagefileUsage) ; sample(sVirtualMem, sAllocatedPageSizeInKB); U32Kilobytes avail_phys, avail_virtual; @@ -140,9 +116,9 @@ void LLMemory::updateMemoryInfo() } #else //not valid for other systems for now. - sAllocatedMemInKB = (U32Bytes)LLMemory::getCurrentRSS(); - sMaxPhysicalMemInKB = (U32Bytes)U32_MAX ; - sAvailPhysicalMemInKB = (U32Bytes)U32_MAX ; + sAllocatedMemInKB = U64Bytes(LLMemory::getCurrentRSS()); + sMaxPhysicalMemInKB = U64Bytes(U32_MAX); + sAvailPhysicalMemInKB = U64Bytes(U32_MAX); #endif return ; @@ -169,7 +145,7 @@ void* LLMemory::tryToAlloc(void* address, U32 size) return address ; #else return (void*)0x01 ; //skip checking -#endif +#endif } //static @@ -183,7 +159,7 @@ void LLMemory::logMemoryInfo(BOOL update) LL_INFOS() << "Current allocated physical memory(KB): " << sAllocatedMemInKB << LL_ENDL ; LL_INFOS() << "Current allocated page size (KB): " << sAllocatedPageSizeInKB << LL_ENDL ; - LL_INFOS() << "Current availabe physical memory(KB): " << sAvailPhysicalMemInKB << LL_ENDL ; + LL_INFOS() << "Current available physical memory(KB): " << sAvailPhysicalMemInKB << LL_ENDL ; LL_INFOS() << "Current max usable memory(KB): " << sMaxPhysicalMemInKB << LL_ENDL ; LL_INFOS() << "--- private pool information -- " << LL_ENDL ; @@ -263,12 +239,12 @@ U32Kilobytes LLMemory::getAllocatedMemKB() #if defined(LL_WINDOWS) +//static U64 LLMemory::getCurrentRSS() { - HANDLE self = GetCurrentProcess(); PROCESS_MEMORY_COUNTERS counters; - - if (!GetProcessMemoryInfo(self, &counters, sizeof(counters))) + + if (!GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters))) { LL_WARNS() << "GetProcessMemoryInfo failed" << LL_ENDL; return 0; @@ -277,20 +253,6 @@ U64 LLMemory::getCurrentRSS() return counters.WorkingSetSize; } -//static -U32 LLMemory::getWorkingSetSize() -{ - PROCESS_MEMORY_COUNTERS pmc ; - U32 ret = 0 ; - - if (GetProcessMemoryInfo( GetCurrentProcess(), &pmc, sizeof(pmc)) ) - { - ret = pmc.WorkingSetSize ; - } - - return ret ; -} - #elif defined(LL_DARWIN) /* @@ -337,11 +299,6 @@ U64 LLMemory::getCurrentRSS() return residentSize; } -U32 LLMemory::getWorkingSetSize() -{ - return 0 ; -} - #elif defined(LL_LINUX) U64 LLMemory::getCurrentRSS() @@ -353,7 +310,7 @@ U64 LLMemory::getCurrentRSS() if (fp == NULL) { LL_WARNS() << "couldn't open " << statPath << LL_ENDL; - goto bail; + return 0; } // Eee-yew! See Documentation/filesystems/proc.txt in your @@ -372,15 +329,9 @@ U64 LLMemory::getCurrentRSS() fclose(fp); -bail: return rss; } -U32 LLMemory::getWorkingSetSize() -{ - return 0 ; -} - #elif LL_SOLARIS #include #include @@ -410,11 +361,6 @@ U64 LLMemory::getCurrentRSS() return((U64)proc_psinfo.pr_rssize * 1024); } -U32 LLMemory::getWorkingSetSize() -{ - return 0 ; -} - #else U64 LLMemory::getCurrentRSS() @@ -422,11 +368,6 @@ U64 LLMemory::getCurrentRSS() return 0; } -U32 LLMemory::getWorkingSetSize() -{ - return 0; -} - #endif //-------------------------------------------------------------------------------------------------- -- cgit v1.2.3 From 2bb19aec989d5964b22f64cc01aa7cbe962e1e6b Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 2 May 2017 11:05:13 -0400 Subject: DRTVWR-418, MAINT-6996: Update Mac LLMemory::getCurrentRSS(). Evidently the Mac implementation of LLMemory::getCurrentRSS() goes back to OS X 10.3, because there was a helpful comment of the form: ------ The API used here is not capable of dealing with 64-bit memory sizes, but is available before 10.4. Once we start requiring 10.4, we can use the updated API, which looks like this: [new current implementation] Of course, this doesn't gain us anything unless we start building the viewer as a 64-bit executable, since that's the only way for our memory allocation to exceed 2^32. ------ Hey, guess what, we're building 64-bit viewers now! Thank you, whoever thoughtfully noted that, both for calling out the issue and sparing us the research. (The comment goes back to Subversion days, so hg blame shows only the merge-to-release changeset.) --- indra/llcommon/llmemory.cpp | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) (limited to 'indra/llcommon/llmemory.cpp') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 6a7e1362f0..9f9c3af892 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -255,19 +255,6 @@ U64 LLMemory::getCurrentRSS() #elif defined(LL_DARWIN) -/* - The API used here is not capable of dealing with 64-bit memory sizes, but is available before 10.4. - - Once we start requiring 10.4, we can use the updated API, which looks like this: - - task_basic_info_64_data_t basicInfo; - mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_64_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) - - Of course, this doesn't gain us anything unless we start building the viewer as a 64-bit executable, since that's the only way - for our memory allocation to exceed 2^32. -*/ - // if (sysctl(ctl, 2, &page_size, &size, NULL, 0) == -1) // { // LL_WARNS() << "Couldn't get page size" << LL_ENDL; @@ -280,9 +267,9 @@ U64 LLMemory::getCurrentRSS() U64 LLMemory::getCurrentRSS() { U64 residentSize = 0; - task_basic_info_data_t basicInfo; - mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) + task_basic_info_64_data_t basicInfo; + mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_64_COUNT; + if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) { residentSize = basicInfo.resident_size; -- cgit v1.2.3 From 49cfd6d9917c7ba35869e33785cbacba7c5e2d98 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Thu, 4 May 2017 18:13:56 -0400 Subject: DRTVWR-418, MAINT-6996: On Mac, obtain total mem, not resident mem. The LLMemory method getCurrentRSS() is defined to return the "resident set size," but in fact on Windows it returns the WorkingSetSize -- and that's actually what callers want from it: the total memory consumed by the application for statistics purposes. It's not really clear what users gain by knowing how much of that is resident in real memory, versus the total consumption. So despite the commentation and the method name itself, on Mac make it return the virtual size consumed. --- indra/llcommon/llmemory.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'indra/llcommon/llmemory.cpp') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index 9f9c3af892..aac6c26d9a 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -271,12 +271,11 @@ U64 LLMemory::getCurrentRSS() mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_64_COUNT; if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) { - residentSize = basicInfo.resident_size; - - // If we ever wanted it, the process virtual size is also available as: - // virtualSize = basicInfo.virtual_size; - -// LL_INFOS() << "resident size is " << residentSize << LL_ENDL; +// residentSize = basicInfo.resident_size; + // Although this method is defined to return the "resident set size," + // in fact what callers want from it is the total virtual memory + // consumed by the application. + residentSize = basicInfo.virtual_size; } else { -- cgit v1.2.3 From 9fa131b088aeb17f3af9157184264ffa5c737610 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 10 May 2017 14:19:44 -0400 Subject: DRTVWR-418, MAINT-6996: Update Mac mem queries (per Drake Arconis) Drake points out that the OS X 64-bit-capable memory-query APIs recommended in comments by some long-ago maintainer are by now themselves obsolete. He offered this patch to update us to current macOS memory APIs. --- indra/llcommon/llmemory.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llcommon/llmemory.cpp') diff --git a/indra/llcommon/llmemory.cpp b/indra/llcommon/llmemory.cpp index aac6c26d9a..049e962638 100644 --- a/indra/llcommon/llmemory.cpp +++ b/indra/llcommon/llmemory.cpp @@ -267,9 +267,9 @@ U64 LLMemory::getCurrentRSS() U64 LLMemory::getCurrentRSS() { U64 residentSize = 0; - task_basic_info_64_data_t basicInfo; - mach_msg_type_number_t basicInfoCount = TASK_BASIC_INFO_64_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO_64, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) + mach_task_basic_info_data_t basicInfo; + mach_msg_type_number_t basicInfoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&basicInfo, &basicInfoCount) == KERN_SUCCESS) { // residentSize = basicInfo.resident_size; // Although this method is defined to return the "resident set size," -- cgit v1.2.3