diff options
author | Merov Linden <merov@lindenlab.com> | 2010-11-11 09:33:29 -0800 |
---|---|---|
committer | Merov Linden <merov@lindenlab.com> | 2010-11-11 09:33:29 -0800 |
commit | 94f44437d7e7ccb54f227e35523fa8a4e027d7d2 (patch) | |
tree | dd899aa6e98b2c10b22fa0db58cfca866a0322de | |
parent | ee8df375449e97476408417ec6dea0d6a8853d73 (diff) | |
parent | 0e07758b540374aa421d2938ce5ed13e2b9eefe4 (diff) |
STORM-151 : merge kdu static linking, add indra/llkdu implementation. Doesn't build yet.
35 files changed, 3615 insertions, 755 deletions
diff --git a/indra/cmake/Copy3rdPartyLibs.cmake b/indra/cmake/Copy3rdPartyLibs.cmake index e852cf463c..176ae9787e 100644 --- a/indra/cmake/Copy3rdPartyLibs.cmake +++ b/indra/cmake/Copy3rdPartyLibs.cmake @@ -60,22 +60,6 @@ if(WINDOWS) set(release_files ${release_files} fmod.dll) endif (FMOD) - #******************************* - # LLKDU - set(internal_llkdu_path "${CMAKE_SOURCE_DIR}/llkdu") - if(NOT EXISTS ${internal_llkdu_path}) - if (EXISTS "${debug_src_dir}/llkdu.dll") - set(debug_llkdu_src "${debug_src_dir}/llkdu.dll") - set(debug_llkdu_dst "${SHARED_LIB_STAGING_DIR_DEBUG}/llkdu.dll") - endif (EXISTS "${debug_src_dir}/llkdu.dll") - - if (EXISTS "${release_src_dir}/llkdu.dll") - set(release_llkdu_src "${release_src_dir}/llkdu.dll") - set(release_llkdu_dst "${SHARED_LIB_STAGING_DIR_RELEASE}/llkdu.dll") - set(relwithdebinfo_llkdu_dst "${SHARED_LIB_STAGING_DIR_RELWITHDEBINFO}/llkdu.dll") - endif (EXISTS "${release_src_dir}/llkdu.dll") - endif (NOT EXISTS ${internal_llkdu_path}) - #******************************* # Copy MS C runtime dlls, required for packaging. # *TODO - Adapt this to support VC9 @@ -174,21 +158,6 @@ elseif(DARWIN) # fmod is statically linked on darwin set(fmod_files "") - #******************************* - # LLKDU - set(internal_llkdu_path "${CMAKE_SOURCE_DIR}/llkdu") - if(NOT EXISTS ${internal_llkdu_path}) - if (EXISTS "${debug_src_dir}/libllkdu.dylib") - set(debug_llkdu_src "${debug_src_dir}/libllkdu.dylib") - set(debug_llkdu_dst "${SHARED_LIB_STAGING_DIR_DEBUG}/libllkdu.dylib") - endif (EXISTS "${debug_src_dir}/libllkdu.dylib") - - if (EXISTS "${release_src_dir}/libllkdu.dylib") - set(release_llkdu_src "${release_src_dir}/libllkdu.dylib") - set(release_llkdu_dst "${SHARED_LIB_STAGING_DIR_RELEASE}/libllkdu.dylib") - set(relwithdebinfo_llkdu_dst "${SHARED_LIB_STAGING_DIR_RELWITHDEBINFO}/libllkdu.dylib") - endif (EXISTS "${release_src_dir}/libllkdu.dylib") - endif (NOT EXISTS ${internal_llkdu_path}) elseif(LINUX) # linux is weird, multiple side by side configurations aren't supported # and we don't seem to have any debug shared libs built yet anyways... @@ -242,21 +211,6 @@ elseif(LINUX) set(release_files ${release_files} "libfmod-3.75.so") endif (FMOD) - #******************************* - # LLKDU - set(internal_llkdu_path "${CMAKE_SOURCE_DIR}/llkdu") - if(NOT EXISTS ${internal_llkdu_path}) - if (EXISTS "${debug_src_dir}/libllkdu.so") - set(debug_llkdu_src "${debug_src_dir}/libllkdu.so") - set(debug_llkdu_dst "${SHARED_LIB_STAGING_DIR_DEBUG}/libllkdu.so") - endif (EXISTS "${debug_src_dir}/libllkdu.so") - - if (EXISTS "${release_src_dir}/libllkdu.so") - set(release_llkdu_src "${release_src_dir}/libllkdu.so") - set(release_llkdu_dst "${SHARED_LIB_STAGING_DIR_RELEASE}/libllkdu.so") - set(relwithdebinfo_llkdu_dst "${SHARED_LIB_STAGING_DIR_RELWITHDEBINFO}/libllkdu.so") - endif (EXISTS "${release_src_dir}/libllkdu.so") - endif(NOT EXISTS ${internal_llkdu_path}) else(WINDOWS) message(STATUS "WARNING: unrecognized platform for staging 3rd party libs, skipping...") set(vivox_src_dir "${CMAKE_SOURCE_DIR}/newview/vivox-runtime/i686-linux") @@ -334,40 +288,29 @@ copy_if_different( ) set(third_party_targets ${third_party_targets} ${out_targets}) -#******************************* -# LLKDU -set(internal_llkdu_path "${CMAKE_SOURCE_DIR}/llkdu") -if(NOT EXISTS ${internal_llkdu_path}) - if (EXISTS "${debug_llkdu_src}") - ADD_CUSTOM_COMMAND( - OUTPUT ${debug_llkdu_dst} - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${debug_llkdu_src} ${debug_llkdu_dst} - DEPENDS ${debug_llkdu_src} - COMMENT "Copying llkdu.dll ${SHARED_LIB_STAGING_DIR_DEBUG}" - ) - set(third_party_targets ${third_party_targets} $} ${debug_llkdu_dst}) - endif (EXISTS "${debug_llkdu_src}") - - if (EXISTS "${release_llkdu_src}") - ADD_CUSTOM_COMMAND( - OUTPUT ${release_llkdu_dst} - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${release_llkdu_src} ${release_llkdu_dst} - DEPENDS ${release_llkdu_src} - COMMENT "Copying llkdu.dll ${SHARED_LIB_STAGING_DIR_RELEASE}" - ) - set(third_party_targets ${third_party_targets} ${release_llkdu_dst}) - - ADD_CUSTOM_COMMAND( - OUTPUT ${relwithdebinfo_llkdu_dst} - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${release_llkdu_src} ${relwithdebinfo_llkdu_dst} - DEPENDS ${release_llkdu_src} - COMMENT "Copying llkdu.dll ${SHARED_LIB_STAGING_DIR_RELWITHDEBINFO}" - ) - set(third_party_targets ${third_party_targets} ${relwithdebinfo_llkdu_dst}) - endif (EXISTS "${release_llkdu_src}") - -endif (NOT EXISTS ${internal_llkdu_path}) - +if (FMOD_SDK_DIR) + copy_if_different( + ${FMOD_SDK_DIR} + "${CMAKE_CURRENT_BINARY_DIR}/Debug" + out_targets + ${fmod_files} + ) + set(all_targets ${all_targets} ${out_targets}) + copy_if_different( + ${FMOD_SDK_DIR} + "${CMAKE_CURRENT_BINARY_DIR}/Release" + out_targets + ${fmod_files} + ) + set(all_targets ${all_targets} ${out_targets}) + copy_if_different( + ${FMOD_SDK_DIR} + "${CMAKE_CURRENT_BINARY_DIR}/RelWithDbgInfo" + out_targets + ${fmod_files} + ) + set(all_targets ${all_targets} ${out_targets}) +endif (FMOD_SDK_DIR) if(NOT STANDALONE) add_custom_target( diff --git a/indra/cmake/LLKDU.cmake b/indra/cmake/LLKDU.cmake index 27c8ada686..25703ee785 100644 --- a/indra/cmake/LLKDU.cmake +++ b/indra/cmake/LLKDU.cmake @@ -1,7 +1,24 @@ # -*- cmake -*- include(Prebuilt) +# USE_KDU can be set when launching cmake or develop.py as an option using the argument -DUSE_KDU:BOOL=ON +# When building using proprietary binaries though (i.e. having access to LL private servers), we always build with KDU if (INSTALL_PROPRIETARY AND NOT STANDALONE) + set(USE_KDU ON) +endif (INSTALL_PROPRIETARY AND NOT STANDALONE) + +if (USE_KDU) use_prebuilt_binary(kdu) + if (WINDOWS) + set(KDU_LIBRARY debug kdu_cored optimized kdu_core) + else (WINDOWS) + set(KDU_LIBRARY kdu) + endif (WINDOWS) + + set(KDU_INCLUDE_DIR ${LIBS_PREBUILT_DIR}/include) + set(LLKDU_LIBRARY llkdu) -endif (INSTALL_PROPRIETARY AND NOT STANDALONE) + set(LLKDU_STATIC_LIBRARY llkdu_static) + set(LLKDU_LIBRARIES ${LLKDU_LIBRARY}) + set(LLKDU_STATIC_LIBRARIES ${LLKDU_STATIC_LIBRARY}) +endif (USE_KDU) diff --git a/indra/integration_tests/llui_libtest/CMakeLists.txt b/indra/integration_tests/llui_libtest/CMakeLists.txt index 452d37d3be..2a00dbee6f 100644 --- a/indra/integration_tests/llui_libtest/CMakeLists.txt +++ b/indra/integration_tests/llui_libtest/CMakeLists.txt @@ -71,6 +71,8 @@ endif (DARWIN) target_link_libraries(llui_libtest llui llmessage + ${LLIMAGE_LIBRARIES} + ${LLIMAGEJ2COJ_LIBRARIES} ${OS_LIBRARIES} ${GOOGLE_PERFTOOLS_LIBRARIES} ) diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 7bad780dd8..478f2fedbd 100644 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -70,6 +70,7 @@ set(llcommon_SOURCE_FILES llmemorystream.cpp llmemtype.cpp llmetrics.cpp + llmetricperformancetester.cpp llmortician.cpp lloptioninterface.cpp llptrto.cpp @@ -186,6 +187,7 @@ set(llcommon_HEADER_FILES llmemorystream.h llmemtype.h llmetrics.h + llmetricperformancetester.h llmortician.h llnametable.h lloptioninterface.h diff --git a/indra/llcommon/llfasttimer_class.cpp b/indra/llcommon/llfasttimer_class.cpp index c45921cdec..bce87ada96 100644 --- a/indra/llcommon/llfasttimer_class.cpp +++ b/indra/llcommon/llfasttimer_class.cpp @@ -56,6 +56,7 @@ bool LLFastTimer::sPauseHistory = 0; bool LLFastTimer::sResetHistory = 0; LLFastTimer::CurTimerData LLFastTimer::sCurTimerData; BOOL LLFastTimer::sLog = FALSE; +std::string LLFastTimer::sLogName = ""; BOOL LLFastTimer::sMetricLog = FALSE; LLMutex* LLFastTimer::sLogLock = NULL; std::queue<LLSD> LLFastTimer::sLogQueue; diff --git a/indra/llcommon/llfasttimer_class.h b/indra/llcommon/llfasttimer_class.h index 1158ac5140..eb9789682b 100644 --- a/indra/llcommon/llfasttimer_class.h +++ b/indra/llcommon/llfasttimer_class.h @@ -211,6 +211,7 @@ public: static std::queue<LLSD> sLogQueue; static BOOL sLog; static BOOL sMetricLog; + static std::string sLogName; static bool sPauseHistory; static bool sResetHistory; static U64 sTimerCycles; diff --git a/indra/llcommon/llmetricperformancetester.cpp b/indra/llcommon/llmetricperformancetester.cpp new file mode 100644 index 0000000000..5fa3a5ea07 --- /dev/null +++ b/indra/llcommon/llmetricperformancetester.cpp @@ -0,0 +1,254 @@ +/** + * @file llmetricperformancetester.cpp + * @brief LLMetricPerformanceTesterBasic and LLMetricPerformanceTesterWithSession classes implementation + * + * $LicenseInfo:firstyear=2004&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "indra_constants.h" +#include "llerror.h" +#include "llsdserialize.h" +#include "llstat.h" +#include "lltreeiterators.h" +#include "llmetricperformancetester.h" + +//---------------------------------------------------------------------------------------------- +// LLMetricPerformanceTesterBasic : static methods and testers management +//---------------------------------------------------------------------------------------------- + +LLMetricPerformanceTesterBasic::name_tester_map_t LLMetricPerformanceTesterBasic::sTesterMap ; + +/*static*/ +void LLMetricPerformanceTesterBasic::cleanClass() +{ + for (name_tester_map_t::iterator iter = sTesterMap.begin() ; iter != sTesterMap.end() ; ++iter) + { + delete iter->second ; + } + sTesterMap.clear() ; +} + +/*static*/ +BOOL LLMetricPerformanceTesterBasic::addTester(LLMetricPerformanceTesterBasic* tester) +{ + llassert_always(tester != NULL); + std::string name = tester->getTesterName() ; + if (getTester(name)) + { + llerrs << "Tester name is already used by some other tester : " << name << llendl ; + return FALSE; + } + + sTesterMap.insert(std::make_pair(name, tester)); + return TRUE; +} + +/*static*/ +LLMetricPerformanceTesterBasic* LLMetricPerformanceTesterBasic::getTester(std::string name) +{ + // Check for the requested metric name + name_tester_map_t::iterator found_it = sTesterMap.find(name) ; + if (found_it != sTesterMap.end()) + { + return found_it->second ; + } + return NULL ; +} + +/*static*/ +// Return TRUE if this metric is requested or if the general default "catch all" metric is requested +BOOL LLMetricPerformanceTesterBasic::isMetricLogRequested(std::string name) +{ + return (LLFastTimer::sMetricLog && ((LLFastTimer::sLogName == name) || (LLFastTimer::sLogName == DEFAULT_METRIC_NAME))); +} + + +//---------------------------------------------------------------------------------------------- +// LLMetricPerformanceTesterBasic : Tester instance methods +//---------------------------------------------------------------------------------------------- + +LLMetricPerformanceTesterBasic::LLMetricPerformanceTesterBasic(std::string name) : + mName(name), + mCount(0) +{ + if (mName == std::string()) + { + llerrs << "LLMetricPerformanceTesterBasic construction invalid : Empty name passed to constructor" << llendl ; + } + + mValidInstance = LLMetricPerformanceTesterBasic::addTester(this) ; +} + +LLMetricPerformanceTesterBasic::~LLMetricPerformanceTesterBasic() +{ +} + +void LLMetricPerformanceTesterBasic::preOutputTestResults(LLSD* sd) +{ + incrementCurrentCount() ; + (*sd)[getCurrentLabelName()]["Name"] = mName ; +} + +void LLMetricPerformanceTesterBasic::postOutputTestResults(LLSD* sd) +{ + LLMutexLock lock(LLFastTimer::sLogLock); + LLFastTimer::sLogQueue.push((*sd)); +} + +void LLMetricPerformanceTesterBasic::outputTestResults() +{ + LLSD sd; + + preOutputTestResults(&sd) ; + outputTestRecord(&sd) ; + postOutputTestResults(&sd) ; +} + +void LLMetricPerformanceTesterBasic::addMetric(std::string str) +{ + mMetricStrings.push_back(str) ; +} + +/*virtual*/ +void LLMetricPerformanceTesterBasic::analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) +{ + resetCurrentCount() ; + + std::string current_label = getCurrentLabelName(); + BOOL in_base = (*base).has(current_label) ; + BOOL in_current = (*current).has(current_label) ; + + while(in_base || in_current) + { + LLSD::String label = current_label ; + + if(in_base && in_current) + { + *os << llformat("%s\n", label.c_str()) ; + + for(U32 index = 0 ; index < mMetricStrings.size() ; index++) + { + switch((*current)[label][ mMetricStrings[index] ].type()) + { + case LLSD::TypeInteger: + compareTestResults(os, mMetricStrings[index], + (S32)((*base)[label][ mMetricStrings[index] ].asInteger()), (S32)((*current)[label][ mMetricStrings[index] ].asInteger())) ; + break ; + case LLSD::TypeReal: + compareTestResults(os, mMetricStrings[index], + (F32)((*base)[label][ mMetricStrings[index] ].asReal()), (F32)((*current)[label][ mMetricStrings[index] ].asReal())) ; + break; + default: + llerrs << "unsupported metric " << mMetricStrings[index] << " LLSD type: " << (S32)(*current)[label][ mMetricStrings[index] ].type() << llendl ; + } + } + } + + incrementCurrentCount(); + current_label = getCurrentLabelName(); + in_base = (*base).has(current_label) ; + in_current = (*current).has(current_label) ; + } +} + +/*virtual*/ +void LLMetricPerformanceTesterBasic::compareTestResults(std::ofstream* os, std::string metric_string, S32 v_base, S32 v_current) +{ + *os << llformat(" ,%s, %d, %d, %d, %.4f\n", metric_string.c_str(), v_base, v_current, + v_current - v_base, (v_base != 0) ? 100.f * v_current / v_base : 0) ; +} + +/*virtual*/ +void LLMetricPerformanceTesterBasic::compareTestResults(std::ofstream* os, std::string metric_string, F32 v_base, F32 v_current) +{ + *os << llformat(" ,%s, %.4f, %.4f, %.4f, %.4f\n", metric_string.c_str(), v_base, v_current, + v_current - v_base, (fabs(v_base) > 0.0001f) ? 100.f * v_current / v_base : 0.f ) ; +} + +//---------------------------------------------------------------------------------------------- +// LLMetricPerformanceTesterWithSession +//---------------------------------------------------------------------------------------------- + +LLMetricPerformanceTesterWithSession::LLMetricPerformanceTesterWithSession(std::string name) : + LLMetricPerformanceTesterBasic(name), + mBaseSessionp(NULL), + mCurrentSessionp(NULL) +{ +} + +LLMetricPerformanceTesterWithSession::~LLMetricPerformanceTesterWithSession() +{ + if (mBaseSessionp) + { + delete mBaseSessionp ; + mBaseSessionp = NULL ; + } + if (mCurrentSessionp) + { + delete mCurrentSessionp ; + mCurrentSessionp = NULL ; + } +} + +/*virtual*/ +void LLMetricPerformanceTesterWithSession::analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) +{ + // Load the base session + resetCurrentCount() ; + mBaseSessionp = loadTestSession(base) ; + + // Load the current session + resetCurrentCount() ; + mCurrentSessionp = loadTestSession(current) ; + + if (!mBaseSessionp || !mCurrentSessionp) + { + llerrs << "Error loading test sessions." << llendl ; + } + + // Compare + compareTestSessions(os) ; + + // Release memory + if (mBaseSessionp) + { + delete mBaseSessionp ; + mBaseSessionp = NULL ; + } + if (mCurrentSessionp) + { + delete mCurrentSessionp ; + mCurrentSessionp = NULL ; + } +} + + +//---------------------------------------------------------------------------------------------- +// LLTestSession +//---------------------------------------------------------------------------------------------- + +LLMetricPerformanceTesterWithSession::LLTestSession::~LLTestSession() +{ +} + diff --git a/indra/llcommon/llmetricperformancetester.h b/indra/llcommon/llmetricperformancetester.h new file mode 100644 index 0000000000..925010ac96 --- /dev/null +++ b/indra/llcommon/llmetricperformancetester.h @@ -0,0 +1,206 @@ +/** + * @file llmetricperformancetester.h + * @brief LLMetricPerformanceTesterBasic and LLMetricPerformanceTesterWithSession classes definition + * + * $LicenseInfo:firstyear=2004&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_METRICPERFORMANCETESTER_H +#define LL_METRICPERFORMANCETESTER_H + +const std::string DEFAULT_METRIC_NAME("metric"); + +/** + * @class LLMetricPerformanceTesterBasic + * @brief Performance Metric Base Class + */ +class LL_COMMON_API LLMetricPerformanceTesterBasic +{ +public: + /** + * @brief Creates a basic tester instance. + * @param[in] name - Unique string identifying this tester instance. + */ + LLMetricPerformanceTesterBasic(std::string name); + virtual ~LLMetricPerformanceTesterBasic(); + + /** + * @return Returns true if the instance has been added to the tester map. + * Need to be tested after creation of a tester instance so to know if the tester is correctly handled. + * A tester might not be added to the map if another tester with the same name already exists. + */ + BOOL isValid() const { return mValidInstance; } + + /** + * @brief Write a set of test results to the log LLSD. + */ + void outputTestResults() ; + + /** + * @brief Compare the test results. + * By default, compares the test results against the baseline one by one, item by item, + * in the increasing order of the LLSD record counter, starting from the first one. + */ + virtual void analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) ; + + /** + * @return Returns the number of the test metrics in this tester instance. + */ + S32 getNumberOfMetrics() const { return mMetricStrings.size() ;} + /** + * @return Returns the metric name at index + * @param[in] index - Index on the list of metrics managed by this tester instance. + */ + std::string getMetricName(S32 index) const { return mMetricStrings[index] ;} + +protected: + /** + * @return Returns the name of this tester instance. + */ + std::string getTesterName() const { return mName ;} + + /** + * @brief Insert a new metric to be managed by this tester instance. + * @param[in] str - Unique string identifying the new metric. + */ + void addMetric(std::string str) ; + + /** + * @brief Compare test results, provided in 2 flavors: compare integers and compare floats. + * @param[out] os - Formatted output string holding the compared values. + * @param[in] metric_string - Name of the metric. + * @param[in] v_base - Base value of the metric. + * @param[in] v_current - Current value of the metric. + */ + virtual void compareTestResults(std::ofstream* os, std::string metric_string, S32 v_base, S32 v_current) ; + virtual void compareTestResults(std::ofstream* os, std::string metric_string, F32 v_base, F32 v_current) ; + + /** + * @brief Reset internal record count. Count starts with 1. + */ + void resetCurrentCount() { mCount = 1; } + /** + * @brief Increment internal record count. + */ + void incrementCurrentCount() { mCount++; } + /** + * @return Returns the label to be used for the current count. It's "TesterName"-"Count". + */ + std::string getCurrentLabelName() const { return llformat("%s-%d", mName.c_str(), mCount) ;} + + /** + * @brief Write a test record to the LLSD. Implementers need to overload this method. + * @param[out] sd - The LLSD record to store metric data into. + */ + virtual void outputTestRecord(LLSD* sd) = 0 ; + +private: + void preOutputTestResults(LLSD* sd) ; + void postOutputTestResults(LLSD* sd) ; + + std::string mName ; // Name of this tester instance + S32 mCount ; // Current record count + BOOL mValidInstance; // TRUE if the instance is managed by the map + std::vector< std::string > mMetricStrings ; // Metrics strings + +// Static members managing the collection of testers +public: + // Map of all the tester instances in use + typedef std::map< std::string, LLMetricPerformanceTesterBasic* > name_tester_map_t; + static name_tester_map_t sTesterMap ; + + /** + * @return Returns a pointer to the tester + * @param[in] name - Name of the tester instance queried. + */ + static LLMetricPerformanceTesterBasic* getTester(std::string name) ; + + /** + * @return Returns TRUE if that metric *or* the default catch all metric has been requested to be logged + * @param[in] name - Name of the tester queried. + */ + static BOOL isMetricLogRequested(std::string name); + + /** + * @return Returns TRUE if there's a tester defined, FALSE otherwise. + */ + static BOOL hasMetricPerformanceTesters() { return !sTesterMap.empty() ;} + /** + * @brief Delete all testers and reset the tester map + */ + static void cleanClass() ; + +private: + // Add a tester to the map. Returns false if adding fails. + static BOOL addTester(LLMetricPerformanceTesterBasic* tester) ; +}; + +/** + * @class LLMetricPerformanceTesterWithSession + * @brief Performance Metric Class with custom session + */ +class LL_COMMON_API LLMetricPerformanceTesterWithSession : public LLMetricPerformanceTesterBasic +{ +public: + /** + * @param[in] name - Unique string identifying this tester instance. + */ + LLMetricPerformanceTesterWithSession(std::string name); + virtual ~LLMetricPerformanceTesterWithSession(); + + /** + * @brief Compare the test results. + * This will be loading the base and current sessions and compare them using the virtual + * abstract methods loadTestSession() and compareTestSessions() + */ + virtual void analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) ; + +protected: + /** + * @class LLMetricPerformanceTesterWithSession::LLTestSession + * @brief Defines an interface for the two abstract virtual functions loadTestSession() and compareTestSessions() + */ + class LL_COMMON_API LLTestSession + { + public: + virtual ~LLTestSession() ; + }; + + /** + * @brief Convert an LLSD log into a test session. + * @param[in] log - The LLSD record + * @return Returns the record as a test session + */ + virtual LLMetricPerformanceTesterWithSession::LLTestSession* loadTestSession(LLSD* log) = 0; + + /** + * @brief Compare the base session and the target session. Assumes base and current sessions have been loaded. + * @param[out] os - The comparison result as a standard stream + */ + virtual void compareTestSessions(std::ofstream* os) = 0; + + LLTestSession* mBaseSessionp; + LLTestSession* mCurrentSessionp; +}; + +#endif + diff --git a/indra/llimage/CMakeLists.txt b/indra/llimage/CMakeLists.txt index a69621a57b..6834267d4b 100644 --- a/indra/llimage/CMakeLists.txt +++ b/indra/llimage/CMakeLists.txt @@ -57,7 +57,6 @@ add_library (llimage ${llimage_SOURCE_FILES}) # Sort by high-level to low-level target_link_libraries(llimage llcommon - llimagej2coj # *HACK: In theory a noop for KDU builds? ${JPEG_LIBRARIES} ${PNG_LIBRARIES} ${ZLIB_LIBRARIES} diff --git a/indra/llimage/llimage.cpp b/indra/llimage/llimage.cpp index 5c33b675ca..b46a99e030 100644 --- a/indra/llimage/llimage.cpp +++ b/indra/llimage/llimage.cpp @@ -52,13 +52,11 @@ LLMutex* LLImage::sMutex = NULL; void LLImage::initClass() { sMutex = new LLMutex(NULL); - LLImageJ2C::openDSO(); } //static void LLImage::cleanupClass() { - LLImageJ2C::closeDSO(); delete sMutex; sMutex = NULL; } diff --git a/indra/llimage/llimagej2c.cpp b/indra/llimage/llimagej2c.cpp index c8c866b7f2..cb2a85fa91 100644 --- a/indra/llimage/llimagej2c.cpp +++ b/indra/llimage/llimagej2c.cpp @@ -24,148 +24,32 @@ */ #include "linden_common.h" -#include "apr_pools.h" -#include "apr_dso.h" - #include "lldir.h" #include "llimagej2c.h" #include "llmemtype.h" +#include "lltimer.h" +#include "llmath.h" typedef LLImageJ2CImpl* (*CreateLLImageJ2CFunction)(); typedef void (*DestroyLLImageJ2CFunction)(LLImageJ2CImpl*); typedef const char* (*EngineInfoLLImageJ2CFunction)(); -//some "private static" variables so we only attempt to load -//dynamic libaries once -CreateLLImageJ2CFunction j2cimpl_create_func; -DestroyLLImageJ2CFunction j2cimpl_destroy_func; -EngineInfoLLImageJ2CFunction j2cimpl_engineinfo_func; -apr_pool_t *j2cimpl_dso_memory_pool; -apr_dso_handle_t *j2cimpl_dso_handle; - -//Declare the prototype for theses functions here, their functionality -//will be implemented in other files which define a derived LLImageJ2CImpl -//but only ONE static library which has the implementation for this -//function should ever be included +// Declare the prototype for theses functions here. Their functionality +// will be implemented in other files which define a derived LLImageJ2CImpl +// but only ONE static library which has the implementation for these +// functions should ever be included. LLImageJ2CImpl* fallbackCreateLLImageJ2CImpl(); void fallbackDestroyLLImageJ2CImpl(LLImageJ2CImpl* impl); const char* fallbackEngineInfoLLImageJ2CImpl(); -//static -//Loads the required "create", "destroy" and "engineinfo" functions needed -void LLImageJ2C::openDSO() -{ - //attempt to load a DSO and get some functions from it - std::string dso_name; - std::string dso_path; - - bool all_functions_loaded = false; - apr_status_t rv; - -#if LL_WINDOWS - dso_name = "llkdu.dll"; -#elif LL_DARWIN - dso_name = "libllkdu.dylib"; -#else - dso_name = "libllkdu.so"; -#endif - - dso_path = gDirUtilp->findFile(dso_name, - gDirUtilp->getAppRODataDir(), - gDirUtilp->getExecutableDir()); - - j2cimpl_dso_handle = NULL; - j2cimpl_dso_memory_pool = NULL; - - //attempt to load the shared library - apr_pool_create(&j2cimpl_dso_memory_pool, NULL); - rv = apr_dso_load(&j2cimpl_dso_handle, - dso_path.c_str(), - j2cimpl_dso_memory_pool); - - //now, check for success - if ( rv == APR_SUCCESS ) - { - //found the dynamic library - //now we want to load the functions we're interested in - CreateLLImageJ2CFunction create_func = NULL; - DestroyLLImageJ2CFunction dest_func = NULL; - EngineInfoLLImageJ2CFunction engineinfo_func = NULL; - - rv = apr_dso_sym((apr_dso_handle_sym_t*)&create_func, - j2cimpl_dso_handle, - "createLLImageJ2CKDU"); - if ( rv == APR_SUCCESS ) - { - //we've loaded the create function ok - //we need to delete via the DSO too - //so lets check for a destruction function - rv = apr_dso_sym((apr_dso_handle_sym_t*)&dest_func, - j2cimpl_dso_handle, - "destroyLLImageJ2CKDU"); - if ( rv == APR_SUCCESS ) - { - //we've loaded the destroy function ok - rv = apr_dso_sym((apr_dso_handle_sym_t*)&engineinfo_func, - j2cimpl_dso_handle, - "engineInfoLLImageJ2CKDU"); - if ( rv == APR_SUCCESS ) - { - //ok, everything is loaded alright - j2cimpl_create_func = create_func; - j2cimpl_destroy_func = dest_func; - j2cimpl_engineinfo_func = engineinfo_func; - all_functions_loaded = true; - } - } - } - } - - if ( !all_functions_loaded ) - { - //something went wrong with the DSO or function loading.. - //fall back onto our satefy impl creation function - -#if 0 - // precious verbose debugging, sadly we can't use our - // 'llinfos' stream etc. this early in the initialisation seq. - char errbuf[256]; - fprintf(stderr, "failed to load syms from DSO %s (%s)\n", - dso_name.c_str(), dso_path.c_str()); - apr_strerror(rv, errbuf, sizeof(errbuf)); - fprintf(stderr, "error: %d, %s\n", rv, errbuf); - apr_dso_error(j2cimpl_dso_handle, errbuf, sizeof(errbuf)); - fprintf(stderr, "dso-error: %d, %s\n", rv, errbuf); -#endif - - if ( j2cimpl_dso_handle ) - { - apr_dso_unload(j2cimpl_dso_handle); - j2cimpl_dso_handle = NULL; - } - - if ( j2cimpl_dso_memory_pool ) - { - apr_pool_destroy(j2cimpl_dso_memory_pool); - j2cimpl_dso_memory_pool = NULL; - } - } -} - -//static -void LLImageJ2C::closeDSO() -{ - if ( j2cimpl_dso_handle ) apr_dso_unload(j2cimpl_dso_handle); - if (j2cimpl_dso_memory_pool) apr_pool_destroy(j2cimpl_dso_memory_pool); -} +// Test data gathering handle +LLImageCompressionTester* LLImageJ2C::sTesterp = NULL ; +const std::string sTesterName("ImageCompressionTester"); //static std::string LLImageJ2C::getEngineInfo() { - if (!j2cimpl_engineinfo_func) - j2cimpl_engineinfo_func = fallbackEngineInfoLLImageJ2CImpl; - - return j2cimpl_engineinfo_func(); + return fallbackEngineInfoLLImageJ2CImpl(); } LLImageJ2C::LLImageJ2C() : LLImageFormatted(IMG_CODEC_J2C), @@ -175,47 +59,32 @@ LLImageJ2C::LLImageJ2C() : LLImageFormatted(IMG_CODEC_J2C), mReversible(FALSE), mAreaUsedForDataSizeCalcs(0) { - //We assume here that if we wanted to create via - //a dynamic library that the approriate open calls were made - //before any calls to this constructor. - - //Therefore, a NULL creation function pointer here means - //we either did not want to create using functions from the dynamic - //library or there were issues loading it, either way - //use our fall back - if ( !j2cimpl_create_func ) - { - j2cimpl_create_func = fallbackCreateLLImageJ2CImpl; - } - - mImpl = j2cimpl_create_func(); + mImpl = fallbackCreateLLImageJ2CImpl(); // Clear data size table for( S32 i = 0; i <= MAX_DISCARD_LEVEL; i++) { // Array size is MAX_DISCARD_LEVEL+1 mDataSizes[i] = 0; } + + // If that test log has ben requested but not yet created, create it + if (LLMetricPerformanceTesterBasic::isMetricLogRequested(sTesterName) && !LLMetricPerformanceTesterBasic::getTester(sTesterName)) + { + sTesterp = new LLImageCompressionTester() ; + if (!sTesterp->isValid()) + { + delete sTesterp; + sTesterp = NULL; + } + } } // virtual LLImageJ2C::~LLImageJ2C() { - //We assume here that if we wanted to destroy via - //a dynamic library that the approriate open calls were made - //before any calls to this destructor. - - //Therefore, a NULL creation function pointer here means - //we either did not want to destroy using functions from the dynamic - //library or there were issues loading it, either way - //use our fall back - if ( !j2cimpl_destroy_func ) - { - j2cimpl_destroy_func = fallbackDestroyLLImageJ2CImpl; - } - if ( mImpl ) { - j2cimpl_destroy_func(mImpl); + fallbackDestroyLLImageJ2CImpl(mImpl); } } @@ -280,6 +149,7 @@ BOOL LLImageJ2C::decode(LLImageRaw *raw_imagep, F32 decode_time) // Returns TRUE to mean done, whether successful or not. BOOL LLImageJ2C::decodeChannels(LLImageRaw *raw_imagep, F32 decode_time, S32 first_channel, S32 max_channel_count ) { + LLTimer elapsed; LLMemType mt1(mMemType); BOOL res = TRUE; @@ -318,6 +188,21 @@ BOOL LLImageJ2C::decodeChannels(LLImageRaw *raw_imagep, F32 decode_time, S32 fir LLImage::setLastError(mLastError); } + LLImageCompressionTester* tester = (LLImageCompressionTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) + { + // Decompression stat gathering + // Note that we *do not* take into account the decompression failures data so we might overestimate the time spent processing + + // Always add the decompression time to the stat + tester->updateDecompressionStats(elapsed.getElapsedTimeF32()) ; + if (res) + { + // The whole data stream is finally decompressed when res is returned as TRUE + tester->updateDecompressionStats(this->getDataSize(), raw_imagep->getDataSize()) ; + } + } + return res; } @@ -330,6 +215,7 @@ BOOL LLImageJ2C::encode(const LLImageRaw *raw_imagep, F32 encode_time) BOOL LLImageJ2C::encode(const LLImageRaw *raw_imagep, const char* comment_text, F32 encode_time) { + LLTimer elapsed; LLMemType mt1(mMemType); resetLastError(); BOOL res = mImpl->encodeImpl(*this, *raw_imagep, comment_text, encode_time, mReversible); @@ -337,6 +223,22 @@ BOOL LLImageJ2C::encode(const LLImageRaw *raw_imagep, const char* comment_text, { LLImage::setLastError(mLastError); } + + LLImageCompressionTester* tester = (LLImageCompressionTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) + { + // Compression stat gathering + // Note that we *do not* take into account the compression failures cases so we night overestimate the time spent processing + + // Always add the compression time to the stat + tester->updateCompressionStats(elapsed.getElapsedTimeF32()) ; + if (res) + { + // The whole data stream is finally compressed when res is returned as TRUE + tester->updateCompressionStats(this->getDataSize(), raw_imagep->getDataSize()) ; + } + } + return res; } @@ -540,3 +442,125 @@ void LLImageJ2C::updateRawDiscardLevel() LLImageJ2CImpl::~LLImageJ2CImpl() { } + +//---------------------------------------------------------------------------------------------- +// Start of LLImageCompressionTester +//---------------------------------------------------------------------------------------------- +LLImageCompressionTester::LLImageCompressionTester() : LLMetricPerformanceTesterBasic(sTesterName) +{ + addMetric("Time Decompression (s)"); + addMetric("Volume In Decompression (kB)"); + addMetric("Volume Out Decompression (kB)"); + addMetric("Decompression Ratio (x:1)"); + addMetric("Perf Decompression (kB/s)"); + + addMetric("Time Compression (s)"); + addMetric("Volume In Compression (kB)"); + addMetric("Volume Out Compression (kB)"); + addMetric("Compression Ratio (x:1)"); + addMetric("Perf Compression (kB/s)"); + + mRunBytesInDecompression = 0; + mRunBytesInCompression = 0; + + mTotalBytesInDecompression = 0; + mTotalBytesOutDecompression = 0; + mTotalBytesInCompression = 0; + mTotalBytesOutCompression = 0; + + mTotalTimeDecompression = 0.0f; + mTotalTimeCompression = 0.0f; +} + +LLImageCompressionTester::~LLImageCompressionTester() +{ + LLImageJ2C::sTesterp = NULL; +} + +//virtual +void LLImageCompressionTester::outputTestRecord(LLSD *sd) +{ + std::string currentLabel = getCurrentLabelName(); + + F32 decompressionPerf = 0.0f; + F32 compressionPerf = 0.0f; + F32 decompressionRate = 0.0f; + F32 compressionRate = 0.0f; + + F32 totalkBInDecompression = (F32)(mTotalBytesInDecompression) / 1000.0; + F32 totalkBOutDecompression = (F32)(mTotalBytesOutDecompression) / 1000.0; + F32 totalkBInCompression = (F32)(mTotalBytesInCompression) / 1000.0; + F32 totalkBOutCompression = (F32)(mTotalBytesOutCompression) / 1000.0; + + if (!is_approx_zero(mTotalTimeDecompression)) + { + decompressionPerf = totalkBInDecompression / mTotalTimeDecompression; + } + if (!is_approx_zero(totalkBInDecompression)) + { + decompressionRate = totalkBOutDecompression / totalkBInDecompression; + } + if (!is_approx_zero(mTotalTimeCompression)) + { + compressionPerf = totalkBInCompression / mTotalTimeCompression; + } + if (!is_approx_zero(totalkBOutCompression)) + { + compressionRate = totalkBInCompression / totalkBOutCompression; + } + + (*sd)[currentLabel]["Time Decompression (s)"] = (LLSD::Real)mTotalTimeDecompression; + (*sd)[currentLabel]["Volume In Decompression (kB)"] = (LLSD::Real)totalkBInDecompression; + (*sd)[currentLabel]["Volume Out Decompression (kB)"]= (LLSD::Real)totalkBOutDecompression; + (*sd)[currentLabel]["Decompression Ratio (x:1)"] = (LLSD::Real)decompressionRate; + (*sd)[currentLabel]["Perf Decompression (kB/s)"] = (LLSD::Real)decompressionPerf; + + (*sd)[currentLabel]["Time Compression (s)"] = (LLSD::Real)mTotalTimeCompression; + (*sd)[currentLabel]["Volume In Compression (kB)"] = (LLSD::Real)totalkBInCompression; + (*sd)[currentLabel]["Volume Out Compression (kB)"] = (LLSD::Real)totalkBOutCompression; + (*sd)[currentLabel]["Compression Ratio (x:1)"] = (LLSD::Real)compressionRate; + (*sd)[currentLabel]["Perf Compression (kB/s)"] = (LLSD::Real)compressionPerf; +} + +void LLImageCompressionTester::updateCompressionStats(const F32 deltaTime) +{ + mTotalTimeCompression += deltaTime; +} + +void LLImageCompressionTester::updateCompressionStats(const S32 bytesCompress, const S32 bytesRaw) +{ + mTotalBytesInCompression += bytesRaw; + mRunBytesInCompression += bytesRaw; + mTotalBytesOutCompression += bytesCompress; + if (mRunBytesInCompression > (1000000)) + { + // Output everything + outputTestResults(); + // Reset the compression data of the run + mRunBytesInCompression = 0; + } +} + +void LLImageCompressionTester::updateDecompressionStats(const F32 deltaTime) +{ + mTotalTimeDecompression += deltaTime; +} + +void LLImageCompressionTester::updateDecompressionStats(const S32 bytesIn, const S32 bytesOut) +{ + mTotalBytesInDecompression += bytesIn; + mRunBytesInDecompression += bytesIn; + mTotalBytesOutDecompression += bytesOut; + if (mRunBytesInDecompression > (1000000)) + { + // Output everything + outputTestResults(); + // Reset the decompression data of the run + mRunBytesInDecompression = 0; + } +} + +//---------------------------------------------------------------------------------------------- +// End of LLTexturePipelineTester +//---------------------------------------------------------------------------------------------- + diff --git a/indra/llimage/llimagej2c.h b/indra/llimage/llimagej2c.h index cdb3faa207..dd5bec8b2e 100644 --- a/indra/llimage/llimagej2c.h +++ b/indra/llimage/llimagej2c.h @@ -29,8 +29,11 @@ #include "llimage.h" #include "llassettype.h" +#include "llmetricperformancetester.h" class LLImageJ2CImpl; +class LLImageCompressionTester ; + class LLImageJ2C : public LLImageFormatted { protected: @@ -69,14 +72,13 @@ public: static S32 calcHeaderSizeJ2C(); static S32 calcDataSizeJ2C(S32 w, S32 h, S32 comp, S32 discard_level, F32 rate = 0.f); - static void openDSO(); - static void closeDSO(); static std::string getEngineInfo(); - + protected: friend class LLImageJ2CImpl; friend class LLImageJ2COJ; friend class LLImageJ2CKDU; + friend class LLImageCompressionTester; void decodeFailed(); void updateRawDiscardLevel(); @@ -90,6 +92,9 @@ protected: BOOL mReversible; LLImageJ2CImpl *mImpl; std::string mLastError; + + // Image compression/decompression tester + static LLImageCompressionTester* sTesterp; }; // Derive from this class to implement JPEG2000 decoding @@ -118,4 +123,40 @@ protected: #define LINDEN_J2C_COMMENT_PREFIX "LL_" +// +// This class is used for performance data gathering only. +// Tracks the image compression / decompression data, +// records and outputs them to the log file. +// +class LLImageCompressionTester : public LLMetricPerformanceTesterBasic +{ + public: + LLImageCompressionTester(); + ~LLImageCompressionTester(); + + void updateDecompressionStats(const F32 deltaTime) ; + void updateDecompressionStats(const S32 bytesIn, const S32 bytesOut) ; + void updateCompressionStats(const F32 deltaTime) ; + void updateCompressionStats(const S32 bytesIn, const S32 bytesOut) ; + + protected: + /*virtual*/ void outputTestRecord(LLSD* sd); + + private: + // + // Data size + // + U32 mTotalBytesInDecompression; // Total bytes fed to decompressor + U32 mTotalBytesOutDecompression; // Total bytes produced by decompressor + U32 mTotalBytesInCompression; // Total bytes fed to compressor + U32 mTotalBytesOutCompression; // Total bytes produced by compressor + U32 mRunBytesInDecompression; // Bytes fed to decompressor in this run + U32 mRunBytesInCompression; // Bytes fed to compressor in this run + // + // Time + // + F32 mTotalTimeDecompression; // Total time spent in computing decompression + F32 mTotalTimeCompression; // Total time spent in computing compression + }; + #endif diff --git a/indra/llkdu/CMakeLists.txt b/indra/llkdu/CMakeLists.txt new file mode 100644 index 0000000000..2806af26c3 --- /dev/null +++ b/indra/llkdu/CMakeLists.txt @@ -0,0 +1,85 @@ +# -*- cmake -*- + +project(llkdu) + +# Visual Studio 2005 has a dumb bug that causes it to fail compilation +# of KDU if building with both optimisation and /WS (treat warnings as +# errors), even when the specific warnings that make it croak are +# disabled. + +set(VS_DISABLE_FATAL_WARNINGS ON) + +include(00-Common) +include(LLCommon) +include(LLImage) +include(LLKDU) +include(LLMath) +#include(LLVFS) +#include(Linking) + +include_directories( + ${LLCOMMON_INCLUDE_DIRS} + ${LLIMAGE_INCLUDE_DIRS} + ${KDU_INCLUDE_DIR} + ${LLMATH_INCLUDE_DIRS} + ) + +set(llkdu_SOURCE_FILES + kdc_flow_control.cpp + kde_flow_control.cpp + kdu_image.cpp + llblockdata.cpp + llblockdecoder.cpp + llblockencoder.cpp + llimagej2ckdu.cpp + llkdumem.cpp + ) + +set(llkdu_HEADER_FILES + CMakeLists.txt + + kdc_flow_control.h + kde_flow_control.h + kdu_image.h + kdu_image_local.h + llblockdata.h + llblockdecoder.h + llblockencoder.h + llimagej2ckdu.h + llkdumem.h + ) + +set_source_files_properties(${llkdu_HEADER_FILES} + PROPERTIES HEADER_FILE_ONLY TRUE) + +list(APPEND llkdu_SOURCE_FILES ${llkdu_HEADER_FILES}) + +if (WINDOWS) + # This turns off the warning about flow control ending in a destructor. + set_source_files_properties( + kdu_image.cpp llkdumem.cpp + PROPERTIES + COMPILE_FLAGS "/wd4702 /wd4722" + ) + + # This turns off the warning about sprintf in the following 2 files. + set_source_files_properties( + kde_flow_control.cpp kdc_flow_control.cpp + PROPERTIES + COMPILE_FLAGS /D_CRT_SECURE_NO_DEPRECATE + ) +endif (WINDOWS) + +if (LLKDU_LIBRARY) + add_library (${LLKDU_STATIC_LIBRARY} ${llkdu_SOURCE_FILES}) + + target_link_libraries( + ${LLKDU_STATIC_LIBRARY} +# ${LLIMAGE_LIBRARIES} +# ${LLVFS_LIBRARIES} + ${LLMATH_LIBRARIES} +# ${LLCOMMON_LIBRARIES} + ${KDU_LIBRARY} +# ${WINDOWS_LIBRARIES} + ) +endif (LLKDU_LIBRARY) diff --git a/indra/llkdu/llblockdata.cpp b/indra/llkdu/llblockdata.cpp new file mode 100644 index 0000000000..6a7bc3e6c4 --- /dev/null +++ b/indra/llkdu/llblockdata.cpp @@ -0,0 +1,162 @@ +/** + * @file llblockdata.cpp + * @brief Image block structure + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llblockdata.h" +#include "llmath.h" + +LLBlockData::LLBlockData(const U32 type) +{ + mType = type; + mWidth = 0; + mHeight = 0; + mRowStride = 0; + mData = NULL; +} + +void LLBlockData::setData(U8 *data, const U32 width, const U32 height, const U32 row_stride) +{ + mData = data; + mWidth = width; + mHeight = height; + if (row_stride) + { + mRowStride = row_stride; + } + else + { + mRowStride = width * 4; + } +} + +U32 LLBlockData::getType() const +{ + return mType; +} + + +U8 *LLBlockData::getData() const +{ + return mData; +} + +U32 LLBlockData::getSize() const +{ + return mWidth*mHeight; +} + +U32 LLBlockData::getWidth() const +{ + return mWidth; +} +U32 LLBlockData::getHeight() const +{ + return mHeight; +} + +U32 LLBlockData::getRowStride() const +{ + return mRowStride; +} + +LLBlockDataU32::LLBlockDataU32() : LLBlockData(BLOCK_TYPE_U32) +{ + mPrecision = 32; +} + +void LLBlockDataU32::setData(U32 *data, const U32 width, const U32 height, const U32 row_stride) +{ + LLBlockData::setData((U8 *)data, width, height, row_stride); +} + +U32 LLBlockDataU32::getSize() const +{ + return mWidth*mHeight*4; +} + +void LLBlockDataU32::setPrecision(const U32 bits) +{ + mPrecision = bits; +} + +U32 LLBlockDataU32::getPrecision() const +{ + return mPrecision; +} + +void LLBlockDataF32::setPrecision(const U32 bits) +{ + mPrecision = bits; +} + +U32 LLBlockDataF32::getPrecision() const +{ + return mPrecision; +} + +void LLBlockDataF32::setData(F32 *data, const U32 width, const U32 height, const U32 row_stride) +{ + LLBlockData::setData((U8 *)data, width, height, row_stride); +} + +void LLBlockDataF32::setMin(const F32 min) +{ + mMin = min; +} + +void LLBlockDataF32::setMax(const F32 max) +{ + mMax = max; +} + +void LLBlockDataF32::calcMinMax() +{ + U32 x, y; + + mMin = *(F32*)mData; + mMax = mMin; + + for (y = 0; y < mHeight; y++) + { + for (x = 0; x < mWidth; x++) + { + F32 data = *(F32*)(mData + y*mRowStride + x*4); + mMin = llmin(data, mMin); + mMax = llmax(data, mMax); + } + } +} + +F32 LLBlockDataF32::getMin() const +{ + return mMin; +} + +F32 LLBlockDataF32::getMax() const +{ + return mMax; +} diff --git a/indra/llkdu/llblockdata.h b/indra/llkdu/llblockdata.h new file mode 100644 index 0000000000..dcc847e7e2 --- /dev/null +++ b/indra/llkdu/llblockdata.h @@ -0,0 +1,107 @@ +/** + * @file llblockdata.h + * @brief Image block structure + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLBLOCKDATA_H +#define LL_LLBLOCKDATA_H + +#include "stdtypes.h" + +////////////////////////////////////////////////// +// +// This class stores all of the information about +// a single channel of raw data, either integer +// or floating point. +// +class LLBlockData +{ +protected: + U32 mType; + U32 mWidth; + U32 mHeight; + U32 mRowStride; + U8 *mData; +public: + enum + { + BLOCK_TYPE_U32 = 1, + BLOCK_TYPE_F32 = 2 + }; + + LLBlockData(const U32 type); + virtual ~LLBlockData() {} + + void setData(U8 *data, const U32 width, const U32 height, const U32 row_stride = 0); + + U32 getType() const; + U8 *getData() const; + virtual U32 getSize() const; + U32 getWidth() const; + U32 getHeight() const; + U32 getRowStride() const; +}; + +class LLBlockDataU32 : public LLBlockData +{ +protected: + U32 mPrecision; +public: + LLBlockDataU32(); + + void setData(U32 *data, const U32 width, const U32 height, const U32 row_stride = 0); + void setPrecision(const U32 bits); + + /*virtual*/ U32 getSize() const; + U32 getPrecision() const; +}; + +class LLBlockDataF32 : public LLBlockData +{ +protected: + U32 mPrecision; + F32 mMin; + F32 mMax; +public: + LLBlockDataF32() + : LLBlockData(LLBlockData::BLOCK_TYPE_F32), + mPrecision(0), + mMin(0.f), + mMax(0.f) + {}; + + void setData(F32 *data, const U32 width, const U32 height, const U32 row_stride = 0); + + void setPrecision(const U32 bits); + void setMin(const F32 min); + void setMax(const F32 max); + + void calcMinMax(); + + U32 getPrecision() const; + F32 getMin() const; + F32 getMax() const; +}; + +#endif // LL_LLBLOCKDATA_H diff --git a/indra/llkdu/llblockdecoder.cpp b/indra/llkdu/llblockdecoder.cpp new file mode 100644 index 0000000000..b4ddb2fba2 --- /dev/null +++ b/indra/llkdu/llblockdecoder.cpp @@ -0,0 +1,273 @@ + /** + * @file llblockdecoder.cpp + * @brief Image block decompression + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llblockdecoder.h" + +// KDU core header files +#include "kdu/kdu_elementary.h" +#include "kdu/kdu_messaging.h" +#include "kdu/kdu_params.h" +#include "kdu/kdu_compressed.h" +#include "kdu/kdu_sample_processing.h" + +// KDU utility functions. +#include "kde_flow_control.h" + +#include "llkdumem.h" + +#include "llblockdata.h" +#include "llerror.h" + + +BOOL LLBlockDecoder::decodeU32(LLBlockDataU32 &block_data, U8 *source_data, const U32 source_size) const +{ + U32 width, height; + + llassert(source_data); + + LLKDUMemSource source(source_data, source_size); + + source.reset(); + + kdu_codestream codestream; + + codestream.create(&source); + codestream.set_fast(); + + kdu_dims dims; + codestream.get_dims(0,dims); + llassert(codestream.get_num_components() == 1); + + width = dims.size.x; + height = dims.size.y; + + // Assumes U32 data. + U8 *output = block_data.getData(); + + kdu_dims tile_indices; + codestream.get_valid_tiles(tile_indices); + + kdu_coords tpos; + tpos.x = 0; + tpos.y = 0; + + // Now we are ready to walk through the tiles processing them one-by-one. + while (tpos.y < tile_indices.size.y) + { + while (tpos.x < tile_indices.size.x) + { + kdu_tile tile = codestream.open_tile(tpos+tile_indices.pos); + + kdu_resolution res = tile.access_component(0).access_resolution(); + kdu_dims tile_dims; + res.get_dims(tile_dims); + kdu_coords offset = tile_dims.pos - dims.pos; + int row_gap = block_data.getRowStride(); // inter-row separation + kdu_byte *buf = output + offset.y*row_gap + offset.x*4; + + kdu_tile_comp tile_comp = tile.access_component(0); + bool reversible = tile_comp.get_reversible(); + U32 precision = tile_comp.get_bit_depth(); + U32 precision_scale = 1 << precision; + llassert(precision >= 8); // Else would have used 16 bit representation + + kdu_resolution comp_res = tile_comp.access_resolution(); // Get top resolution + kdu_dims comp_dims; + comp_res.get_dims(comp_dims); + + bool use_shorts = (tile_comp.get_bit_depth(true) <= 16); + + kdu_line_buf line; + kdu_sample_allocator allocator; + kdu_pull_ifc engine; + + line.pre_create(&allocator, comp_dims.size.x, reversible, use_shorts); + if (res.which() == 0) // No DWT levels used + { + engine = kdu_decoder(res.access_subband(LL_BAND), &allocator, use_shorts); + } + else + { + engine = kdu_synthesis(res, &allocator, use_shorts); + } + allocator.finalize(); // Actually creates buffering resources + + line.create(); // Grabs resources from the allocator. + + // Do the actual processing + while (tile_dims.size.y--) + { + engine.pull(line, true); + int width = line.get_width(); + + llassert(line.get_buf32()); + llassert(!line.is_absolute()); + // Decompressed samples have a 32-bit representation (integer or float) + kdu_sample32 *sp = line.get_buf32(); + // Transferring normalized floating point data. + U32 *dest_u32 = (U32 *)buf; + for (; width > 0; width--, sp++, dest_u32++) + { + if (sp->fval < -0.5f) + { + *dest_u32 = 0; + } + else + { + *dest_u32 = (U32)((sp->fval + 0.5f)*precision_scale); + } + } + buf += row_gap; + } + engine.destroy(); + tile.close(); + tpos.x++; + } + tpos.y++; + tpos.x = 0; + } + codestream.destroy(); + + return TRUE; +} + +BOOL LLBlockDecoder::decodeF32(LLBlockDataF32 &block_data, U8 *source_data, const U32 source_size, const F32 min, const F32 max) const +{ + U32 width, height; + F32 range, range_inv, float_offset; + bool use_shorts = false; + + range = max - min; + range_inv = 1.f / range; + float_offset = 0.5f*(max + min); + + llassert(source_data); + + LLKDUMemSource source(source_data, source_size); + + source.reset(); + + kdu_codestream codestream; + + codestream.create(&source); + codestream.set_fast(); + + kdu_dims dims; + codestream.get_dims(0,dims); + llassert(codestream.get_num_components() == 1); + + width = dims.size.x; + height = dims.size.y; + + // Assumes F32 data. + U8 *output = block_data.getData(); + + kdu_dims tile_indices; + codestream.get_valid_tiles(tile_indices); + + kdu_coords tpos; + tpos.x = 0; + tpos.y = 0; + + // Now we are ready to walk through the tiles processing them one-by-one. + while (tpos.y < tile_indices.size.y) + { + while (tpos.x < tile_indices.size.x) + { + kdu_tile tile = codestream.open_tile(tpos+tile_indices.pos); + + kdu_resolution res = tile.access_component(0).access_resolution(); + kdu_dims tile_dims; + res.get_dims(tile_dims); + kdu_coords offset = tile_dims.pos - dims.pos; + int row_gap = block_data.getRowStride(); // inter-row separation + kdu_byte *buf = output + offset.y*row_gap + offset.x*4; + + kdu_tile_comp tile_comp = tile.access_component(0); + bool reversible = tile_comp.get_reversible(); + + kdu_resolution comp_res = tile_comp.access_resolution(); // Get top resolution + kdu_dims comp_dims; + comp_res.get_dims(comp_dims); + + kdu_line_buf line; + kdu_sample_allocator allocator; + kdu_pull_ifc engine; + + line.pre_create(&allocator, comp_dims.size.x, reversible, use_shorts); + if (res.which() == 0) // No DWT levels used + { + engine = kdu_decoder(res.access_subband(LL_BAND), &allocator, use_shorts); + } + else + { + engine = kdu_synthesis(res, &allocator, use_shorts); + } + allocator.finalize(); // Actually creates buffering resources + + line.create(); // Grabs resources from the allocator. + + // Do the actual processing + while (tile_dims.size.y--) + { + engine.pull(line, true); + int width = line.get_width(); + + llassert(line.get_buf32()); + llassert(!line.is_absolute()); + // Decompressed samples have a 32-bit representation (integer or float) + kdu_sample32 *sp = line.get_buf32(); + // Transferring normalized floating point data. + F32 *dest_f32 = (F32 *)buf; + for (; width > 0; width--, sp++, dest_f32++) + { + if (sp->fval < -0.5f) + { + *dest_f32 = min; + } + else if (sp->fval > 0.5f) + { + *dest_f32 = max; + } + else + { + *dest_f32 = (sp->fval) * range + float_offset; + } + } + buf += row_gap; + } + engine.destroy(); + tile.close(); + tpos.x++; + } + tpos.y++; + tpos.x = 0; + } + codestream.destroy(); + return TRUE; +} diff --git a/indra/llkdu/llblockdecoder.h b/indra/llkdu/llblockdecoder.h new file mode 100644 index 0000000000..97959d338e --- /dev/null +++ b/indra/llkdu/llblockdecoder.h @@ -0,0 +1,42 @@ +/** + * @file llblockdecoder.h + * @brief Image block decompression + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLBLOCKDECODER_H +#define LL_LLBLOCKDECODER_H + +#include "stdtypes.h" + +class LLBlockDataU32; +class LLBlockDataF32; + +class LLBlockDecoder +{ +public: + BOOL decodeU32(LLBlockDataU32 &block_data, U8 *source_data, const U32 source_size) const; + BOOL decodeF32(LLBlockDataF32 &block_data, U8 *source_data, const U32 source_size, const F32 min, const F32 max) const; +}; + +#endif // LL_LLBLOCKDECODER_H diff --git a/indra/llkdu/llblockencoder.cpp b/indra/llkdu/llblockencoder.cpp new file mode 100644 index 0000000000..f19841e36f --- /dev/null +++ b/indra/llkdu/llblockencoder.cpp @@ -0,0 +1,343 @@ + /** + * @file llblockencoder.cpp + * @brief Image block compression + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llblockencoder.h" + +// KDU core header files +#include "kdu/kdu_elementary.h" +#include "kdu/kdu_messaging.h" +#include "kdu/kdu_params.h" +#include "kdu/kdu_compressed.h" +#include "kdu/kdu_sample_processing.h" + +// KDU utility functions. +#include "kdc_flow_control.h" + +#include "llkdumem.h" + +#include "llblockdata.h" +#include "llerror.h" + +LLBlockEncoder::LLBlockEncoder() +{ + mBPP = 0.f; +} + +U8 *LLBlockEncoder::encode(const LLBlockData &block_data, U32 &output_size) const +{ + switch (block_data.getType()) + { + case LLBlockData::BLOCK_TYPE_U32: + { + LLBlockDataU32 &bd_u32 = (LLBlockDataU32 &)block_data; + return encodeU32(bd_u32, output_size); + } + case LLBlockData::BLOCK_TYPE_F32: + { + LLBlockDataF32 &bd_f32 = (LLBlockDataF32 &)block_data; + return encodeF32(bd_f32, output_size); + } + default: + llerrs << "Unsupported block type!" << llendl; + output_size = 0; + return NULL; + } +} + +U8 *LLBlockEncoder::encodeU32(const LLBlockDataU32 &block_data, U32 &output_size) const +{ + // OK, for now, just use the standard KDU encoder, with a single channel + // integer channel. + + // Collect simple arguments. + bool allow_rate_prediction, allow_shorts, mem, quiet, no_weights; + + allow_rate_prediction = true; + allow_shorts = false; + no_weights = false; + bool use_absolute = false; + mem = false; + quiet = false; + + // Set codestream options + siz_params siz; + S16 precision = block_data.getPrecision(); + + siz.set(Sdims,0,0,(U16)block_data.getHeight()); + siz.set(Sdims,0,1,(U16)block_data.getWidth()); + siz.set(Ssigned,0,0,false); + siz.set(Scomponents,0,0,1); + siz.set(Sprecision,0,0, precision); + + // Construct the `kdu_codestream' object and parse all remaining arguments. + output_size = block_data.getSize(); + if (output_size < 1000) + { + output_size = 1000; + } + + U8 *output_buffer = new U8[output_size]; + + LLKDUMemTarget output(output_buffer, output_size, block_data.getSize()); + + kdu_codestream codestream; + codestream.create(&siz,&output); + + codestream.access_siz()->parse_string("Clayers=1"); + codestream.access_siz()->finalize_all(); + + kdu_tile tile = codestream.open_tile(kdu_coords(0,0)); + + // Open tile-components and create processing engines and resources + kdu_dims dims; + kdu_sample_allocator allocator; + kdu_tile_comp tile_comp; + kdu_line_buf line; + kdu_push_ifc engine; + + tile_comp = tile.access_component(0); + kdu_resolution res = tile_comp.access_resolution(); // Get top resolution + + res.get_dims(dims); + + line.pre_create(&allocator,dims.size.x, use_absolute, allow_shorts); + + if (res.which() == 0) // No DWT levels (should not occur in this example) + { + engine = kdu_encoder(res.access_subband(LL_BAND),&allocator, use_absolute); + } + else + { + engine = kdu_analysis(res,&allocator, use_absolute); + } + + allocator.finalize(); // Actually creates buffering resources + line.create(); // Grabs resources from the allocator. + + // Now walk through the lines of the buffer, pushing them into the + // relevant tile-component processing engines. + + U32 *source_u32 = NULL; + F32 scale_inv = 1.f / (1 << precision); + + S32 y; + for (y = 0; y < dims.size.y; y++) + { + source_u32 = (U32*)(block_data.getData() + y * block_data.getRowStride()); + kdu_sample32 *dest = line.get_buf32(); + for (S32 n=dims.size.x; n > 0; n--, dest++, source_u32++) + { + // Just pack it in, for now. + dest->fval = (F32)(*source_u32) * scale_inv - 0.5f;// - 0.5f; + } + engine.push(line, true); + } + + // Cleanup + engine.destroy(); // engines are interfaces; no default destructors + + // Produce the final compressed output. + kdu_long layer_bytes[1] = {0}; + + layer_bytes[0] = (kdu_long) (mBPP*block_data.getWidth()*block_data.getHeight()); + // Here we are not requesting specific sizes for any of the 12 + // quality layers. As explained in the description of + // "kdu_codestream::flush" (see "kdu_compressed.h"), the rate allocator + // will then assign the layers in such a way as to achieve roughly + // two quality layers per octave change in bit-rate, with the final + // layer reaching true lossless quality. + + codestream.flush(layer_bytes,1); + // You can see how many bytes were assigned + // to each quality layer by looking at the entries of `layer_bytes' here. + // The flush function can do a lot of interesting things which you may + // want to spend some time looking into. In addition to targeting + // specific bit-rates for each quality layer, it can be configured to + // use rate-distortion slope thresholds of your choosing and to return + // the thresholds which it finds to be best for any particular set of + // target layer sizes. This opens the door to feedback-oriented rate + // control for video. You should also look into the + // "kdu_codestream::set_max_bytes" and + // "kdu_codestream::set_min_slope_threshold" functions which can be + // used to significantly speed up compression. + codestream.destroy(); // All done: simple as that. + + // Now that we're done encoding, create the new data buffer for the compressed + // image and stick it there. + + U8 *output_data = new U8[output_size]; + + memcpy(output_data, output_buffer, output_size); + + output.close(); // Not really necessary here. + + return output_data; +} + +U8 *LLBlockEncoder::encodeF32(const LLBlockDataF32 &block_data, U32 &output_size) const +{ + // OK, for now, just use the standard KDU encoder, with a single channel + // integer channel. + + // Collect simple arguments. + bool allow_rate_prediction, allow_shorts, mem, quiet, no_weights; + + allow_rate_prediction = true; + allow_shorts = false; + no_weights = false; + bool use_absolute = false; + mem = false; + quiet = false; + + F32 min, max, range, range_inv, offset; + min = block_data.getMin(); + max = block_data.getMax(); + range = max - min; + range_inv = 1.f / range; + offset = 0.5f*(max + min); + + // Set codestream options + siz_params siz; + S16 precision = block_data.getPrecision(); // Assume precision is always 32 bits for floating point. + + siz.set(Sdims,0,0,(U16)block_data.getHeight()); + siz.set(Sdims,0,1,(U16)block_data.getWidth()); + siz.set(Ssigned,0,0,false); + siz.set(Scomponents,0,0,1); + siz.set(Sprecision,0,0, precision); + + // Construct the `kdu_codestream' object and parse all remaining arguments. + output_size = block_data.getSize(); + if (output_size < 1000) + { + output_size = 1000; + } + + U8 *output_buffer = new U8[output_size*2]; + + LLKDUMemTarget output(output_buffer, output_size, block_data.getSize()); + + kdu_codestream codestream; + codestream.create(&siz,&output); + + codestream.access_siz()->parse_string("Clayers=1"); + codestream.access_siz()->finalize_all(); + + kdu_tile tile = codestream.open_tile(kdu_coords(0,0)); + + // Open tile-components and create processing engines and resources + kdu_dims dims; + kdu_sample_allocator allocator; + kdu_tile_comp tile_comp; + kdu_line_buf line; + kdu_push_ifc engine; + + tile_comp = tile.access_component(0); + kdu_resolution res = tile_comp.access_resolution(); // Get top resolution + + res.get_dims(dims); + + line.pre_create(&allocator,dims.size.x, use_absolute, allow_shorts); + + if (res.which() == 0) // No DWT levels (should not occur in this example) + { + engine = kdu_encoder(res.access_subband(LL_BAND),&allocator, use_absolute); + } + else + { + engine = kdu_analysis(res,&allocator, use_absolute); + } + + allocator.finalize(); // Actually creates buffering resources + line.create(); // Grabs resources from the allocator. + + // Now walk through the lines of the buffer, pushing them into the + // relevant tile-component processing engines. + + F32 *source_f32 = NULL; + + S32 y; + for (y = 0; y < dims.size.y; y++) + { + source_f32 = (F32*)(block_data.getData() + y * block_data.getRowStride()); + kdu_sample32 *dest = line.get_buf32(); + for (S32 n=dims.size.x; n > 0; n--, dest++, source_f32++) + { + dest->fval = ((*source_f32) - offset) * range_inv; + } + engine.push(line, true); + } + + // Cleanup + engine.destroy(); // engines are interfaces; no default destructors + + // Produce the final compressed output. + kdu_long layer_bytes[1] = {0}; + + layer_bytes[0] = (kdu_long) (mBPP*block_data.getWidth()*block_data.getHeight()); + // Here we are not requesting specific sizes for any of the 12 + // quality layers. As explained in the description of + // "kdu_codestream::flush" (see "kdu_compressed.h"), the rate allocator + // will then assign the layers in such a way as to achieve roughly + // two quality layers per octave change in bit-rate, with the final + // layer reaching true lossless quality. + + codestream.flush(layer_bytes,1); + // You can see how many bytes were assigned + // to each quality layer by looking at the entries of `layer_bytes' here. + // The flush function can do a lot of interesting things which you may + // want to spend some time looking into. In addition to targeting + // specific bit-rates for each quality layer, it can be configured to + // use rate-distortion slope thresholds of your choosing and to return + // the thresholds which it finds to be best for any particular set of + // target layer sizes. This opens the door to feedback-oriented rate + // control for video. You should also look into the + // "kdu_codestream::set_max_bytes" and + // "kdu_codestream::set_min_slope_threshold" functions which can be + // used to significantly speed up compression. + codestream.destroy(); // All done: simple as that. + + // Now that we're done encoding, create the new data buffer for the compressed + // image and stick it there. + + U8 *output_data = new U8[output_size]; + + memcpy(output_data, output_buffer, output_size); + + output.close(); // Not really necessary here. + + delete[] output_buffer; + + return output_data; +} + + +void LLBlockEncoder::setBPP(const F32 bpp) +{ + mBPP = bpp; +} diff --git a/indra/llkdu/llblockencoder.h b/indra/llkdu/llblockencoder.h new file mode 100644 index 0000000000..21381a27fa --- /dev/null +++ b/indra/llkdu/llblockencoder.h @@ -0,0 +1,49 @@ +/** + * @file llblockencoder.h + * @brief Image block compression + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLBLOCKENCODER_H +#define LL_LLBLOCKENCODER_H + +#include "stdtypes.h" + +class LLBlockData; +class LLBlockDataU32; +class LLBlockDataF32; + +class LLBlockEncoder +{ + F32 mBPP; // bits per point +public: + LLBlockEncoder(); + U8 *encode(const LLBlockData &block_data, U32 &output_size) const; + U8 *encodeU32(const LLBlockDataU32 &block_data, U32 &output_size) const; + U8 *encodeF32(const LLBlockDataF32 &block_data, U32 &output_size) const; + + void setBPP(const F32 bpp); +}; + +#endif // LL_LLBLOCKENCODER_H + diff --git a/indra/llkdu/llimagej2ckdu.cpp b/indra/llkdu/llimagej2ckdu.cpp new file mode 100644 index 0000000000..1785aa111d --- /dev/null +++ b/indra/llkdu/llimagej2ckdu.cpp @@ -0,0 +1,1006 @@ + /** + * @file llimagej2ckdu.cpp + * @brief This is an implementation of JPEG2000 encode/decode using Kakadu + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" +#include "llimagej2ckdu.h" + +// KDU utility functions. +#include "kde_flow_control.h" +#include "kdc_flow_control.h" + +#include "lltimer.h" +#include "llpointer.h" +#include "llkdumem.h" + + +// +// Kakadu specific implementation +// +void set_default_colour_weights(kdu_params *siz); + +const char* engineInfoLLImageJ2CKDU() +{ + return "KDU"; +} + +LLImageJ2CKDU* createLLImageJ2CKDU() +{ + return new LLImageJ2CKDU(); +} + +void destroyLLImageJ2CKDU(LLImageJ2CKDU* kdu) +{ + delete kdu; + kdu = NULL; +} + +LLImageJ2CImpl* fallbackCreateLLImageJ2CImpl() +{ + return new LLImageJ2CKDU(); +} + +void fallbackDestroyLLImageJ2CImpl(LLImageJ2CImpl* impl) +{ + delete impl; + impl = NULL; +} + +const char* fallbackEngineInfoLLImageJ2CImpl() +{ + return engineInfoLLImageJ2CKDU(); +} + +class LLKDUDecodeState +{ +public: + + S32 mNumComponents; + BOOL mUseYCC; + kdu_dims mDims; + kdu_sample_allocator mAllocator; + kdu_tile_comp mComps[4]; + kdu_line_buf mLines[4]; + kdu_pull_ifc mEngines[4]; + bool mReversible[4]; // Some components may be reversible and others not. + int mBitDepths[4]; // Original bit-depth may be quite different from 8. + + kdu_tile mTile; + kdu_byte *mBuf; + S32 mRowGap; + + LLKDUDecodeState(kdu_tile tile, kdu_byte *buf, S32 row_gap); + ~LLKDUDecodeState(); + BOOL processTileDecode(F32 decode_time, BOOL limit_time = TRUE); + +public: + int *AssignLayerBytes(siz_params *siz, int &num_specs); + + void setupCodeStream(BOOL keep_codestream, LLImageJ2CKDU::ECodeStreamMode mode); + BOOL initDecode(LLImageRaw &raw_image, F32 decode_time, LLImageJ2CKDU::ECodeStreamMode mode, S32 first_channel, S32 max_channel_count ); +}; + +void ll_kdu_error( void ) +{ + // *FIX: This exception is bad, bad, bad. It gets thrown from a + // destructor which can lead imediate program termination! + throw "ll_kdu_error() throwing an exception"; +} +// Stuff for new kdu error handling. + +class LLKDUMessageWarning : public kdu_message +{ +public: + /*virtual*/ void put_text(const char *string); + + static LLKDUMessageWarning sDefaultMessage; +}; + +class LLKDUMessageError : public kdu_message +{ +public: + /*virtual*/ void put_text(const char *string); + /*virtual*/ void flush(bool end_of_message=false); + static LLKDUMessageError sDefaultMessage; +}; + +void LLKDUMessageWarning::put_text(const char *s) +{ + llinfos << "KDU Warning: " << s << llendl; +} + +void LLKDUMessageError::put_text(const char *s) +{ + llinfos << "KDU Error: " << s << llendl; +} + +void LLKDUMessageError::flush(bool end_of_message) +{ + if( end_of_message ) + { + throw "KDU throwing an exception"; + } +} + +LLKDUMessageWarning LLKDUMessageWarning::sDefaultMessage; +LLKDUMessageError LLKDUMessageError::sDefaultMessage; +static bool kdu_message_initialized = false; + +LLImageJ2CKDU::LLImageJ2CKDU() : LLImageJ2CImpl(), +mInputp(NULL), +mCodeStreamp(NULL), +mTPosp(NULL), +mTileIndicesp(NULL), +mRawImagep(NULL), +mDecodeState(NULL) +{ +} + +LLImageJ2CKDU::~LLImageJ2CKDU() +{ + cleanupCodeStream(); // in case destroyed before decode completed +} + +// Stuff for new simple decode +void transfer_bytes(kdu_byte *dest, kdu_line_buf &src, int gap, int precision); + +void LLImageJ2CKDU::setupCodeStream(LLImageJ2C &base, BOOL keep_codestream, ECodeStreamMode mode) +{ + S32 data_size = base.getDataSize(); + S32 max_bytes = base.getMaxBytes() ? base.getMaxBytes() : data_size; + + ////////////// + // + // Initialization + // + if (!kdu_message_initialized) + { + kdu_message_initialized = true; + kdu_customize_errors(&LLKDUMessageError::sDefaultMessage); + kdu_customize_warnings(&LLKDUMessageWarning::sDefaultMessage); + } + + if (mCodeStreamp) + { + mCodeStreamp->destroy(); + delete mCodeStreamp; + mCodeStreamp = NULL; + } + + + if (!mInputp) + { + llassert(base.getData()); + // The compressed data has been loaded. + // Setup the source for the codestrea + mInputp = new LLKDUMemSource(base.getData(), data_size); + } + + llassert(mInputp); + mInputp->reset(); + mCodeStreamp = new kdu_codestream; + + mCodeStreamp->create(mInputp); + + + // Set the maximum number of bytes to use from the codestrea + mCodeStreamp->set_max_bytes(max_bytes); + + // If you want to flip or rotate the image for some reason, change + // the resolution, or identify a restricted region of interest, this is + // the place to do it. You may use "kdu_codestream::change_appearance" + // and "kdu_codestream::apply_input_restrictions" for this purpose. + // If you wish to truncate the code-stream prior to decompression, you + // may use "kdu_codestream::set_max_bytes". + // If you wish to retain all compressed data so that the material + // can be decompressed multiple times, possibly with different appearance + // parameters, you should call "kdu_codestream::set_persistent" here. + // There are a variety of other features which must be enabled at + // this point if you want to take advantage of the See the + // descriptions appearing with the "kdu_codestream" interface functions + // in "kdu_compressed.h" for an itemized account of these capabilities. + + + switch( mode ) + { + case MODE_FAST: + mCodeStreamp->set_fast(); + break; + case MODE_RESILIENT: + mCodeStreamp->set_resilient(); + break; + case MODE_FUSSY: + mCodeStreamp->set_fussy(); + break; + default: + llassert(0); + mCodeStreamp->set_fast(); + } + + kdu_dims dims; + mCodeStreamp->get_dims(0,dims); + + S32 components = mCodeStreamp->get_num_components(); + + if (components >= 3) + { // Check that components have consistent dimensions (for PPM file) + kdu_dims dims1; mCodeStreamp->get_dims(1,dims1); + kdu_dims dims2; mCodeStreamp->get_dims(2,dims2); + if ((dims1 != dims) || (dims2 != dims)) + { + llerrs << "Components don't have matching dimensions!" << llendl; + } + } + + base.setSize(dims.size.x, dims.size.y, components); + + if (!keep_codestream) + { + mCodeStreamp->destroy(); + delete mCodeStreamp; + mCodeStreamp = NULL; + delete mInputp; + mInputp = NULL; + } +} + +void LLImageJ2CKDU::cleanupCodeStream() +{ + delete mInputp; + mInputp = NULL; + + delete mDecodeState; + mDecodeState = NULL; + + if (mCodeStreamp) + { + mCodeStreamp->destroy(); + delete mCodeStreamp; + mCodeStreamp = NULL; + } + + delete mTPosp; + mTPosp = NULL; + + delete mTileIndicesp; + mTileIndicesp = NULL; +} + +BOOL LLImageJ2CKDU::initDecode(LLImageJ2C &base, LLImageRaw &raw_image, F32 decode_time, ECodeStreamMode mode, S32 first_channel, S32 max_channel_count ) +{ + base.resetLastError(); + + // *FIX: kdu calls our callback function if there's an error, and then bombs. + // To regain control, we throw an exception, and catch it here. + try + { + base.updateRawDiscardLevel(); + setupCodeStream(base, TRUE, mode); + + /* + // + // Not being used OpenJPEG doesn't support it, just deprecate it. + // + + // Find the Linden Lab comment in the chain of comments + kdu_codestream_comment comment; + comment = mCodeStreamp->get_comment(); + while (comment.get_text()) + { + const char* text = comment.get_text(); + if( text == strstr( text, LINDEN_J2C_COMMENT_PREFIX) ) + { + mCommentText = text; + break; + } + //llinfos << "CS comment: " << comment.get_text() << llendl; + comment = mCodeStreamp->get_comment(comment); + } + */ + + mRawImagep = &raw_image; + mCodeStreamp->change_appearance(false, true, false); + mCodeStreamp->apply_input_restrictions(first_channel,max_channel_count,base.getRawDiscardLevel(),0,NULL); + + kdu_dims dims; mCodeStreamp->get_dims(0,dims); + S32 channels = base.getComponents() - first_channel; + if( channels > max_channel_count ) + { + channels = max_channel_count; + } + raw_image.resize(dims.size.x, dims.size.y, channels); + + // llinfos << "Resizing to " << dims.size.x << ":" << dims.size.y << llendl; + if (!mTileIndicesp) + { + mTileIndicesp = new kdu_dims; + } + mCodeStreamp->get_valid_tiles(*mTileIndicesp); + if (!mTPosp) + { + mTPosp = new kdu_coords; + mTPosp->y = 0; + mTPosp->x = 0; + } + } + catch (const char* msg) + { + base.setLastError(ll_safe_string(msg)); + return FALSE; + } + catch (...) + { + base.setLastError("Unknown J2C error"); + return FALSE; + } + + + return TRUE; +} + + +// Returns TRUE to mean done, whether successful or not. +BOOL LLImageJ2CKDU::decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decode_time, S32 first_channel, S32 max_channel_count) +{ + ECodeStreamMode mode = MODE_FAST; + + LLTimer decode_timer; + + if (!mCodeStreamp) + { + if (!initDecode(base, raw_image, decode_time, mode, first_channel, max_channel_count)) + { + // Initializing the J2C decode failed, bail out. + cleanupCodeStream(); + return TRUE; // done + } + } + + // These can probably be grabbed from what's saved in the class. + kdu_dims dims; + mCodeStreamp->get_dims(0,dims); + + // Now we are ready to walk through the tiles processing them one-by-one. + kdu_byte *buffer = raw_image.getData(); + + while (mTPosp->y < mTileIndicesp->size.y) + { + while (mTPosp->x < mTileIndicesp->size.x) + { + try + { + if (!mDecodeState) + { + kdu_tile tile = mCodeStreamp->open_tile(*(mTPosp)+mTileIndicesp->pos); + + // Find the region of the buffer occupied by this + // tile. Note that we have no control over + // sub-sampling factors which might have been used + // during compression and so it can happen that tiles + // (at the image component level) actually have + // different dimensions. For this reason, we cannot + // figure out the buffer region occupied by a tile + // directly from the tile indices. Instead, we query + // the highest resolution of the first tile-component + // concerning its location and size on the canvas -- + // the `dims' object already holds the location and + // size of the entire image component on the same + // canvas coordinate system. Comparing the two tells + // us where the current tile is in the buffer. + S32 channels = base.getComponents() - first_channel; + if( channels > max_channel_count ) + { + channels = max_channel_count; + } + kdu_resolution res = tile.access_component(0).access_resolution(); + kdu_dims tile_dims; res.get_dims(tile_dims); + kdu_coords offset = tile_dims.pos - dims.pos; + int row_gap = channels*dims.size.x; // inter-row separation + kdu_byte *buf = buffer + offset.y*row_gap + offset.x*channels; + mDecodeState = new LLKDUDecodeState(tile, buf, row_gap); + } + // Do the actual processing + F32 remaining_time = decode_time - decode_timer.getElapsedTimeF32(); + // This is where we do the actual decode. If we run out of time, return false. + if (mDecodeState->processTileDecode(remaining_time, (decode_time > 0.0f))) + { + delete mDecodeState; + mDecodeState = NULL; + } + else + { + // Not finished decoding yet. + // setLastError("Ran out of time while decoding"); + return FALSE; + } + } + catch( const char* msg ) + { + base.setLastError(ll_safe_string(msg)); + base.decodeFailed(); + cleanupCodeStream(); + return TRUE; // done + } + catch( ... ) + { + base.setLastError( "Unknown J2C error" ); + base.decodeFailed(); + cleanupCodeStream(); + return TRUE; // done + } + + + mTPosp->x++; + } + mTPosp->y++; + mTPosp->x = 0; + } + + cleanupCodeStream(); + + return TRUE; +} + + +BOOL LLImageJ2CKDU::encodeImpl(LLImageJ2C &base, const LLImageRaw &raw_image, const char* comment_text, F32 encode_time, BOOL reversible) +{ + // Collect simple arguments. + + bool transpose, vflip, hflip; + bool allow_rate_prediction, allow_shorts, mem, quiet, no_weights; + int cpu_iterations; + std::ostream *record_stream; + + transpose = false; + record_stream = NULL; + allow_rate_prediction = true; + allow_shorts = true; + no_weights = false; + cpu_iterations = -1; + mem = false; + quiet = false; + vflip = true; + hflip = false; + + try + { + // Set up input image files. + + siz_params siz; + // Should set rate someplace here. + + LLKDUMemIn mem_in(raw_image.getData(), + raw_image.getDataSize(), + raw_image.getWidth(), + raw_image.getHeight(), + raw_image.getComponents(), + &siz); + + base.setSize(raw_image.getWidth(), raw_image.getHeight(), raw_image.getComponents()); + + int num_components = raw_image.getComponents(); + + siz.set(Scomponents,0,0,num_components); + siz.set(Sdims,0,0,base.getHeight()); // Height of first image component + siz.set(Sdims,0,1,base.getWidth()); // Width of first image component + siz.set(Sprecision,0,0,8); // Image samples have original bit-depth of 8 + siz.set(Ssigned,0,0,false); // Image samples are originally unsigned + + kdu_params *siz_ref = &siz; siz_ref->finalize(); + siz_params transformed_siz; // Use this one to construct code-strea + transformed_siz.copy_from(&siz,-1,-1,-1,0,transpose,false,false); + + // Construct the `kdu_codestream' object and parse all remaining arguments. + + U32 max_output_size = base.getWidth()*base.getHeight()*base.getComponents(); + if (max_output_size < 1000) + { + max_output_size = 1000; + } + U8 *output_buffer = new U8[max_output_size]; + + U32 output_size = max_output_size; // gets modified + LLKDUMemTarget output(output_buffer, output_size, base.getWidth()*base.getHeight()*base.getComponents()); + if (output_size > max_output_size) + { + llerrs << llformat("LLImageJ2C::encode output_size(%d) > max_output_size(%d)", + output_size,max_output_size) << llendl; + } + + kdu_codestream codestream; + codestream.create(&transformed_siz,&output); + + if (comment_text) + { + // Set the comments for the codestream + kdu_codestream_comment comment = codestream.add_comment(); + comment.put_text(comment_text); + } + + // Set codestream options + int num_layer_specs = 0; + + kdu_long layer_bytes[64]; + U32 max_bytes = 0; + + if ((num_components >= 3) && !no_weights) + { + set_default_colour_weights(codestream.access_siz()); + } + + if (reversible) + { + // If we're doing reversible, assume we're not using quality layers. + // Yes, I know this is incorrect! + codestream.access_siz()->parse_string("Creversible=yes"); + codestream.access_siz()->parse_string("Clayers=1"); + num_layer_specs = 1; + layer_bytes[0] = 0; + } + + else + { + // Rate is the argument passed into the LLImageJ2C which + // specifies the target compression rate. The default is 8:1. + // Possibly if max_bytes < 500, we should just use the default setting? + if (base.mRate != 0.f) + { + max_bytes = (U32)(base.mRate*base.getWidth()*base.getHeight()*base.getComponents()); + } + else + { + max_bytes = (U32)(base.getWidth()*base.getHeight()*base.getComponents()*0.125); + } + + const U32 min_bytes = FIRST_PACKET_SIZE; + if (max_bytes > min_bytes) + { + U32 i; + // This code is where we specify the target number of bytes for + // each layer. Not sure if we should do this for small images + // or not. The goal is to have this roughly align with + // different quality levels that we decode at. + for (i = min_bytes; i < max_bytes; i*=4) + { + if (i == min_bytes * 4) + { + i = 2000; + } + layer_bytes[num_layer_specs] = i; + num_layer_specs++; + } + layer_bytes[num_layer_specs] = max_bytes; + num_layer_specs++; + + std::string layer_string = llformat("Clayers=%d",num_layer_specs); + codestream.access_siz()->parse_string(layer_string.c_str()); + } + else + { + layer_bytes[0] = min_bytes; + num_layer_specs = 1; + std::string layer_string = llformat("Clayers=%d",num_layer_specs); + codestream.access_siz()->parse_string(layer_string.c_str()); + } + } + codestream.access_siz()->finalize_all(); + if (cpu_iterations >= 0) + { + codestream.collect_timing_stats(cpu_iterations); + } + codestream.change_appearance(transpose,vflip,hflip); + + // Now we are ready for sample data processing. + + int x_tnum; + kdu_dims tile_indices; codestream.get_valid_tiles(tile_indices); + kdc_flow_control **tile_flows = new kdc_flow_control *[tile_indices.size.x]; + for (x_tnum=0; x_tnum < tile_indices.size.x; x_tnum++) + { + tile_flows[x_tnum] = new kdc_flow_control(&mem_in,codestream,x_tnum,allow_shorts); + } + bool done = false; + + while (!done) + { + while (!done) + { // Process a row of tiles line by line. + done = true; + for (x_tnum=0; x_tnum < tile_indices.size.x; x_tnum++) + { + if (tile_flows[x_tnum]->advance_components()) + { + done = false; + tile_flows[x_tnum]->process_components(); + } + } + } + for (x_tnum=0; x_tnum < tile_indices.size.x; x_tnum++) + { + if (tile_flows[x_tnum]->advance_tile()) + { + done = false; + } + } + } + int sample_bytes = 0; + for (x_tnum=0; x_tnum < tile_indices.size.x; x_tnum++) + { + sample_bytes += tile_flows[x_tnum]->get_buffer_memory(); + delete tile_flows[x_tnum]; + } + delete [] tile_flows; + + // Produce the compressed output. + + codestream.flush(layer_bytes,num_layer_specs, NULL, true, false); + + // Cleanup + + codestream.destroy(); + if (record_stream != NULL) + { + delete record_stream; + } + + + // Now that we're done encoding, create the new data buffer for the compressed + // image and stick it there. + + base.copyData(output_buffer, output_size); + base.updateData(); // set width, height + delete[] output_buffer; + } + catch(const char* msg) + { + base.setLastError(ll_safe_string(msg)); + return FALSE; + } + catch( ... ) + { + base.setLastError( "Unknown J2C error" ); + return FALSE; + } + + return TRUE; +} + +BOOL LLImageJ2CKDU::getMetadata(LLImageJ2C &base) +{ + // *FIX: kdu calls our callback function if there's an error, and + // then bombs. To regain control, we throw an exception, and + // catch it here. + try + { + setupCodeStream(base, FALSE, MODE_FAST); + return TRUE; + } + catch( const char* msg ) + { + base.setLastError(ll_safe_string(msg)); + return FALSE; + } + catch( ... ) + { + base.setLastError( "Unknown J2C error" ); + return FALSE; + } + +} + + +void set_default_colour_weights(kdu_params *siz) +{ + kdu_params *cod = siz->access_cluster(COD_params); + assert(cod != NULL); + + bool can_use_ycc = true; + bool rev0=false; + int depth0=0, sub_x0=1, sub_y0=1; + for (int c=0; c < 3; c++) + { + int depth=0; siz->get(Sprecision,c,0,depth); + int sub_y=1; siz->get(Ssampling,c,0,sub_y); + int sub_x=1; siz->get(Ssampling,c,1,sub_x); + kdu_params *coc = cod->access_relation(-1,c); + bool rev=false; coc->get(Creversible,0,0,rev); + if (c == 0) + { rev0=rev; depth0=depth; sub_x0=sub_x; sub_y0=sub_y; } + else if ((rev != rev0) || (depth != depth0) || + (sub_x != sub_x0) || (sub_y != sub_y0)) + can_use_ycc = false; + } + if (!can_use_ycc) + return; + + bool use_ycc; + if (!cod->get(Cycc,0,0,use_ycc)) + cod->set(Cycc,0,0,use_ycc=true); + if (!use_ycc) + return; + float weight; + if (cod->get(Clev_weights,0,0,weight) || + cod->get(Cband_weights,0,0,weight)) + return; // Weights already specified explicitly. + + /* These example weights are adapted from numbers generated by Marcus Nadenau + at EPFL, for a viewing distance of 15 cm and a display resolution of + 300 DPI. */ + + cod->parse_string("Cband_weights:C0=" + "{0.0901},{0.2758},{0.2758}," + "{0.7018},{0.8378},{0.8378},{1}"); + cod->parse_string("Cband_weights:C1=" + "{0.0263},{0.0863},{0.0863}," + "{0.1362},{0.2564},{0.2564}," + "{0.3346},{0.4691},{0.4691}," + "{0.5444},{0.6523},{0.6523}," + "{0.7078},{0.7797},{0.7797},{1}"); + cod->parse_string("Cband_weights:C2=" + "{0.0773},{0.1835},{0.1835}," + "{0.2598},{0.4130},{0.4130}," + "{0.5040},{0.6464},{0.6464}," + "{0.7220},{0.8254},{0.8254}," + "{0.8769},{0.9424},{0.9424},{1}"); +} + + +/******************************************************************************/ +/* transfer_bytes */ +/******************************************************************************/ + +void transfer_bytes(kdu_byte *dest, kdu_line_buf &src, int gap, int precision) +/* Transfers source samples from the supplied line buffer into the output +byte buffer, spacing successive output samples apart by `gap' bytes +(to allow for interleaving of colour components). The function performs +all necessary level shifting, type conversion, rounding and truncation. */ +{ + int width = src.get_width(); + if (src.get_buf32() != NULL) + { // Decompressed samples have a 32-bit representation (integer or float) + assert(precision >= 8); // Else would have used 16 bit representation + kdu_sample32 *sp = src.get_buf32(); + if (!src.is_absolute()) + { // Transferring normalized floating point data. + float scale16 = (float)(1<<16); + kdu_int32 val; + + for (; width > 0; width--, sp++, dest+=gap) + { + val = (kdu_int32)(sp->fval*scale16); + val = (val+128)>>8; // May be faster than true rounding + val += 128; + if (val & ((-1)<<8)) + { + val = (val<0)?0:255; + } + *dest = (kdu_byte) val; + } + } + else + { // Transferring 32-bit absolute integers. + kdu_int32 val; + kdu_int32 downshift = precision-8; + kdu_int32 offset = (1<<downshift)>>1; + + for (; width > 0; width--, sp++, dest+=gap) + { + val = sp->ival; + val = (val+offset)>>downshift; + val += 128; + if (val & ((-1)<<8)) + { + val = (val<0)?0:255; + } + *dest = (kdu_byte) val; + } + } + } + else + { // Source data is 16 bits. + kdu_sample16 *sp = src.get_buf16(); + if (!src.is_absolute()) + { // Transferring 16-bit fixed point quantities + kdu_int16 val; + + if (precision >= 8) + { // Can essentially ignore the bit-depth. + for (; width > 0; width--, sp++, dest+=gap) + { + val = sp->ival; + val += (1<<(KDU_FIX_POINT-8))>>1; + val >>= (KDU_FIX_POINT-8); + val += 128; + if (val & ((-1)<<8)) + { + val = (val<0)?0:255; + } + *dest = (kdu_byte) val; + } + } + else + { // Need to force zeros into one or more least significant bits. + kdu_int16 downshift = KDU_FIX_POINT-precision; + kdu_int16 upshift = 8-precision; + kdu_int16 offset = 1<<(downshift-1); + + for (; width > 0; width--, sp++, dest+=gap) + { + val = sp->ival; + val = (val+offset)>>downshift; + val <<= upshift; + val += 128; + if (val & ((-1)<<8)) + { + val = (val<0)?0:(256-(1<<upshift)); + } + *dest = (kdu_byte) val; + } + } + } + else + { // Transferring 16-bit absolute integers. + kdu_int16 val; + + if (precision >= 8) + { + kdu_int16 downshift = precision-8; + kdu_int16 offset = (1<<downshift)>>1; + + for (; width > 0; width--, sp++, dest+=gap) + { + val = sp->ival; + val = (val+offset)>>downshift; + val += 128; + if (val & ((-1)<<8)) + { + val = (val<0)?0:255; + } + *dest = (kdu_byte) val; + } + } + else + { + kdu_int16 upshift = 8-precision; + + for (; width > 0; width--, sp++, dest+=gap) + { + val = sp->ival; + val <<= upshift; + val += 128; + if (val & ((-1)<<8)) + { + val = (val<0)?0:(256-(1<<upshift)); + } + *dest = (kdu_byte) val; + } + } + } + } +} + +LLKDUDecodeState::LLKDUDecodeState(kdu_tile tile, kdu_byte *buf, S32 row_gap) +{ + S32 c; + + mTile = tile; + mBuf = buf; + mRowGap = row_gap; + + mNumComponents = tile.get_num_components(); + + llassert(mNumComponents<=4); + mUseYCC = tile.get_ycc(); + + for (c=0; c<4; ++c) + { + mReversible[c] = false; + mBitDepths[c] = 0; + } + + // Open tile-components and create processing engines and resources + for (c=0; c < mNumComponents; c++) + { + mComps[c] = mTile.access_component(c); + mReversible[c] = mComps[c].get_reversible(); + mBitDepths[c] = mComps[c].get_bit_depth(); + kdu_resolution res = mComps[c].access_resolution(); // Get top resolution + kdu_dims comp_dims; res.get_dims(comp_dims); + if (c == 0) + { + mDims = comp_dims; + } + else + { + llassert(mDims == comp_dims); // Safety check; the caller has ensured this + } + bool use_shorts = (mComps[c].get_bit_depth(true) <= 16); + mLines[c].pre_create(&mAllocator,mDims.size.x,mReversible[c],use_shorts); + if (res.which() == 0) // No DWT levels used + { + mEngines[c] = kdu_decoder(res.access_subband(LL_BAND),&mAllocator,use_shorts); + } + else + { + mEngines[c] = kdu_synthesis(res,&mAllocator,use_shorts); + } + } + mAllocator.finalize(); // Actually creates buffering resources + for (c=0; c < mNumComponents; c++) + { + mLines[c].create(); // Grabs resources from the allocator. + } +} + +LLKDUDecodeState::~LLKDUDecodeState() +{ + S32 c; + // Cleanup + for (c=0; c < mNumComponents; c++) + { + mEngines[c].destroy(); // engines are interfaces; no default destructors + } + + mTile.close(); +} + +BOOL LLKDUDecodeState::processTileDecode(F32 decode_time, BOOL limit_time) +/* Decompresses a tile, writing the data into the supplied byte buffer. +The buffer contains interleaved image components, if there are any. +Although you may think of the buffer as belonging entirely to this tile, +the `buf' pointer may actually point into a larger buffer representing +multiple tiles. For this reason, `row_gap' is needed to identify the +separation between consecutive rows in the real buffer. */ +{ + S32 c; + // Now walk through the lines of the buffer, recovering them from the + // relevant tile-component processing engines. + + LLTimer decode_timer; + while (mDims.size.y--) + { + for (c=0; c < mNumComponents; c++) + { + mEngines[c].pull(mLines[c],true); + } + if ((mNumComponents >= 3) && mUseYCC) + { + kdu_convert_ycc_to_rgb(mLines[0],mLines[1],mLines[2]); + } + for (c=0; c < mNumComponents; c++) + { + transfer_bytes(mBuf+c,mLines[c],mNumComponents,mBitDepths[c]); + } + mBuf += mRowGap; + if (mDims.size.y % 10) + { + if (limit_time && decode_timer.getElapsedTimeF32() > decode_time) + { + return FALSE; + } + } + } + return TRUE; +} diff --git a/indra/llkdu/llimagej2ckdu.h b/indra/llkdu/llimagej2ckdu.h new file mode 100644 index 0000000000..5794ebdc68 --- /dev/null +++ b/indra/llkdu/llimagej2ckdu.h @@ -0,0 +1,92 @@ +/** + * @file llimagej2ckdu.h + * @brief This is an implementation of JPEG2000 encode/decode using Kakadu + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLIMAGEJ2CKDU_H +#define LL_LLIMAGEJ2CKDU_H + +#include "llimagej2c.h" + +// +// +// +// KDU core header files +#include "kdu/kdu_elementary.h" +#include "kdu/kdu_messaging.h" +#include "kdu/kdu_params.h" +#include "kdu/kdu_compressed.h" +#include "kdu/kdu_sample_processing.h" + +class LLKDUDecodeState; +class LLKDUMemSource; + +class LLImageJ2CKDU : public LLImageJ2CImpl +{ +public: + enum ECodeStreamMode + { + MODE_FAST = 0, + MODE_RESILIENT = 1, + MODE_FUSSY = 2 + }; + +public: + LLImageJ2CKDU(); + virtual ~LLImageJ2CKDU(); + +protected: + /*virtual*/ BOOL getMetadata(LLImageJ2C &base); + /*virtual*/ BOOL decodeImpl(LLImageJ2C &base, LLImageRaw &raw_image, F32 decode_time, S32 first_channel, S32 max_channel_count); + /*virtual*/ BOOL encodeImpl(LLImageJ2C &base, const LLImageRaw &raw_image, const char* comment_text, F32 encode_time=0.0, + BOOL reversible=FALSE); + + void setupCodeStream(LLImageJ2C &base, BOOL keep_codestream, ECodeStreamMode mode); + void cleanupCodeStream(); + BOOL initDecode(LLImageJ2C &base, LLImageRaw &raw_image, F32 decode_time, ECodeStreamMode mode, S32 first_channel, S32 max_channel_count ); + + // Encode variable + LLKDUMemSource *mInputp; + kdu_codestream *mCodeStreamp; + kdu_coords *mTPosp; // tile position + kdu_dims *mTileIndicesp; + + // Temporary variables for in-progress decodes... + LLImageRaw *mRawImagep; + LLKDUDecodeState *mDecodeState; +}; + +#if LL_WINDOWS +# define LLSYMEXPORT __declspec(dllexport) +#elif LL_LINUX +# define LLSYMEXPORT __attribute__ ((visibility("default"))) +#else +# define LLSYMEXPORT /**/ +#endif + +extern "C" LLSYMEXPORT const char* engineInfoLLImageJ2CKDU(); +extern "C" LLSYMEXPORT LLImageJ2CKDU* createLLImageJ2CKDU(); +extern "C" LLSYMEXPORT void destroyLLImageJ2CKDU(LLImageJ2CKDU* kdu); + +#endif diff --git a/indra/llkdu/llkdumem.cpp b/indra/llkdu/llkdumem.cpp new file mode 100644 index 0000000000..80f4c444d1 --- /dev/null +++ b/indra/llkdu/llkdumem.cpp @@ -0,0 +1,392 @@ + /** + * @file llkdumem.cpp + * @brief Helper class for kdu memory management + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "llkdumem.h" + +// Various image utility functions from kdu +#include "llerror.h" + +#if defined(LL_WINDOWS) +# pragma warning(disable: 4702) // unreachable code +#endif + +LLKDUMemIn::LLKDUMemIn(const U8 *data, + const U32 size, + const U16 width, + const U16 height, + const U8 in_num_components, + siz_params *siz) +{ + U8 n; + + first_comp_idx = 0; + rows = height; + cols = width; + num_components = in_num_components; + alignment_bytes = 0; + + for (n=0; n<3; ++n) + { + precision[n] = 0; + } + + for (n=0; n < num_components; ++n) + { + siz->set(Sdims,n,0,rows); + siz->set(Sdims,n,1,cols); + siz->set(Ssigned,n,0,false); + siz->set(Sprecision,n,0,8); + } + incomplete_lines = NULL; + free_lines = NULL; + num_unread_rows = rows; + + mData = data; + mDataSize = size; + mCurPos = 0; +} + + +LLKDUMemIn::~LLKDUMemIn() +{ + if ((num_unread_rows > 0) || (incomplete_lines != NULL)) + { kdu_warning w; + w << "Not all rows of image components " + << first_comp_idx << " through " + << first_comp_idx+num_components-1 + << " were consumed!"; + } + image_line_buf *tmp; + while ((tmp=incomplete_lines) != NULL) + { + incomplete_lines = tmp->next; + delete tmp; + } + while ((tmp=free_lines) != NULL) + { + free_lines = tmp->next; + delete tmp; + } +} + + +bool LLKDUMemIn::get(int comp_idx, kdu_line_buf &line, int x_tnum) +{ + int idx = comp_idx - this->first_comp_idx; + assert((idx >= 0) && (idx < num_components)); + x_tnum = x_tnum*num_components+idx; + image_line_buf *scan, *prev=NULL; + for (scan=incomplete_lines; scan != NULL; prev=scan, scan=scan->next) + { + assert(scan->next_x_tnum >= x_tnum); + if (scan->next_x_tnum == x_tnum) + { + break; + } + } + if (scan == NULL) + { // Need to read a new image line. + assert(x_tnum == 0); // Must consume in very specific order. + if (num_unread_rows == 0) + { + return false; + } + if ((scan = free_lines) == NULL) + { + scan = new image_line_buf(cols+3,num_components); + } + free_lines = scan->next; + if (prev == NULL) + { + incomplete_lines = scan; + } + else + { + prev->next = scan; + } + + // Copy from image buffer into scan. + memcpy(scan->buf, mData+mCurPos, cols*num_components); + mCurPos += cols*num_components; + + num_unread_rows--; + scan->accessed_samples = 0; + scan->next_x_tnum = 0; + } + + assert((cols-scan->accessed_samples) >= line.get_width()); + + int comp_offset = idx; + kdu_byte *sp = scan->buf+num_components*scan->accessed_samples + comp_offset; + int n=line.get_width(); + + if (line.get_buf32() != NULL) + { + kdu_sample32 *dp = line.get_buf32(); + if (line.is_absolute()) + { // 32-bit absolute integers + for (; n > 0; n--, sp+=num_components, dp++) + { + dp->ival = ((kdu_int32)(*sp)) - 128; + } + } + else + { // true 32-bit floats + for (; n > 0; n--, sp+=num_components, dp++) + { + dp->fval = (((float)(*sp)) / 256.0F) - 0.5F; + } + } + } + else + { + kdu_sample16 *dp = line.get_buf16(); + if (line.is_absolute()) + { // 16-bit absolute integers + for (; n > 0; n--, sp+=num_components, dp++) + { + dp->ival = ((kdu_int16)(*sp)) - 128; + } + } + else + { // 16-bit normalized representation. + for (; n > 0; n--, sp+=num_components, dp++) + { + dp->ival = (((kdu_int16)(*sp)) - 128) << (KDU_FIX_POINT-8); + } + } + } + + scan->next_x_tnum++; + if (idx == (num_components-1)) + { + scan->accessed_samples += line.get_width(); + } + if (scan->accessed_samples == cols) + { // Send empty line to free list. + assert(scan == incomplete_lines); + incomplete_lines = scan->next; + scan->next = free_lines; + free_lines = scan; + } + + return true; +} + + + +LLKDUMemOut::LLKDUMemOut(U8 *data, siz_params *siz, U8 in_num_components) +{ + int is_signed = 0; + int n; + + // Allocate memory segment + + first_comp_idx = 0; + if (!(siz->get(Scomponents,0,0,num_components) && + siz->get(Sdims,first_comp_idx,0,rows) && + siz->get(Sdims,first_comp_idx,1,cols) && + siz->get(Ssigned,first_comp_idx,0,is_signed))) + { + kdu_error e; e << "Attempting to create output image files before " + "all information is available concerning the image component " + "dimensions, bit-depth and signed/unsigned characteristics."; + } + num_components -= first_comp_idx; + + for (n=0; n < 3; n++) + { + precision[n] = 0; + } + + for (n=0; n < num_components; n++) + { + int prec; + + if (!(siz->compare(Sdims,first_comp_idx+n,0,rows) && + siz->compare(Sdims,first_comp_idx+n,1,cols) && + siz->compare(Ssigned,first_comp_idx+n,0,is_signed))) + { + assert(n > 0); + num_components = 1; + break; + } + if (!siz->get(Sprecision,first_comp_idx+n,0,prec)) + { + kdu_error e; e << "Attempting to create output image data before " + "all information is available concerning the image component " + "dimensions, bit-depth and signed/unsigned characteristics."; + } + llassert(n < 3); + precision[n] = prec; + } + if (is_signed) + { + kdu_warning w; + w << "Signed sample values will be written to the " + "BMP file as unsigned 8-bit quantities, centered about 128."; + } + + mCurPos = 0; + mData = data; + mDataSize = rows*cols*num_components; + + incomplete_lines = NULL; + free_lines = NULL; + num_unwritten_rows = rows; +} + +LLKDUMemOut::~LLKDUMemOut() +{ + if ((num_unwritten_rows > 0) || (incomplete_lines != NULL)) + { + kdu_warning w; + w << "Not all rows of image components " + << first_comp_idx << " through " + << first_comp_idx+num_components-1 + << " were completed!"; + } + image_line_buf *tmp; + + while ((tmp=incomplete_lines) != NULL) + { + incomplete_lines = tmp->next; + delete tmp; + } + + while ((tmp=free_lines) != NULL) + { + free_lines = tmp->next; + delete tmp; + } + + // Should either clean up or leave alone the data buffer... +// cout << "Done Destroying" << endl; +} + +void LLKDUMemOut::put(int comp_idx, kdu_line_buf &line, int x_tnum) +{ + int idx = 0; + + idx = comp_idx - this->first_comp_idx; + + assert((idx >= 0) && (idx < num_components)); + x_tnum = x_tnum*num_components+idx; + image_line_buf *scan, *prev=NULL; + for (scan=incomplete_lines; scan != NULL; prev=scan, scan=scan->next) + { + assert(scan->next_x_tnum >= x_tnum); + if (scan->next_x_tnum == x_tnum) + { + break; + } + } + if (scan == NULL) + { // Need to open a new line buffer + assert(x_tnum == 0); // Must consume in very specific order. + if ((scan = free_lines) == NULL) + { + scan = new image_line_buf(cols+3,num_components); + } + free_lines = scan->next; + if (prev == NULL) + { + incomplete_lines = scan; + } + else + { + prev->next = scan; + } + scan->accessed_samples = 0; + scan->next_x_tnum = 0; + } + + assert((cols-scan->accessed_samples) >= line.get_width()); + + int comp_offset = idx; + + if (line.get_buf32() != NULL) + { + if (line.is_absolute()) + { + convert_ints_to_bytes(line.get_buf32(), + scan->buf+num_components*scan->accessed_samples+comp_offset, + line.get_width(),precision[idx],num_components); + } + else + { + convert_floats_to_bytes(line.get_buf32(), + scan->buf+num_components*scan->accessed_samples+comp_offset, + line.get_width(),precision[idx],num_components); + } + } + else + { + if (line.is_absolute()) + { + convert_shorts_to_bytes(line.get_buf16(), + scan->buf+num_components*scan->accessed_samples+comp_offset, + line.get_width(),precision[idx],num_components); + } + else + { + convert_fixpoint_to_bytes(line.get_buf16(), + scan->buf+num_components*scan->accessed_samples+comp_offset, + line.get_width(),precision[idx],num_components); + } + } + + scan->next_x_tnum++; + if (idx == (num_components-1)) + { + scan->accessed_samples += line.get_width(); + } + if (scan->accessed_samples == cols) + { + // Write completed line and send it to the free list. + if (num_unwritten_rows == 0) + { + kdu_error e; e << "Attempting to write too many lines to image " + "file for components " << first_comp_idx << " through " + << first_comp_idx+num_components-1 << "."; + } + if ((mCurPos + cols*num_components) > mDataSize) + { + llerrs << "LLKDUMemOut::put() too much data" << llendl; + } + // Write the data to the output buffer. + memcpy(mData+mCurPos, scan->buf, cols*num_components); + mCurPos += cols*num_components; + + num_unwritten_rows--; + assert(scan == incomplete_lines); + incomplete_lines = scan->next; + scan->next = free_lines; + free_lines = scan; + } +} diff --git a/indra/llkdu/llkdumem.h b/indra/llkdu/llkdumem.h new file mode 100644 index 0000000000..fecb4653db --- /dev/null +++ b/indra/llkdu/llkdumem.h @@ -0,0 +1,167 @@ +/** + * @file llkdumem.h + * @brief Helper class for kdu memory management + * + * $LicenseInfo:firstyear=2010&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLKDUMEM_H +#define LL_LLKDUMEM_H + +// Support classes for reading and writing from memory buffers +// for KDU +#include "kdu_image.h" +#include "kdu/kdu_elementary.h" +#include "kdu/kdu_messaging.h" +#include "kdu/kdu_params.h" +#include "kdu/kdu_compressed.h" +#include "kdu/kdu_sample_processing.h" +#include "kdu_image_local.h" +#include "stdtypes.h" + +class LLKDUMemSource: public kdu_compressed_source +{ +public: // Member functions + LLKDUMemSource(U8 *input_buffer, U32 size) + { + mData = input_buffer; + mSize = size; + mCurPos = 0; + } + + ~LLKDUMemSource() + { + } + + int read(kdu_byte *buf, int num_bytes) + { + U32 num_out; + num_out = num_bytes; + + if ((mSize - mCurPos) < (U32)num_bytes) + { + num_out = mSize -mCurPos; + } + memcpy(buf, mData + mCurPos, num_out); + mCurPos += num_out; + return num_out; + } + + void reset() + { + mCurPos = 0; + } +private: // Data + U8 *mData; + U32 mSize; + U32 mCurPos; +}; + +class LLKDUMemTarget: public kdu_compressed_target +{ +public: // Member functions + LLKDUMemTarget(U8 *output_buffer, U32 &output_size, const U32 buffer_size) + { + mData = output_buffer; + mSize = buffer_size; + mCurPos = 0; + mOutputSize = &output_size; + } + + ~LLKDUMemTarget() + { + } + + bool write(const kdu_byte *buf, int num_bytes) + { + U32 num_out; + num_out = num_bytes; + + if ((mSize - mCurPos) < (U32)num_bytes) + { + num_out = mSize - mCurPos; + memcpy(mData + mCurPos, buf, num_out); + return false; + } + memcpy(mData + mCurPos, buf, num_out); + mCurPos += num_out; + *mOutputSize = mCurPos; + return true; + } +private: // Data + U8 *mData; + U32 mSize; + U32 mCurPos; + U32 *mOutputSize; +}; + + +class LLKDUMemIn : public kdu_image_in_base +{ +public: // Member functions + LLKDUMemIn(const U8 *data, + const U32 size, + const U16 rows, + const U16 cols, + U8 in_num_components, + siz_params *siz); + ~LLKDUMemIn(); + bool get(int comp_idx, kdu_line_buf &line, int x_tnum); + + const U8 *mData; +private: // Data + int first_comp_idx; + int num_components; + int rows, cols; + int alignment_bytes; // Number of 0's at end of each line. + int precision[3]; + image_line_buf *incomplete_lines; // Each "sample" represents a full pixel + image_line_buf *free_lines; + int num_unread_rows; + + U32 mCurPos; + U32 mDataSize; +}; + +class LLKDUMemOut : public kdu_image_out_base +{ +public: // Member functions + LLKDUMemOut(U8 *data, siz_params *siz, U8 in_num_components); + LLKDUMemOut(siz_params *siz, U8 in_num_components); + ~LLKDUMemOut(); + void put(int comp_idx, kdu_line_buf &line, int x_tnum); + + U8 *mData; +private: // Data + int first_comp_idx; + int num_components; + int rows, cols; + int precision[3]; + image_line_buf *incomplete_lines; // Each "sample" represents a full pixel + image_line_buf *free_lines; + int num_unwritten_rows; + + U32 mCurPos; + U32 mDataSize; +}; + +#endif diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt index bf885e5934..2515321a6c 100644 --- a/indra/newview/CMakeLists.txt +++ b/indra/newview/CMakeLists.txt @@ -290,7 +290,6 @@ set(viewer_SOURCE_FILES llmediadataclient.cpp llmemoryview.cpp llmenucommands.cpp - llmetricperformancetester.cpp llmimetypes.cpp llmorphview.cpp llmoveview.cpp @@ -822,7 +821,6 @@ set(viewer_HEADER_FILES llmediadataclient.h llmemoryview.h llmenucommands.h - llmetricperformancetester.h llmimetypes.h llmorphview.h llmoveview.h @@ -1446,11 +1444,6 @@ if (WINDOWS) # In the meantime, if you have any ideas on how to easily maintain one list, either here or in viewer_manifest.py # and have the build deps get tracked *please* tell me about it. - if(LLKDU_LIBRARY) - # Configure a var for llkdu which may not exist for all builds. - set(LLKDU_DLL_SOURCE ${SHARED_LIB_STAGING_DIR}/${CMAKE_CFG_INTDIR}/llkdu.dll) - endif(LLKDU_LIBRARY) - if(USE_GOOGLE_PERFTOOLS) # Configure a var for tcmalloc location, if used. # Note the need to specify multiple names explicitly. @@ -1467,7 +1460,6 @@ if (WINDOWS) #${SHARED_LIB_STAGING_DIR}/${CMAKE_CFG_INTDIR}/libtcmalloc_minimal.dll => None ... Skipping libtcmalloc_minimal.dll ${CMAKE_SOURCE_DIR}/../etc/message.xml ${CMAKE_SOURCE_DIR}/../scripts/messages/message_template.msg - ${LLKDU_DLL_SOURCE} ${SHARED_LIB_STAGING_DIR}/${CMAKE_CFG_INTDIR}/llcommon.dll ${SHARED_LIB_STAGING_DIR}/${CMAKE_CFG_INTDIR}/libapr-1.dll ${SHARED_LIB_STAGING_DIR}/${CMAKE_CFG_INTDIR}/libaprutil-1.dll @@ -1646,7 +1638,6 @@ target_link_libraries(${VIEWER_BINARY_NAME} ${LLAUDIO_LIBRARIES} ${LLCHARACTER_LIBRARIES} ${LLIMAGE_LIBRARIES} - ${LLIMAGEJ2COJ_LIBRARIES} ${LLINVENTORY_LIBRARIES} ${LLMESSAGE_LIBRARIES} ${LLPLUGIN_LIBRARIES} @@ -1682,6 +1673,17 @@ target_link_libraries(${VIEWER_BINARY_NAME} ${GOOGLE_PERFTOOLS_LIBRARIES} ) +if (LLKDU_LIBRARY) + target_link_libraries(${VIEWER_BINARY_NAME} + ${LLKDU_STATIC_LIBRARIES} + ${KDU_LIBRARY} + ) +else (LLKDU_LIBRARY) + target_link_libraries(${VIEWER_BINARY_NAME} + ${LLIMAGEJ2COJ_LIBRARIES} + ) +endif (LLKDU_LIBRARY) + build_version(viewer) set(ARTWORK_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE PATH diff --git a/indra/newview/app_settings/cmd_line.xml b/indra/newview/app_settings/cmd_line.xml index ba3b6a42a4..0562cf5480 100644 --- a/indra/newview/app_settings/cmd_line.xml +++ b/indra/newview/app_settings/cmd_line.xml @@ -118,6 +118,8 @@ <map> <key>desc</key> <string>Log metrics for benchmarking</string> + <key>count</key> + <integer>1</integer> <key>map-to</key> <string>LogMetrics</string> </map> diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 60ed37bdfb..bf0f948a6d 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -510,16 +510,10 @@ class LLFastTimerLogThread : public LLThread public: std::string mFile; - LLFastTimerLogThread() : LLThread("fast timer log") + LLFastTimerLogThread(std::string& test_name) : LLThread("fast timer log") { - if(LLFastTimer::sLog) - { - mFile = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "performance.slp"); - } - if(LLFastTimer::sMetricLog) - { - mFile = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "metric.slp"); - } + std::string file_name = test_name + std::string(".slp"); + mFile = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, file_name); } void run() @@ -535,6 +529,7 @@ public: os.close(); } + }; //virtual @@ -1643,22 +1638,16 @@ bool LLAppViewer::cleanup() { llinfos << "Analyzing performance" << llendl; - if(LLFastTimer::sLog) - { - LLFastTimerView::doAnalysis( - gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "performance_baseline.slp"), - gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "performance.slp"), - gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "performance_report.csv")); - } - if(LLFastTimer::sMetricLog) - { - LLFastTimerView::doAnalysis( - gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "metric_baseline.slp"), - gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "metric.slp"), - gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "metric_report.csv")); - } + std::string baseline_name = LLFastTimer::sLogName + "_baseline.slp"; + std::string current_name = LLFastTimer::sLogName + ".slp"; + std::string report_name = LLFastTimer::sLogName + "_report.csv"; + + LLFastTimerView::doAnalysis( + gDirUtilp->getExpandedFilename(LL_PATH_LOGS, baseline_name), + gDirUtilp->getExpandedFilename(LL_PATH_LOGS, current_name), + gDirUtilp->getExpandedFilename(LL_PATH_LOGS, report_name)); } - LLMetricPerformanceTester::cleanClass() ; + LLMetricPerformanceTesterBasic::cleanClass() ; llinfos << "Cleaning up Media and Textures" << llendflush; @@ -1765,7 +1754,7 @@ bool LLAppViewer::initThreads() if (LLFastTimer::sLog || LLFastTimer::sMetricLog) { LLFastTimer::sLogLock = new LLMutex(NULL); - mFastTimerLogThread = new LLFastTimerLogThread(); + mFastTimerLogThread = new LLFastTimerLogThread(LLFastTimer::sLogName); mFastTimerLogThread->start(); } @@ -2116,11 +2105,25 @@ bool LLAppViewer::initConfiguration() if (clp.hasOption("logperformance")) { LLFastTimer::sLog = TRUE; + LLFastTimer::sLogName = std::string("performance"); } - if(clp.hasOption("logmetrics")) + if (clp.hasOption("logmetrics")) { LLFastTimer::sMetricLog = TRUE ; + // '--logmetrics' can be specified with a named test metric argument so the data gathering is done only on that test + // In the absence of argument, every metric is gathered (makes for a rather slow run and hard to decipher report...) + std::string test_name = clp.getOption("logmetrics")[0]; + llinfos << "'--logmetrics' argument : " << test_name << llendl; + if (test_name == "") + { + llwarns << "No '--logmetrics' argument given, will output all metrics to " << DEFAULT_METRIC_NAME << llendl; + LLFastTimer::sLogName = DEFAULT_METRIC_NAME; + } + else + { + LLFastTimer::sLogName = test_name; + } } if (clp.hasOption("graphicslevel")) diff --git a/indra/newview/llappviewer.h b/indra/newview/llappviewer.h index a40cd83182..a14ab4362f 100644 --- a/indra/newview/llappviewer.h +++ b/indra/newview/llappviewer.h @@ -251,7 +251,9 @@ private: LLWatchdogTimeout* mMainloopTimeout; + // For performance and metric gathering LLThread* mFastTimerLogThread; + // for tracking viewer<->region circuit death bool mAgentRegionLastAlive; LLUUID mAgentRegionLastID; diff --git a/indra/newview/llfasttimerview.cpp b/indra/newview/llfasttimerview.cpp index a09c0ea0f8..92a3b9b2f5 100644 --- a/indra/newview/llfasttimerview.cpp +++ b/indra/newview/llfasttimerview.cpp @@ -1086,14 +1086,22 @@ LLSD LLFastTimerView::analyzePerformanceLogDefault(std::istream& is) //static void LLFastTimerView::doAnalysisDefault(std::string baseline, std::string target, std::string output) { + // Open baseline and current target, exit if one is inexistent + std::ifstream base_is(baseline.c_str()); + std::ifstream target_is(target.c_str()); + if (!base_is.is_open() || !target_is.is_open()) + { + llwarns << "'-analyzeperformance' error : baseline or current target file inexistent" << llendl; + base_is.close(); + target_is.close(); + return; + } //analyze baseline - std::ifstream base_is(baseline.c_str()); LLSD base = analyzePerformanceLogDefault(base_is); base_is.close(); //analyze current - std::ifstream target_is(target.c_str()); LLSD current = analyzePerformanceLogDefault(target_is); target_is.close(); @@ -1154,15 +1162,15 @@ LLSD LLFastTimerView::analyzeMetricPerformanceLog(std::istream& is) { std::string label = iter->first; - LLMetricPerformanceTester* tester = LLMetricPerformanceTester::getTester(iter->second["Name"].asString()) ; + LLMetricPerformanceTesterBasic* tester = LLMetricPerformanceTesterBasic::getTester(iter->second["Name"].asString()) ; if(tester) { ret[label]["Name"] = iter->second["Name"] ; - S32 num_of_strings = tester->getNumOfMetricStrings() ; - for(S32 index = 0 ; index < num_of_strings ; index++) + S32 num_of_metrics = tester->getNumberOfMetrics() ; + for(S32 index = 0 ; index < num_of_metrics ; index++) { - ret[label][ tester->getMetricString(index) ] = iter->second[ tester->getMetricString(index) ] ; + ret[label][ tester->getMetricName(index) ] = iter->second[ tester->getMetricName(index) ] ; } } } @@ -1172,20 +1180,43 @@ LLSD LLFastTimerView::analyzeMetricPerformanceLog(std::istream& is) } //static +void LLFastTimerView::outputAllMetrics() +{ + if (LLMetricPerformanceTesterBasic::hasMetricPerformanceTesters()) + { + for (LLMetricPerformanceTesterBasic::name_tester_map_t::iterator iter = LLMetricPerformanceTesterBasic::sTesterMap.begin(); + iter != LLMetricPerformanceTesterBasic::sTesterMap.end(); ++iter) + { + LLMetricPerformanceTesterBasic* tester = ((LLMetricPerformanceTesterBasic*)iter->second); + tester->outputTestResults(); + } + } +} + +//static void LLFastTimerView::doAnalysisMetrics(std::string baseline, std::string target, std::string output) { - if(!LLMetricPerformanceTester::hasMetricPerformanceTesters()) + if(!LLMetricPerformanceTesterBasic::hasMetricPerformanceTesters()) { return ; } - //analyze baseline + // Open baseline and current target, exit if one is inexistent std::ifstream base_is(baseline.c_str()); + std::ifstream target_is(target.c_str()); + if (!base_is.is_open() || !target_is.is_open()) + { + llwarns << "'-analyzeperformance' error : baseline or current target file inexistent" << llendl; + base_is.close(); + target_is.close(); + return; + } + + //analyze baseline LLSD base = analyzeMetricPerformanceLog(base_is); base_is.close(); //analyze current - std::ifstream target_is(target.c_str()); LLSD current = analyzeMetricPerformanceLog(target_is); target_is.close(); @@ -1193,10 +1224,10 @@ void LLFastTimerView::doAnalysisMetrics(std::string baseline, std::string target std::ofstream os(output.c_str()); os << "Label, Metric, Base(B), Target(T), Diff(T-B), Percentage(100*T/B)\n"; - for(LLMetricPerformanceTester::name_tester_map_t::iterator iter = LLMetricPerformanceTester::sTesterMap.begin() ; - iter != LLMetricPerformanceTester::sTesterMap.end() ; ++iter) + for(LLMetricPerformanceTesterBasic::name_tester_map_t::iterator iter = LLMetricPerformanceTesterBasic::sTesterMap.begin() ; + iter != LLMetricPerformanceTesterBasic::sTesterMap.end() ; ++iter) { - LLMetricPerformanceTester* tester = ((LLMetricPerformanceTester*)iter->second) ; + LLMetricPerformanceTesterBasic* tester = ((LLMetricPerformanceTesterBasic*)iter->second) ; tester->analyzePerformance(&os, &base, ¤t) ; } diff --git a/indra/newview/llfasttimerview.h b/indra/newview/llfasttimerview.h index 3788897cec..1a54a53f09 100644 --- a/indra/newview/llfasttimerview.h +++ b/indra/newview/llfasttimerview.h @@ -37,6 +37,7 @@ public: static BOOL sAnalyzePerformance; + static void outputAllMetrics(); static void doAnalysis(std::string baseline, std::string target, std::string output); private: diff --git a/indra/newview/llmetricperformancetester.cpp b/indra/newview/llmetricperformancetester.cpp deleted file mode 100644 index 903c97378e..0000000000 --- a/indra/newview/llmetricperformancetester.cpp +++ /dev/null @@ -1,252 +0,0 @@ -/** - * @file llmetricperformancetester.cpp - * @brief LLMetricPerformanceTester class implementation - * - * $LicenseInfo:firstyear=2004&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#include "llviewerprecompiledheaders.h" - -#include "indra_constants.h" -#include "llerror.h" -#include "llmath.h" -#include "llfontgl.h" -#include "llsdserialize.h" -#include "llstat.h" -#include "lltreeiterators.h" -#include "llmetricperformancetester.h" - -LLMetricPerformanceTester::name_tester_map_t LLMetricPerformanceTester::sTesterMap ; - -//static -void LLMetricPerformanceTester::initClass() -{ -} -//static -void LLMetricPerformanceTester::cleanClass() -{ - for(name_tester_map_t::iterator iter = sTesterMap.begin() ; iter != sTesterMap.end() ; ++iter) - { - delete iter->second ; - } - sTesterMap.clear() ; -} - -//static -void LLMetricPerformanceTester::addTester(LLMetricPerformanceTester* tester) -{ - if(!tester) - { - llerrs << "invalid tester!" << llendl ; - return ; - } - - std::string name = tester->getName() ; - if(getTester(name)) - { - llerrs << "Tester name is used by some other tester: " << name << llendl ; - return ; - } - - sTesterMap.insert(std::make_pair(name, tester)); - - return ; -} - -//static -LLMetricPerformanceTester* LLMetricPerformanceTester::getTester(std::string label) -{ - name_tester_map_t::iterator found_it = sTesterMap.find(label) ; - if(found_it != sTesterMap.end()) - { - return found_it->second ; - } - - return NULL ; -} - -LLMetricPerformanceTester::LLMetricPerformanceTester(std::string name, BOOL use_default_performance_analysis) - : mName(name), - mBaseSessionp(NULL), - mCurrentSessionp(NULL), - mCount(0), - mUseDefaultPerformanceAnalysis(use_default_performance_analysis) -{ - if(mName == std::string()) - { - llerrs << "invalid name." << llendl ; - } - - LLMetricPerformanceTester::addTester(this) ; -} - -/*virtual*/ -LLMetricPerformanceTester::~LLMetricPerformanceTester() -{ - if(mBaseSessionp) - { - delete mBaseSessionp ; - mBaseSessionp = NULL ; - } - if(mCurrentSessionp) - { - delete mCurrentSessionp ; - mCurrentSessionp = NULL ; - } -} - -void LLMetricPerformanceTester::incLabel() -{ - mCurLabel = llformat("%s-%d", mName.c_str(), mCount++) ; -} -void LLMetricPerformanceTester::preOutputTestResults(LLSD* sd) -{ - incLabel() ; - (*sd)[mCurLabel]["Name"] = mName ; -} -void LLMetricPerformanceTester::postOutputTestResults(LLSD* sd) -{ - LLMutexLock lock(LLFastTimer::sLogLock); - LLFastTimer::sLogQueue.push((*sd)); -} - -void LLMetricPerformanceTester::outputTestResults() -{ - LLSD sd ; - preOutputTestResults(&sd) ; - - outputTestRecord(&sd) ; - - postOutputTestResults(&sd) ; -} - -void LLMetricPerformanceTester::addMetricString(std::string str) -{ - mMetricStrings.push_back(str) ; -} - -const std::string& LLMetricPerformanceTester::getMetricString(U32 index) const -{ - return mMetricStrings[index] ; -} - -void LLMetricPerformanceTester::prePerformanceAnalysis() -{ - mCount = 0 ; - incLabel() ; -} - -// -//default analyzing the performance -// -/*virtual*/ -void LLMetricPerformanceTester::analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) -{ - if(mUseDefaultPerformanceAnalysis)//use default performance analysis - { - prePerformanceAnalysis() ; - - BOOL in_base = (*base).has(mCurLabel) ; - BOOL in_current = (*current).has(mCurLabel) ; - - while(in_base || in_current) - { - LLSD::String label = mCurLabel ; - - if(in_base && in_current) - { - *os << llformat("%s\n", label.c_str()) ; - - for(U32 index = 0 ; index < mMetricStrings.size() ; index++) - { - switch((*current)[label][ mMetricStrings[index] ].type()) - { - case LLSD::TypeInteger: - compareTestResults(os, mMetricStrings[index], - (S32)((*base)[label][ mMetricStrings[index] ].asInteger()), (S32)((*current)[label][ mMetricStrings[index] ].asInteger())) ; - break ; - case LLSD::TypeReal: - compareTestResults(os, mMetricStrings[index], - (F32)((*base)[label][ mMetricStrings[index] ].asReal()), (F32)((*current)[label][ mMetricStrings[index] ].asReal())) ; - break; - default: - llerrs << "unsupported metric " << mMetricStrings[index] << " LLSD type: " << (S32)(*current)[label][ mMetricStrings[index] ].type() << llendl ; - } - } - } - - incLabel() ; - in_base = (*base).has(mCurLabel) ; - in_current = (*current).has(mCurLabel) ; - } - }//end of default - else - { - //load the base session - prePerformanceAnalysis() ; - mBaseSessionp = loadTestSession(base) ; - - //load the current session - prePerformanceAnalysis() ; - mCurrentSessionp = loadTestSession(current) ; - - if(!mBaseSessionp || !mCurrentSessionp) - { - llerrs << "memory error during loading test sessions." << llendl ; - } - - //compare - compareTestSessions(os) ; - - //release memory - if(mBaseSessionp) - { - delete mBaseSessionp ; - mBaseSessionp = NULL ; - } - if(mCurrentSessionp) - { - delete mCurrentSessionp ; - mCurrentSessionp = NULL ; - } - } -} - -//virtual -void LLMetricPerformanceTester::compareTestResults(std::ofstream* os, std::string metric_string, S32 v_base, S32 v_current) -{ - *os << llformat(" ,%s, %d, %d, %d, %.4f\n", metric_string.c_str(), v_base, v_current, - v_current - v_base, (v_base != 0) ? 100.f * v_current / v_base : 0) ; -} - -//virtual -void LLMetricPerformanceTester::compareTestResults(std::ofstream* os, std::string metric_string, F32 v_base, F32 v_current) -{ - *os << llformat(" ,%s, %.4f, %.4f, %.4f, %.4f\n", metric_string.c_str(), v_base, v_current, - v_current - v_base, (fabs(v_base) > 0.0001f) ? 100.f * v_current / v_base : 0.f ) ; -} - -//virtual -LLMetricPerformanceTester::LLTestSession::~LLTestSession() -{ -} - diff --git a/indra/newview/llmetricperformancetester.h b/indra/newview/llmetricperformancetester.h deleted file mode 100644 index 6f5dc03564..0000000000 --- a/indra/newview/llmetricperformancetester.h +++ /dev/null @@ -1,153 +0,0 @@ -/** - * @file LLMetricPerformanceTester.h - * @brief LLMetricPerformanceTester class definition - * - * $LicenseInfo:firstyear=2004&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LL_METRICPERFORMANCETESTER_H -#define LL_METRICPERFORMANCETESTER_H - -class LLMetricPerformanceTester -{ -public: - // - //name passed to the constructor is a unique string for each tester. - //an error is reported if the name is already used by some other tester. - // - LLMetricPerformanceTester(std::string name, BOOL use_default_performance_analysis) ; - virtual ~LLMetricPerformanceTester(); - - // - //return the name of the tester - // - std::string getName() const { return mName ;} - // - //return the number of the test metrics in this tester - // - S32 getNumOfMetricStrings() const { return mMetricStrings.size() ;} - // - //return the metric string at the index - // - const std::string& getMetricString(U32 index) const ; - - // - //this function to compare the test results. - //by default, it compares the test results against the baseline one by one, item by item, - //in the increasing order of the LLSD label counter, starting from the first one. - //you can define your own way to analyze performance by passing FALSE to "use_default_performance_analysis", - //and implement two abstract virtual functions below: loadTestSession(...) and compareTestSessions(...). - // - void analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) ; - -protected: - // - //insert metric strings used in the tester. - // - void addMetricString(std::string str) ; - - // - //increase LLSD label by 1 - // - void incLabel() ; - - // - //the function to write a set of test results to the log LLSD. - // - void outputTestResults() ; - - // - //compare the test results. - //you can write your own to overwrite the default one. - // - virtual void compareTestResults(std::ofstream* os, std::string metric_string, S32 v_base, S32 v_current) ; - virtual void compareTestResults(std::ofstream* os, std::string metric_string, F32 v_base, F32 v_current) ; - - // - //for performance analysis use - //it defines an interface for the two abstract virtual functions loadTestSession(...) and compareTestSessions(...). - //please make your own test session class derived from it. - // - class LLTestSession - { - public: - virtual ~LLTestSession() ; - }; - - // - //load a test session for log LLSD - //you need to implement it only when you define your own way to analyze performance. - //otherwise leave it empty. - // - virtual LLMetricPerformanceTester::LLTestSession* loadTestSession(LLSD* log) = 0 ; - // - //compare the base session and the target session - //you need to implement it only when you define your own way to analyze performance. - //otherwise leave it empty. - // - virtual void compareTestSessions(std::ofstream* os) = 0 ; - // - //the function to write a set of test results to the log LLSD. - //you have to write you own version of this function. - // - virtual void outputTestRecord(LLSD* sd) = 0 ; - -private: - void preOutputTestResults(LLSD* sd) ; - void postOutputTestResults(LLSD* sd) ; - void prePerformanceAnalysis() ; - -protected: - // - //the unique name string of the tester - // - std::string mName ; - // - //the current label counter for the log LLSD - // - std::string mCurLabel ; - S32 mCount ; - - BOOL mUseDefaultPerformanceAnalysis ; - LLTestSession* mBaseSessionp ; - LLTestSession* mCurrentSessionp ; - - //metrics strings - std::vector< std::string > mMetricStrings ; - -//static members -private: - static void addTester(LLMetricPerformanceTester* tester) ; - -public: - typedef std::map< std::string, LLMetricPerformanceTester* > name_tester_map_t; - static name_tester_map_t sTesterMap ; - - static LLMetricPerformanceTester* getTester(std::string label) ; - static BOOL hasMetricPerformanceTesters() {return !sTesterMap.empty() ;} - - static void initClass() ; - static void cleanClass() ; -}; - -#endif - diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index d6d38de225..13fd51f473 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1574,7 +1574,6 @@ bool LLTextureFetch::createRequest(const std::string& url, const LLUUID& id, con if (!url.empty() && (!exten.empty() && LLImageBase::getCodecFromExtension(exten) != IMG_CODEC_J2C)) { // Only do partial requests for J2C at the moment - //llinfos << "Merov : LLTextureFetch::createRequest(), blocking fetch on " << url << llendl; desired_size = MAX_IMAGE_DATA_SIZE; desired_discard = 0; } diff --git a/indra/newview/llviewertexture.cpp b/indra/newview/llviewertexture.cpp index f96b93da4d..6160510c0e 100644 --- a/indra/newview/llviewertexture.cpp +++ b/indra/newview/llviewertexture.cpp @@ -72,6 +72,7 @@ LLPointer<LLViewerFetchedTexture> LLViewerFetchedTexture::sDefaultImagep = NULL; LLPointer<LLViewerFetchedTexture> LLViewerFetchedTexture::sSmokeImagep = NULL; LLViewerMediaTexture::media_map_t LLViewerMediaTexture::sMediaMap ; LLTexturePipelineTester* LLViewerTextureManager::sTesterp = NULL ; +const std::string sTesterName("TextureTester"); S32 LLViewerTexture::sImageCount = 0; S32 LLViewerTexture::sRawCount = 0; @@ -341,9 +342,14 @@ void LLViewerTextureManager::init() LLViewerTexture::initClass() ; - if(LLFastTimer::sMetricLog) + if (LLMetricPerformanceTesterBasic::isMetricLogRequested(sTesterName) && !LLMetricPerformanceTesterBasic::getTester(sTesterName)) { - LLViewerTextureManager::sTesterp = new LLTexturePipelineTester() ; + sTesterp = new LLTexturePipelineTester() ; + if (!sTesterp->isValid()) + { + delete sTesterp; + sTesterp = NULL; + } } } @@ -408,9 +414,10 @@ void LLViewerTexture::updateClass(const F32 velocity, const F32 angular_velocity { sCurrentTime = gFrameTimeSeconds ; - if(LLViewerTextureManager::sTesterp) + LLTexturePipelineTester* tester = (LLTexturePipelineTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) { - LLViewerTextureManager::sTesterp->update() ; + tester->update() ; } LLViewerMediaTexture::updateClass() ; @@ -603,9 +610,10 @@ bool LLViewerTexture::bindDefaultImage(S32 stage) //check if there is cached raw image and switch to it if possible switchToCachedImage() ; - if(LLViewerTextureManager::sTesterp) + LLTexturePipelineTester* tester = (LLTexturePipelineTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) { - LLViewerTextureManager::sTesterp->updateGrayTextureBinding() ; + tester->updateGrayTextureBinding() ; } return res; } @@ -1066,9 +1074,10 @@ BOOL LLViewerTexture::isLargeImage() //virtual void LLViewerTexture::updateBindStatsForTester() { - if(LLViewerTextureManager::sTesterp) + LLTexturePipelineTester* tester = (LLTexturePipelineTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) { - LLViewerTextureManager::sTesterp->updateTextureBindingStats(this) ; + tester->updateTextureBindingStats(this) ; } } @@ -1849,10 +1858,11 @@ bool LLViewerFetchedTexture::updateFetch() // We may have data ready regardless of whether or not we are finished (e.g. waiting on write) if (mRawImage.notNull()) { - if(LLViewerTextureManager::sTesterp) + LLTexturePipelineTester* tester = (LLTexturePipelineTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) { mIsFetched = TRUE ; - LLViewerTextureManager::sTesterp->updateTextureLoadingStats(this, mRawImage, LLAppViewer::getTextureFetch()->isFromLocalCache(mID)) ; + tester->updateTextureLoadingStats(this, mRawImage, LLAppViewer::getTextureFetch()->isFromLocalCache(mID)) ; } mRawDiscardLevel = fetch_discard; if ((mRawImage->getDataSize() > 0 && mRawDiscardLevel >= 0) && @@ -3076,9 +3086,10 @@ void LLViewerLODTexture::scaleDown() { switchToCachedImage() ; - if(LLViewerTextureManager::sTesterp) + LLTexturePipelineTester* tester = (LLTexturePipelineTester*)LLMetricPerformanceTesterBasic::getTester(sTesterName); + if (tester) { - LLViewerTextureManager::sTesterp->setStablizingTime() ; + tester->setStablizingTime() ; } } } @@ -3588,23 +3599,22 @@ F32 LLViewerMediaTexture::getMaxVirtualSize() //---------------------------------------------------------------------------------------------- //start of LLTexturePipelineTester //---------------------------------------------------------------------------------------------- -LLTexturePipelineTester::LLTexturePipelineTester() : - LLMetricPerformanceTester("TextureTester", FALSE) -{ - addMetricString("TotalBytesLoaded") ; - addMetricString("TotalBytesLoadedFromCache") ; - addMetricString("TotalBytesLoadedForLargeImage") ; - addMetricString("TotalBytesLoadedForSculpties") ; - addMetricString("StartFetchingTime") ; - addMetricString("TotalGrayTime") ; - addMetricString("TotalStablizingTime") ; - addMetricString("StartTimeLoadingSculpties") ; - addMetricString("EndTimeLoadingSculpties") ; - - addMetricString("Time") ; - addMetricString("TotalBytesBound") ; - addMetricString("TotalBytesBoundForLargeImage") ; - addMetricString("PercentageBytesBound") ; +LLTexturePipelineTester::LLTexturePipelineTester() : LLMetricPerformanceTesterWithSession(sTesterName) +{ + addMetric("TotalBytesLoaded") ; + addMetric("TotalBytesLoadedFromCache") ; + addMetric("TotalBytesLoadedForLargeImage") ; + addMetric("TotalBytesLoadedForSculpties") ; + addMetric("StartFetchingTime") ; + addMetric("TotalGrayTime") ; + addMetric("TotalStablizingTime") ; + addMetric("StartTimeLoadingSculpties") ; + addMetric("EndTimeLoadingSculpties") ; + + addMetric("Time") ; + addMetric("TotalBytesBound") ; + addMetric("TotalBytesBoundForLargeImage") ; + addMetric("PercentageBytesBound") ; mTotalBytesLoaded = 0 ; mTotalBytesLoadedFromCache = 0 ; @@ -3616,7 +3626,7 @@ LLTexturePipelineTester::LLTexturePipelineTester() : LLTexturePipelineTester::~LLTexturePipelineTester() { - LLViewerTextureManager::sTesterp = NULL ; + LLViewerTextureManager::sTesterp = NULL; } void LLTexturePipelineTester::update() @@ -3682,22 +3692,23 @@ void LLTexturePipelineTester::reset() //virtual void LLTexturePipelineTester::outputTestRecord(LLSD *sd) { - (*sd)[mCurLabel]["TotalBytesLoaded"] = (LLSD::Integer)mTotalBytesLoaded ; - (*sd)[mCurLabel]["TotalBytesLoadedFromCache"] = (LLSD::Integer)mTotalBytesLoadedFromCache ; - (*sd)[mCurLabel]["TotalBytesLoadedForLargeImage"] = (LLSD::Integer)mTotalBytesLoadedForLargeImage ; - (*sd)[mCurLabel]["TotalBytesLoadedForSculpties"] = (LLSD::Integer)mTotalBytesLoadedForSculpties ; + std::string currentLabel = getCurrentLabelName(); + (*sd)[currentLabel]["TotalBytesLoaded"] = (LLSD::Integer)mTotalBytesLoaded ; + (*sd)[currentLabel]["TotalBytesLoadedFromCache"] = (LLSD::Integer)mTotalBytesLoadedFromCache ; + (*sd)[currentLabel]["TotalBytesLoadedForLargeImage"] = (LLSD::Integer)mTotalBytesLoadedForLargeImage ; + (*sd)[currentLabel]["TotalBytesLoadedForSculpties"] = (LLSD::Integer)mTotalBytesLoadedForSculpties ; - (*sd)[mCurLabel]["StartFetchingTime"] = (LLSD::Real)mStartFetchingTime ; - (*sd)[mCurLabel]["TotalGrayTime"] = (LLSD::Real)mTotalGrayTime ; - (*sd)[mCurLabel]["TotalStablizingTime"] = (LLSD::Real)mTotalStablizingTime ; + (*sd)[currentLabel]["StartFetchingTime"] = (LLSD::Real)mStartFetchingTime ; + (*sd)[currentLabel]["TotalGrayTime"] = (LLSD::Real)mTotalGrayTime ; + (*sd)[currentLabel]["TotalStablizingTime"] = (LLSD::Real)mTotalStablizingTime ; - (*sd)[mCurLabel]["StartTimeLoadingSculpties"] = (LLSD::Real)mStartTimeLoadingSculpties ; - (*sd)[mCurLabel]["EndTimeLoadingSculpties"] = (LLSD::Real)mEndTimeLoadingSculpties ; + (*sd)[currentLabel]["StartTimeLoadingSculpties"] = (LLSD::Real)mStartTimeLoadingSculpties ; + (*sd)[currentLabel]["EndTimeLoadingSculpties"] = (LLSD::Real)mEndTimeLoadingSculpties ; - (*sd)[mCurLabel]["Time"] = LLImageGL::sLastFrameTime ; - (*sd)[mCurLabel]["TotalBytesBound"] = (LLSD::Integer)mLastTotalBytesUsed ; - (*sd)[mCurLabel]["TotalBytesBoundForLargeImage"] = (LLSD::Integer)mLastTotalBytesUsedForLargeImage ; - (*sd)[mCurLabel]["PercentageBytesBound"] = (LLSD::Real)(100.f * mLastTotalBytesUsed / mTotalBytesLoaded) ; + (*sd)[currentLabel]["Time"] = LLImageGL::sLastFrameTime ; + (*sd)[currentLabel]["TotalBytesBound"] = (LLSD::Integer)mLastTotalBytesUsed ; + (*sd)[currentLabel]["TotalBytesBoundForLargeImage"] = (LLSD::Integer)mLastTotalBytesUsedForLargeImage ; + (*sd)[currentLabel]["PercentageBytesBound"] = (LLSD::Real)(100.f * mLastTotalBytesUsed / mTotalBytesLoaded) ; } void LLTexturePipelineTester::updateTextureBindingStats(const LLViewerTexture* imagep) @@ -3786,7 +3797,7 @@ void LLTexturePipelineTester::compareTestSessions(std::ofstream* os) } //compare and output the comparison - *os << llformat("%s\n", mName.c_str()) ; + *os << llformat("%s\n", getTesterName().c_str()) ; *os << llformat("AggregateResults\n") ; compareTestResults(os, "TotalFetchingTime", base_sessionp->mTotalFetchingTime, current_sessionp->mTotalFetchingTime) ; @@ -3841,7 +3852,7 @@ void LLTexturePipelineTester::compareTestSessions(std::ofstream* os) } //virtual -LLMetricPerformanceTester::LLTestSession* LLTexturePipelineTester::loadTestSession(LLSD* log) +LLMetricPerformanceTesterWithSession::LLTestSession* LLTexturePipelineTester::loadTestSession(LLSD* log) { LLTexturePipelineTester::LLTextureTestSession* sessionp = new LLTexturePipelineTester::LLTextureTestSession() ; if(!sessionp) @@ -3868,12 +3879,11 @@ LLMetricPerformanceTester::LLTestSession* LLTexturePipelineTester::loadTestSessi sessionp->mInstantPerformanceList[sessionp->mInstantPerformanceListCounter].mTime = 0.f ; //load a session - BOOL in_log = (*log).has(mCurLabel) ; - while(in_log) + std::string currentLabel = getCurrentLabelName(); + BOOL in_log = (*log).has(currentLabel) ; + while (in_log) { - LLSD::String label = mCurLabel ; - incLabel() ; - in_log = (*log).has(mCurLabel) ; + LLSD::String label = currentLabel ; if(sessionp->mInstantPerformanceListCounter >= (S32)sessionp->mInstantPerformanceList.size()) { @@ -3939,7 +3949,11 @@ LLMetricPerformanceTester::LLTestSession* LLTexturePipelineTester::loadTestSessi sessionp->mInstantPerformanceList[sessionp->mInstantPerformanceListCounter].mAverageBytesUsedForLargeImagePerSecond = 0 ; sessionp->mInstantPerformanceList[sessionp->mInstantPerformanceListCounter].mAveragePercentageBytesUsedPerSecond = 0.f ; sessionp->mInstantPerformanceList[sessionp->mInstantPerformanceListCounter].mTime = 0.f ; - } + } + // Next label + incrementCurrentCount() ; + currentLabel = getCurrentLabelName(); + in_log = (*log).has(currentLabel) ; } sessionp->mTotalFetchingTime += total_fetching_time ; diff --git a/indra/newview/llviewertexture.h b/indra/newview/llviewertexture.h index b779396293..b5636bbdc7 100644 --- a/indra/newview/llviewertexture.h +++ b/indra/newview/llviewertexture.h @@ -735,7 +735,7 @@ public: //it tracks the activities of the texture pipeline //records them, and outputs them to log files // -class LLTexturePipelineTester : public LLMetricPerformanceTester +class LLTexturePipelineTester : public LLMetricPerformanceTesterWithSession { enum { @@ -751,8 +751,6 @@ public: void updateGrayTextureBinding() ; void setStablizingTime() ; - /*virtual*/ void analyzePerformance(std::ofstream* os, LLSD* base, LLSD* current) ; - private: void reset() ; void updateStablizingTime() ; @@ -823,7 +821,7 @@ private: S32 mInstantPerformanceListCounter ; }; - /*virtual*/ LLMetricPerformanceTester::LLTestSession* loadTestSession(LLSD* log) ; + /*virtual*/ LLMetricPerformanceTesterWithSession::LLTestSession* loadTestSession(LLSD* log) ; /*virtual*/ void compareTestSessions(std::ofstream* os) ; }; diff --git a/install.xml b/install.xml index 391d83b224..db148f1c61 100644 --- a/install.xml +++ b/install.xml @@ -830,23 +830,30 @@ <key>darwin</key> <map> <key>md5sum</key> - <string>ae18dd120807a46ac961b881a631ad94</string> + <string>3b40e7170dea82c1443e8d90cd44a13d</string> <key>url</key> - <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/indra_private-2.1.1-darwin-20100820.tar.bz2</uri> + <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/kdu-4.2.1-darwin-20080926.tar.bz2</uri> </map> <key>linux</key> <map> <key>md5sum</key> - <string>b1f15bbabb68445e55ce23a2aeaca598</string> + <string>a6d2f0995c25d7f53bd12b8ec0d6b462</string> <key>url</key> - <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/indra_private-2.1.1-linux-20100826.tar.bz2</uri> + <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/kdu-4.2.1-linux-20080930.tar.bz2</uri> + </map> + <key>linux64</key> + <map> + <key>md5sum</key> + <string>f4e2e2b3440594527729a8c85119e508</string> + <key>url</key> + <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/kdu-5.2.1-linux64-20080926.tar.bz2</uri> </map> <key>windows</key> <map> <key>md5sum</key> - <string>0e2fe621ce99085eba00d86d9a3bc130</string> + <string>1b9f61140f8b599cdae5e00d21dbb177</string> <key>url</key> - <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/indra_private-2.1.1-windows-20100820.tar.bz2</uri> + <uri>scp:install-packages.lindenlab.com:/local/www/install-packages/doc/kdu-4.2.1-windows-20080926.tar.bz2</uri> </map> </map> </map> |