From c8726aba303bcf1207b730a344536e25491420bc Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Tue, 10 Nov 2015 09:48:56 -0500 Subject: remove execute permission from many files that should not have it --- indra/llcrashlogger/CMakeLists.txt | 0 indra/llcrashlogger/llcrashlogger.cpp | 0 indra/llcrashlogger/llcrashlogger.h | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 indra/llcrashlogger/CMakeLists.txt mode change 100755 => 100644 indra/llcrashlogger/llcrashlogger.cpp mode change 100755 => 100644 indra/llcrashlogger/llcrashlogger.h (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/CMakeLists.txt b/indra/llcrashlogger/CMakeLists.txt old mode 100755 new mode 100644 diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp old mode 100755 new mode 100644 diff --git a/indra/llcrashlogger/llcrashlogger.h b/indra/llcrashlogger/llcrashlogger.h old mode 100755 new mode 100644 -- cgit v1.2.3 From 57488bc3812d9d308cc9e869c2088ba8212bc851 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 13 Apr 2016 14:41:38 -0400 Subject: add logging around crash reporting, with minor code cleanups --- indra/llcrashlogger/llcrashlogger.cpp | 65 +++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 29 deletions(-) (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 91f6e469d1..8e0a725954 100644 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -70,12 +70,15 @@ protected: void LLCrashLoggerHandler::onSuccess(LLCore::HttpResponse * response, const LLSD &content) { + LL_DEBUGS("CRASHREPORT") << "Request to " << response->getRequestURL() << "succeeded" << LL_ENDL; gBreak = true; gSent = true; } void LLCrashLoggerHandler::onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status) { + LL_WARNS("CRASHREPORT") << "Request to " << response->getRequestURL() + << " failed: " << status.toString() << LL_ENDL; gBreak = true; } @@ -231,8 +234,8 @@ void LLCrashLogger::gatherFiles() LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL); } - LL_INFOS() << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL; - LL_INFOS() << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL; } else { @@ -268,25 +271,27 @@ void LLCrashLogger::gatherFiles() for(std::map::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr) { std::ifstream f((*itr).second.c_str()); - if(!f.is_open()) - { - LL_INFOS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL; - continue; - } - std::stringstream s; - s << f.rdbuf(); + if(f.is_open()) + { + std::stringstream s; + s << f.rdbuf(); - std::string crash_info = s.str(); - if(itr->first == "SecondLifeLog") - { - if(!mCrashInfo["DebugLog"].has("StartupState")) - { - mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); - } - trimSLLog(crash_info); - } + std::string crash_info = s.str(); + if(itr->first == "SecondLifeLog") + { + if(!mCrashInfo["DebugLog"].has("StartupState")) + { + mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + } + trimSLLog(crash_info); + } - mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + } + else + { + LL_WARNS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL; + } } std::string minidump_path; @@ -376,6 +381,7 @@ bool LLCrashLogger::runCrashLogPost(std::string host, LLSD data, std::string msg { updateApplication(llformat("%s, try %d...", msg.c_str(), i+1)); + LL_INFOS("CRASHREPORT") << "POST crash data to " << host << LL_ENDL; LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(httpRequest.get(), LLCore::HttpRequest::DEFAULT_POLICY_ID, 0, host, data, httpOpts, LLCore::HttpHeaders::ptr_t(), LLCore::HttpHandler::ptr_t(new LLCrashLoggerHandler)); @@ -412,6 +418,8 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) "SecondLifeCrashReport"); std::string report_file = dump_path + ".log"; + LL_DEBUGS("CRASHREPORT") << "sending " << report_file << LL_ENDL; + gatherFiles(); LLSD post_data; @@ -426,11 +434,11 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) bool sent = false; - //*TODO: Translate - updateApplication("DEBUG: crash host in send logs "+mCrashHost); - if(mCrashHost != "") - { - std::string msg = "Using derived crash server... "; + if(mCrashHost != "") + { + LL_WARNS("CRASHREPORT") << "Sending crash data to server from CrashHostUrl '" << mCrashHost << "'" << LL_ENDL; + + std::string msg = "Using override crash server... "; msg = msg+mCrashHost.c_str(); updateApplication(msg.c_str()); @@ -470,11 +478,13 @@ bool LLCrashLogger::sendCrashLogs() void LLCrashLogger::updateApplication(const std::string& message) { - if (!message.empty()) LL_INFOS() << message << LL_ENDL; + if (!message.empty()) LL_INFOS("CRASHREPORT") << message << LL_ENDL; } bool LLCrashLogger::init() { + LL_DEBUGS("CRASHREPORT") << LL_ENDL; + LLCore::LLHttp::initialize(); // We assume that all the logs we're looking for reside on the current drive @@ -498,7 +508,7 @@ bool LLCrashLogger::init() // Set the log file to crashreport.log LLError::logToFile(log_file); //NOTE: Until this line, LL_INFOS LL_WARNS, etc are blown to the ether. - LL_INFOS() << "Crash reporter file rotation complete." << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL; mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND, "Controls behavior when viewer crashes " @@ -506,9 +516,6 @@ bool LLCrashLogger::init() "1 = always send crash report, " "2 = never send crash report)"); - // LL_INFOS() << "Loading crash behavior setting" << LL_ENDL; - // mCrashBehavior = loadCrashBehaviorSetting(); - init_curl(); LLCore::HttpRequest::createService(); LLCore::HttpRequest::startThread(); -- cgit v1.2.3 From ecdb190d70a81294eebde7e53cf6a92139ba53d5 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 20 Apr 2016 11:52:00 -0400 Subject: MAINT-6322 fix merge error that prevented crash dumps from being located for upload (and add better logging) --- indra/llcrashlogger/llcrashlogger.cpp | 150 +++++++++++++++++++++++++++------- 1 file changed, 121 insertions(+), 29 deletions(-) (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 8e0a725954..4bfc2d40dd 100644 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -162,6 +162,10 @@ bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename ) log_file.close(); return true; } + else + { + LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL; + } return false; } @@ -194,6 +198,11 @@ bool LLCrashLogger::readMinidump(std::string minidump_path) mCrashInfo["Minidump"] = data; } + else + { + LL_WARNS("CRASHREPORT") << "failed to open minidump "<0?true:false); } @@ -270,27 +279,36 @@ void LLCrashLogger::gatherFiles() for(std::map::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr) { - std::ifstream f((*itr).second.c_str()); - if(f.is_open()) + std::string file = (*itr).second; + if (!file.empty()) { - std::stringstream s; - s << f.rdbuf(); - - std::string crash_info = s.str(); - if(itr->first == "SecondLifeLog") + LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL; + std::ifstream f(file.c_str()); + if(f.is_open()) { - if(!mCrashInfo["DebugLog"].has("StartupState")) + std::stringstream s; + s << f.rdbuf(); + + std::string crash_info = s.str(); + if(itr->first == "SecondLifeLog") { - mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + if(!mCrashInfo["DebugLog"].has("StartupState")) + { + mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + } + trimSLLog(crash_info); } - trimSLLog(crash_info); - } - mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + } + else + { + LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL; + } } else { - LL_WARNS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL; + LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL; } } @@ -301,20 +319,21 @@ void LLCrashLogger::gatherFiles() if (has_minidump) { minidump_path = mDebugLog["MinidumpPath"].asString(); - } - - if (has_minidump) - { has_minidump = readMinidump(minidump_path); } + else + { + LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL; + } if (!has_minidump) //Viewer was probably so hosed it couldn't write remaining data. Try brute force. { - //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file. + //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file. typedef std::vector vec; std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,""); + LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL; vec file_vec = gDirUtilp->getFilesInDir(pathname); - for(vec::const_iterator iter=file_vec.begin(); iter!=file_vec.end(); ++iter) + for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter) { if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) ) { @@ -330,15 +349,27 @@ void LLCrashLogger::gatherFiles() minidump_path = *iter; has_minidump = readMinidump(fullname); mDebugLog["MinidumpPath"] = fullname; - if (has_minidump) - { - break; - } + } + else + { + LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL; } } + else + { + LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL; + } } + else + { + LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL; + } } } + else + { + LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL; + } } LLSD LLCrashLogger::constructPostData() @@ -458,22 +489,63 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) bool LLCrashLogger::sendCrashLogs() { - + LLSD locks = mKeyMaster.getProcessList(); + LLSD newlocks = LLSD::emptyArray(); + LLSD opts = getOptionData(PRIORITY_COMMAND_LINE); LLSD rec; - if ( opts.has("dumpdir") ) + if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") ) { rec["pid"]=opts["pid"]; rec["dumpdir"]=opts["dumpdir"]; rec["procname"]=opts["procname"]; } - else + + if (locks.isArray()) { - return false; - } + for (LLSD::array_iterator lock=locks.beginArray(); + lock !=locks.endArray(); + ++lock) + { + if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") ) + { + if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) ) + { + newlocks.append(*lock); + } + else + { + //TODO: This is a hack but I didn't want to include boost in another file or retest everything related to lldir + if (LLCrashLock::fileExists((*lock)["dumpdir"].asString())) + { + //the viewer cleans up the log directory on clean shutdown + //but is ignorant of the locking table. + if (!sendCrashLog((*lock)["dumpdir"].asString())) + { + newlocks.append(*lock); //Failed to send log so don't delete it. + } + else + { + mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString()); + } + } + } + } + else + { + LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL; + } + } + } - return sendCrashLog(rec["dumpdir"].asString()); + if (rec) + { + newlocks.append(rec); + } + + mKeyMaster.putProcessList(newlocks); + return true; } void LLCrashLogger::updateApplication(const std::string& message) @@ -510,6 +582,26 @@ bool LLCrashLogger::init() LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL; + // Handle locking + bool locked = mKeyMaster.requestMaster(); //Request master locking file. wait time is defaulted to 300S + + while (!locked && mKeyMaster.isWaiting()) + { + LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL; +#if LL_WINDOWS + Sleep(1000); +#else + sleep(1); +#endif + locked = mKeyMaster.checkMaster(); + } + + if (!locked) + { + LL_WARNS("CRASHREPORT") << "Unable to get master lock. Another crash reporter may be hung." << LL_ENDL; + return false; + } + mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND, "Controls behavior when viewer crashes " "(0 = ask before sending crash report, " -- cgit v1.2.3 From 928f8fb608b2e7a32eee8fa65e437bf22446f81e Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 20 Apr 2016 12:26:28 -0400 Subject: reset crash dump upload retries and timeouts --- indra/llcrashlogger/llcrashlogger.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 4bfc2d40dd..fb47cf30da 100644 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -56,6 +56,9 @@ BOOL gSent = false; int LLCrashLogger::ssl_mutex_count = 0; LLCoreInt::HttpMutex ** LLCrashLogger::ssl_mutex_list = NULL; +#define CRASH_UPLOAD_RETRIES 3 /* seconds */ +#define CRASH_UPLOAD_TIMEOUT 180 /* seconds */ + class LLCrashLoggerHandler : public LLHttpSDHandler { LOG_CLASS(LLCrashLoggerHandler); @@ -473,13 +476,13 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) msg = msg+mCrashHost.c_str(); updateApplication(msg.c_str()); - sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), 3, 5); + sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); } if(!sent) { updateApplication("Using default server..."); - sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), 3, 5); + sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); } mSentCrashLogs = sent; -- cgit v1.2.3