From a6bc849992ea120e6d05ac4b6ddf61c9ee807588 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 13 Apr 2016 14:41:38 -0400 Subject: add logging around crash reporting, with minor code cleanups --- indra/llcrashlogger/llcrashlogger.cpp | 65 +++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 29 deletions(-) (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 474c837107..9fd6b66513 100755 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -69,12 +69,15 @@ protected: void LLCrashLoggerHandler::onSuccess(LLCore::HttpResponse * response, const LLSD &content) { + LL_DEBUGS("CRASHREPORT") << "Request to " << response->getRequestURL() << "succeeded" << LL_ENDL; gBreak = true; gSent = true; } void LLCrashLoggerHandler::onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status) { + LL_WARNS("CRASHREPORT") << "Request to " << response->getRequestURL() + << " failed: " << status.toString() << LL_ENDL; gBreak = true; } @@ -230,8 +233,8 @@ void LLCrashLogger::gatherFiles() LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL); } - LL_INFOS() << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL; - LL_INFOS() << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL; } else { @@ -267,25 +270,27 @@ void LLCrashLogger::gatherFiles() for(std::map::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr) { std::ifstream f((*itr).second.c_str()); - if(!f.is_open()) - { - LL_INFOS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL; - continue; - } - std::stringstream s; - s << f.rdbuf(); + if(f.is_open()) + { + std::stringstream s; + s << f.rdbuf(); - std::string crash_info = s.str(); - if(itr->first == "SecondLifeLog") - { - if(!mCrashInfo["DebugLog"].has("StartupState")) - { - mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); - } - trimSLLog(crash_info); - } + std::string crash_info = s.str(); + if(itr->first == "SecondLifeLog") + { + if(!mCrashInfo["DebugLog"].has("StartupState")) + { + mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + } + trimSLLog(crash_info); + } - mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + } + else + { + LL_WARNS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL; + } } std::string minidump_path; @@ -375,6 +380,7 @@ bool LLCrashLogger::runCrashLogPost(std::string host, LLSD data, std::string msg { updateApplication(llformat("%s, try %d...", msg.c_str(), i+1)); + LL_INFOS("CRASHREPORT") << "POST crash data to " << host << LL_ENDL; LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(httpRequest.get(), LLCore::HttpRequest::DEFAULT_POLICY_ID, 0, host, data, httpOpts, LLCore::HttpHeaders::ptr_t(), LLCore::HttpHandler::ptr_t(new LLCrashLoggerHandler)); @@ -411,6 +417,8 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) "SecondLifeCrashReport"); std::string report_file = dump_path + ".log"; + LL_DEBUGS("CRASHREPORT") << "sending " << report_file << LL_ENDL; + gatherFiles(); LLSD post_data; @@ -425,11 +433,11 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) bool sent = false; - //*TODO: Translate - updateApplication("DEBUG: crash host in send logs "+mCrashHost); - if(mCrashHost != "") - { - std::string msg = "Using derived crash server... "; + if(mCrashHost != "") + { + LL_WARNS("CRASHREPORT") << "Sending crash data to server from CrashHostUrl '" << mCrashHost << "'" << LL_ENDL; + + std::string msg = "Using override crash server... "; msg = msg+mCrashHost.c_str(); updateApplication(msg.c_str()); @@ -469,11 +477,13 @@ bool LLCrashLogger::sendCrashLogs() void LLCrashLogger::updateApplication(const std::string& message) { - if (!message.empty()) LL_INFOS() << message << LL_ENDL; + if (!message.empty()) LL_INFOS("CRASHREPORT") << message << LL_ENDL; } bool LLCrashLogger::init() { + LL_DEBUGS("CRASHREPORT") << LL_ENDL; + LLCore::LLHttp::initialize(); // We assume that all the logs we're looking for reside on the current drive @@ -497,7 +507,7 @@ bool LLCrashLogger::init() // Set the log file to crashreport.log LLError::logToFile(log_file); //NOTE: Until this line, LL_INFOS LL_WARNS, etc are blown to the ether. - LL_INFOS() << "Crash reporter file rotation complete." << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL; mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND, "Controls behavior when viewer crashes " @@ -505,9 +515,6 @@ bool LLCrashLogger::init() "1 = always send crash report, " "2 = never send crash report)"); - // LL_INFOS() << "Loading crash behavior setting" << LL_ENDL; - // mCrashBehavior = loadCrashBehaviorSetting(); - init_curl(); LLCore::HttpRequest::createService(); LLCore::HttpRequest::startThread(); -- cgit v1.2.3 From ddd476c90c99c75a4add6c00243a90e84e025402 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 20 Apr 2016 11:52:00 -0400 Subject: MAINT-6322 fix merge error that prevented crash dumps from being located for upload (and add better logging) --- indra/llcrashlogger/llcrashlogger.cpp | 150 +++++++++++++++++++++++++++------- 1 file changed, 121 insertions(+), 29 deletions(-) (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 9fd6b66513..7e52eb231a 100755 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -161,6 +161,10 @@ bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename ) log_file.close(); return true; } + else + { + LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL; + } return false; } @@ -193,6 +197,11 @@ bool LLCrashLogger::readMinidump(std::string minidump_path) mCrashInfo["Minidump"] = data; } + else + { + LL_WARNS("CRASHREPORT") << "failed to open minidump "<0?true:false); } @@ -269,27 +278,36 @@ void LLCrashLogger::gatherFiles() for(std::map::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr) { - std::ifstream f((*itr).second.c_str()); - if(f.is_open()) + std::string file = (*itr).second; + if (!file.empty()) { - std::stringstream s; - s << f.rdbuf(); - - std::string crash_info = s.str(); - if(itr->first == "SecondLifeLog") + LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL; + std::ifstream f(file.c_str()); + if(f.is_open()) { - if(!mCrashInfo["DebugLog"].has("StartupState")) + std::stringstream s; + s << f.rdbuf(); + + std::string crash_info = s.str(); + if(itr->first == "SecondLifeLog") { - mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + if(!mCrashInfo["DebugLog"].has("StartupState")) + { + mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + } + trimSLLog(crash_info); } - trimSLLog(crash_info); - } - mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + } + else + { + LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL; + } } else { - LL_WARNS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL; + LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL; } } @@ -300,20 +318,21 @@ void LLCrashLogger::gatherFiles() if (has_minidump) { minidump_path = mDebugLog["MinidumpPath"].asString(); - } - - if (has_minidump) - { has_minidump = readMinidump(minidump_path); } + else + { + LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL; + } if (!has_minidump) //Viewer was probably so hosed it couldn't write remaining data. Try brute force. { - //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file. + //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file. typedef std::vector vec; std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,""); + LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL; vec file_vec = gDirUtilp->getFilesInDir(pathname); - for(vec::const_iterator iter=file_vec.begin(); iter!=file_vec.end(); ++iter) + for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter) { if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) ) { @@ -329,15 +348,27 @@ void LLCrashLogger::gatherFiles() minidump_path = *iter; has_minidump = readMinidump(fullname); mDebugLog["MinidumpPath"] = fullname; - if (has_minidump) - { - break; - } + } + else + { + LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL; } } + else + { + LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL; + } } + else + { + LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL; + } } } + else + { + LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL; + } } LLSD LLCrashLogger::constructPostData() @@ -457,22 +488,63 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) bool LLCrashLogger::sendCrashLogs() { - + LLSD locks = mKeyMaster.getProcessList(); + LLSD newlocks = LLSD::emptyArray(); + LLSD opts = getOptionData(PRIORITY_COMMAND_LINE); LLSD rec; - if ( opts.has("dumpdir") ) + if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") ) { rec["pid"]=opts["pid"]; rec["dumpdir"]=opts["dumpdir"]; rec["procname"]=opts["procname"]; } - else + + if (locks.isArray()) { - return false; - } + for (LLSD::array_iterator lock=locks.beginArray(); + lock !=locks.endArray(); + ++lock) + { + if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") ) + { + if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) ) + { + newlocks.append(*lock); + } + else + { + //TODO: This is a hack but I didn't want to include boost in another file or retest everything related to lldir + if (LLCrashLock::fileExists((*lock)["dumpdir"].asString())) + { + //the viewer cleans up the log directory on clean shutdown + //but is ignorant of the locking table. + if (!sendCrashLog((*lock)["dumpdir"].asString())) + { + newlocks.append(*lock); //Failed to send log so don't delete it. + } + else + { + mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString()); + } + } + } + } + else + { + LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL; + } + } + } - return sendCrashLog(rec["dumpdir"].asString()); + if (rec) + { + newlocks.append(rec); + } + + mKeyMaster.putProcessList(newlocks); + return true; } void LLCrashLogger::updateApplication(const std::string& message) @@ -509,6 +581,26 @@ bool LLCrashLogger::init() LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL; + // Handle locking + bool locked = mKeyMaster.requestMaster(); //Request master locking file. wait time is defaulted to 300S + + while (!locked && mKeyMaster.isWaiting()) + { + LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL; +#if LL_WINDOWS + Sleep(1000); +#else + sleep(1); +#endif + locked = mKeyMaster.checkMaster(); + } + + if (!locked) + { + LL_WARNS("CRASHREPORT") << "Unable to get master lock. Another crash reporter may be hung." << LL_ENDL; + return false; + } + mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND, "Controls behavior when viewer crashes " "(0 = ask before sending crash report, " -- cgit v1.2.3 From 6de1d311f906058be07fc7f5a0431a23d53f26b1 Mon Sep 17 00:00:00 2001 From: Oz Linden Date: Wed, 20 Apr 2016 12:26:28 -0400 Subject: reset crash dump upload retries and timeouts --- indra/llcrashlogger/llcrashlogger.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'indra/llcrashlogger') diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 7e52eb231a..69a806bbea 100755 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -55,6 +55,9 @@ BOOL gSent = false; int LLCrashLogger::ssl_mutex_count = 0; LLCoreInt::HttpMutex ** LLCrashLogger::ssl_mutex_list = NULL; +#define CRASH_UPLOAD_RETRIES 3 /* seconds */ +#define CRASH_UPLOAD_TIMEOUT 180 /* seconds */ + class LLCrashLoggerHandler : public LLHttpSDHandler { LOG_CLASS(LLCrashLoggerHandler); @@ -472,13 +475,13 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir) msg = msg+mCrashHost.c_str(); updateApplication(msg.c_str()); - sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), 3, 5); + sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); } if(!sent) { updateApplication("Using default server..."); - sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), 3, 5); + sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); } mSentCrashLogs = sent; -- cgit v1.2.3