diff options
Diffstat (limited to 'indra/llcrashlogger/llcrashlogger.cpp')
-rw-r--r-- | indra/llcrashlogger/llcrashlogger.cpp | 1372 |
1 files changed, 686 insertions, 686 deletions
diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index e02894c271..46f7c40f06 100644 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -1,686 +1,686 @@ - /** -* @file llcrashlogger.cpp -* @brief Crash logger implementation -* -* $LicenseInfo:firstyear=2003&license=viewerlgpl$ -* Second Life Viewer Source Code -* Copyright (C) 2010, Linden Research, Inc. -* -* This library is free software; you can redistribute it and/or -* modify it under the terms of the GNU Lesser General Public -* License as published by the Free Software Foundation; -* version 2.1 of the License only. -* -* This library is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -* Lesser General Public License for more details. -* -* You should have received a copy of the GNU Lesser General Public -* License along with this library; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -* -* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA -* $/LicenseInfo$ -*/ - -#include <cstdio> -#include <cstdlib> -#include <sstream> -#include <map> - -#include "llcrashlogger.h" -#include "llcrashlock.h" -#include "linden_common.h" -#include "llstring.h" -#include "indra_constants.h" // CRASH_BEHAVIOR_... -#include "llerror.h" -#include "llerrorcontrol.h" -#include "lltimer.h" -#include "lldir.h" -#include "llfile.h" -#include "llsdserialize.h" -#include "llproxy.h" -#include "llcorehttputil.h" -#include "llhttpsdhandler.h" -#include "httpcommon.h" -#include "httpresponse.h" -#include "llcleanup.h" - -#include <curl/curl.h> -#include <openssl/crypto.h> - -bool gBreak = false; -bool gSent = false; - -int LLCrashLogger::ssl_mutex_count = 0; -LLCoreInt::HttpMutex ** LLCrashLogger::ssl_mutex_list = NULL; - -#define CRASH_UPLOAD_RETRIES 3 /* seconds */ -#define CRASH_UPLOAD_TIMEOUT 180 /* seconds */ - -class LLCrashLoggerHandler : public LLHttpSDHandler -{ - LOG_CLASS(LLCrashLoggerHandler); -public: - LLCrashLoggerHandler() {} - -protected: - virtual void onSuccess(LLCore::HttpResponse * response, const LLSD &content); - virtual void onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status); - -}; - -void LLCrashLoggerHandler::onSuccess(LLCore::HttpResponse * response, const LLSD &content) -{ - LL_DEBUGS("CRASHREPORT") << "Request to " << response->getRequestURL() << "succeeded" << LL_ENDL; - gBreak = true; - gSent = true; -} - -void LLCrashLoggerHandler::onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status) -{ - LL_WARNS("CRASHREPORT") << "Request to " << response->getRequestURL() - << " failed: " << status.toString() << LL_ENDL; - gBreak = true; -} - -LLCrashLogger::LLCrashLogger() : - mCrashBehavior(CRASH_BEHAVIOR_ALWAYS_SEND), - mCrashInPreviousExec(false), - mCrashSettings("CrashSettings"), - mSentCrashLogs(false), - mCrashHost("") -{ -} - -LLCrashLogger::~LLCrashLogger() -{ - -} - -// TRIM_SIZE must remain larger than LINE_SEARCH_SIZE. -const int TRIM_SIZE = 128000; -const int LINE_SEARCH_DIST = 500; -const std::string SKIP_TEXT = "\n ...Skipping... \n"; -void trimSLLog(std::string& sllog) -{ - if(sllog.length() > TRIM_SIZE * 2) - { - std::string::iterator head = sllog.begin() + TRIM_SIZE; - std::string::iterator tail = sllog.begin() + sllog.length() - TRIM_SIZE; - std::string::iterator new_head = std::find(head, head - LINE_SEARCH_DIST, '\n'); - if(new_head != head - LINE_SEARCH_DIST) - { - head = new_head; - } - - std::string::iterator new_tail = std::find(tail, tail + LINE_SEARCH_DIST, '\n'); - if(new_tail != tail + LINE_SEARCH_DIST) - { - tail = new_tail; - } - - sllog.erase(head, tail); - sllog.insert(head, SKIP_TEXT.begin(), SKIP_TEXT.end()); - } -} - -std::string getStartupStateFromLog(std::string& sllog) -{ - std::string startup_state = "STATE_FIRST"; - std::string startup_token = "Startup state changing from "; - - int index = sllog.rfind(startup_token); - if (index < 0 || index + startup_token.length() > sllog.length()) { - return startup_state; - } - - // find new line - char cur_char = sllog[index + startup_token.length()]; - std::string::size_type newline_loc = index + startup_token.length(); - while(cur_char != '\n' && newline_loc < sllog.length()) - { - newline_loc++; - cur_char = sllog[newline_loc]; - } - - // get substring and find location of " to " - std::string state_line = sllog.substr(index, newline_loc - index); - std::string::size_type state_index = state_line.find(" to "); - startup_state = state_line.substr(state_index + 4, state_line.length() - state_index - 4); - - return startup_state; -} - -bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename ) -{ - std::string db_file_name = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,filename); - llifstream log_file(db_file_name.c_str()); - - // Look for it in the given file - if (log_file.is_open()) - { - LLSDSerialize::fromXML(dest, log_file); - log_file.close(); - return true; - } - else - { - LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL; - } - return false; -} - -void LLCrashLogger::mergeLogs( LLSD src_sd ) -{ - LLSD::map_iterator iter = src_sd.beginMap(); - LLSD::map_iterator end = src_sd.endMap(); - for( ; iter != end; ++iter) - { - mDebugLog[iter->first] = iter->second; - } -} - -bool LLCrashLogger::readMinidump(std::string minidump_path) -{ - size_t length=0; - - llifstream minidump_stream(minidump_path.c_str(), std::ios_base::in | std::ios_base::binary); - if (minidump_stream.is_open()) - { - minidump_stream.seekg(0, std::ios::end); - length = (size_t)minidump_stream.tellg(); - LL_WARNS("CRASHREPORT") << "minidump length "<< length <<LL_ENDL; - minidump_stream.seekg(0, std::ios::beg); - - LLSD::Binary data; - data.resize(length); - - minidump_stream.read(reinterpret_cast<char *>(&(data[0])),length); - minidump_stream.close(); - - mCrashInfo["Minidump"] = data; - } - else - { - LL_WARNS("CRASHREPORT") << "failed to open minidump "<<minidump_path<<LL_ENDL; - } - - return length > 0; -} - -void LLCrashLogger::gatherFiles() -{ - updateApplication("Gathering logs..."); - - LLSD static_sd; - LLSD dynamic_sd; - //if we ever want to change the endpoint we send crashes to - //we can construct a file download ( a la feature table filename for example) - //containing the new endpoint - LLSD endpoint; - std::string grid; - std::string fqdn; - - bool has_logs = readFromXML( static_sd, "static_debug_info.log" ); - has_logs |= readFromXML( dynamic_sd, "dynamic_debug_info.log" ); - - - if ( has_logs ) - { - mDebugLog = static_sd; - mergeLogs(dynamic_sd); - mCrashInPreviousExec = mDebugLog["CrashNotHandled"].asBoolean(); - - mFileMap["SecondLifeLog"] = mDebugLog["SLLog"].asString(); - mFileMap["SettingsXml"] = mDebugLog["SettingsFilename"].asString(); - mFileMap["CrashHostUrl"] = loadCrashURLSetting(); - if(mDebugLog.has("CAFilename")) - { - LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, - LLCore::HttpRequest::GLOBAL_POLICY_ID, mDebugLog["CAFilename"].asString(), NULL); - } - else - { - LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, - LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL); - } - - LL_INFOS("CRASHREPORT") << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL; - LL_INFOS("CRASHREPORT") << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL; - } - else - { - // Figure out the filename of the second life log - LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, - LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL); - - mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"SecondLife.log"); - mFileMap["SettingsXml"] = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS,"settings.xml"); - } - - if (!gDirUtilp->fileExists(mFileMap["SecondLifeLog"]) ) //We would prefer to get this from the per-run but here's our fallback. - { - mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,"SecondLife.old"); - } - - gatherPlatformSpecificFiles(); - - - if ( has_logs && (mFileMap["CrashHostUrl"] != "") ) - { - mCrashHost = mFileMap["CrashHostUrl"]; - } - - //default to agni, per product - mAltCrashHost = "http://viewercrashreport.agni.lindenlab.com/cgi-bin/viewercrashreceiver.py"; - - mCrashInfo["DebugLog"] = mDebugLog; - mFileMap["StatsLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"stats.log"); - - updateApplication("Encoding files..."); - - for(std::map<std::string, std::string>::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr) - { - std::string file = (*itr).second; - if (!file.empty()) - { - LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL; - llifstream f(file.c_str()); - if(f.is_open()) - { - std::stringstream s; - s << f.rdbuf(); - - std::string crash_info = s.str(); - if(itr->first == "SecondLifeLog") - { - if(!mCrashInfo["DebugLog"].has("StartupState")) - { - mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); - } - trimSLLog(crash_info); - } - - mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); - } - else - { - LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL; - } - } - else - { - LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL; - } - } - - std::string minidump_path; - // Add minidump as binary. - bool has_minidump = mDebugLog.has("MinidumpPath"); - - if (has_minidump) - { - minidump_path = mDebugLog["MinidumpPath"].asString(); - has_minidump = readMinidump(minidump_path); - } - else - { - LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL; - } - - if (!has_minidump) //Viewer was probably so hosed it couldn't write remaining data. Try brute force. - { - //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file. - typedef std::vector<std::string> vec; - std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,""); - LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL; - vec file_vec = gDirUtilp->getFilesInDir(pathname); - for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter) - { - if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) ) - { - std::string fullname = pathname + *iter; - llifstream fdat(fullname.c_str(), std::ifstream::binary); - if (fdat) - { - char buf[5]; - fdat.read(buf,4); - fdat.close(); - if (!strncmp(buf,"MDMP",4)) - { - minidump_path = *iter; - has_minidump = readMinidump(fullname); - mDebugLog["MinidumpPath"] = fullname; - } - else - { - LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL; - } - } - else - { - LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL; - } - } - else - { - LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL; - } - } - } - else - { - LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL; - } -} - -LLSD LLCrashLogger::constructPostData() -{ - return mCrashInfo; -} - -const char* const CRASH_SETTINGS_FILE = "settings_crash_behavior.xml"; - -std::string LLCrashLogger::loadCrashURLSetting() -{ - - // First check user_settings (in the user's home dir) - std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, CRASH_SETTINGS_FILE); - if (! mCrashSettings.loadFromFile(filename)) - { - // Next check app_settings (in the SL program dir) - std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, CRASH_SETTINGS_FILE); - mCrashSettings.loadFromFile(filename); - } - - if (! mCrashSettings.controlExists("CrashHostUrl")) - { - return ""; - } - else - { - return mCrashSettings.getString("CrashHostUrl"); - } -} - -bool LLCrashLogger::runCrashLogPost(std::string host, LLSD data, std::string msg, int retries, int timeout) -{ - LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); - LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); - - httpOpts->setTimeout(timeout); - httpOpts->setSSLVerifyPeer(false); - - for(int i = 0; i < retries; ++i) - { - updateApplication(llformat("%s, try %d...", msg.c_str(), i+1)); - - LL_INFOS("CRASHREPORT") << "POST crash data to " << host << LL_ENDL; - LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(httpRequest.get(), LLCore::HttpRequest::DEFAULT_POLICY_ID, - host, data, httpOpts, LLCore::HttpHeaders::ptr_t(), LLCore::HttpHandler::ptr_t(new LLCrashLoggerHandler)); - - if (handle == LLCORE_HTTP_HANDLE_INVALID) - { - LLCore::HttpStatus status = httpRequest->getStatus(); - LL_WARNS("CRASHREPORT") << "Request POST failed to " << host << " with status of [" << - status.getType() << "]\"" << status.toString() << "\"" << LL_ENDL; - return false; - } - - while(!gBreak) - { - ms_sleep(250); - updateApplication(); // No new message, just pump the IO - httpRequest->update(0L); - } - if(gSent) - { - return gSent; - } - - LL_WARNS("CRASHREPORT") << "Failed to send crash report to \"" << host << "\"" << LL_ENDL; - } - return gSent; -} - -bool LLCrashLogger::sendCrashLog(std::string dump_dir) -{ - - gDirUtilp->setDumpDir( dump_dir ); - - std::string dump_path = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, - "SecondLifeCrashReport"); - std::string report_file = dump_path + ".log"; - - LL_DEBUGS("CRASHREPORT") << "sending " << report_file << LL_ENDL; - - gatherFiles(); - - LLSD post_data; - post_data = constructPostData(); - - updateApplication("Sending reports..."); - - llofstream out_file(report_file.c_str()); - LLSDSerialize::toPrettyXML(post_data, out_file); - out_file.flush(); - out_file.close(); - - bool sent = false; - - if(mCrashHost != "") - { - LL_WARNS("CRASHREPORT") << "Sending crash data to server from CrashHostUrl '" << mCrashHost << "'" << LL_ENDL; - - std::string msg = "Using override crash server... "; - msg = msg+mCrashHost.c_str(); - updateApplication(msg.c_str()); - - sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); - } - - if(!sent) - { - updateApplication("Using default server..."); - sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); - } - - mSentCrashLogs = sent; - - return sent; -} - -bool LLCrashLogger::sendCrashLogs() -{ - LLSD locks = mKeyMaster.getProcessList(); - LLSD newlocks = LLSD::emptyArray(); - - LLSD opts = getOptionData(PRIORITY_COMMAND_LINE); - LLSD rec; - - if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") ) - { - rec["pid"]=opts["pid"]; - rec["dumpdir"]=opts["dumpdir"]; - rec["procname"]=opts["procname"]; - } - - if (locks.isArray()) - { - for (LLSD::array_iterator lock=locks.beginArray(); - lock !=locks.endArray(); - ++lock) - { - if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") ) - { - if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) ) - { - newlocks.append(*lock); - } - else - { - //TODO: This is a hack but I didn't want to include boost in another file or retest everything related to lldir - if (LLCrashLock::fileExists((*lock)["dumpdir"].asString())) - { - //the viewer cleans up the log directory on clean shutdown - //but is ignorant of the locking table. - if (!sendCrashLog((*lock)["dumpdir"].asString())) - { - newlocks.append(*lock); //Failed to send log so don't delete it. - } - else - { - mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString()); - } - } - } - } - else - { - LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL; - } - } - } - - if (rec) - { - newlocks.append(rec); - } - - mKeyMaster.putProcessList(newlocks); - return true; -} - -void LLCrashLogger::updateApplication(const std::string& message) -{ - if (!message.empty()) LL_INFOS("CRASHREPORT") << message << LL_ENDL; -} - -bool LLCrashLogger::init() -{ - LL_DEBUGS("CRASHREPORT") << LL_ENDL; - - LLCore::LLHttp::initialize(); - - // We assume that all the logs we're looking for reside on the current drive - gDirUtilp->initAppDirs("SecondLife"); - - LLError::initForApplication(gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, ""), gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "")); - - // Default to the product name "Second Life" (this is overridden by the -name argument) - mProductName = "Second Life"; - - // Rename current log file to ".old" - std::string old_log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log.old"); - std::string log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log"); - -#if LL_WINDOWS - LLAPRFile::remove(old_log_file); -#endif - - LLFile::rename(log_file.c_str(), old_log_file.c_str()); - - // Set the log file to crashreport.log - LLError::logToFile(log_file); //NOTE: Until this line, LL_INFOS LL_WARNS, etc are blown to the ether. - - LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL; - - // Handle locking - bool locked = mKeyMaster.requestMaster(); //Request master locking file. wait time is defaulted to 300S - - while (!locked && mKeyMaster.isWaiting()) - { - LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL; -#if LL_WINDOWS - Sleep(1000); -#else - ::sleep(1); -#endif - locked = mKeyMaster.checkMaster(); - } - - if (!locked) - { - LL_WARNS("CRASHREPORT") << "Unable to get master lock. Another crash reporter may be hung." << LL_ENDL; - return false; - } - - mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND, - "Controls behavior when viewer crashes " - "(0 = ask before sending crash report, " - "1 = always send crash report, " - "2 = never send crash report)"); - - init_curl(); - LLCore::HttpRequest::createService(); - LLCore::HttpRequest::startThread(); - - return true; -} - -// For cleanup code common to all platforms. -void LLCrashLogger::commonCleanup() -{ - term_curl(); - LLError::logToFile(""); //close crashreport.log - SUBSYSTEM_CLEANUP(LLProxy); -} - -void LLCrashLogger::init_curl() -{ - curl_global_init(CURL_GLOBAL_ALL); - - ssl_mutex_count = CRYPTO_num_locks(); - if (ssl_mutex_count > 0) - { - ssl_mutex_list = new LLCoreInt::HttpMutex *[ssl_mutex_count]; - - for (int i(0); i < ssl_mutex_count; ++i) - { - ssl_mutex_list[i] = new LLCoreInt::HttpMutex; - } - - CRYPTO_set_locking_callback(ssl_locking_callback); - CRYPTO_THREADID_set_callback(ssl_thread_id_callback); - } -} - - -void LLCrashLogger::term_curl() -{ - CRYPTO_set_locking_callback(NULL); - for (int i(0); i < ssl_mutex_count; ++i) - { - delete ssl_mutex_list[i]; - } - delete[] ssl_mutex_list; -} - - -void LLCrashLogger::ssl_thread_id_callback(CRYPTO_THREADID* pthreadid) -{ -#if LL_WINDOWS - CRYPTO_THREADID_set_pointer(pthreadid, GetCurrentThread()); -#else - CRYPTO_THREADID_set_pointer(pthreadid, reinterpret_cast<void*>(pthread_self())); -#endif -} - - -void LLCrashLogger::ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) -{ - if (type >= 0 && type < ssl_mutex_count) - { - if (mode & CRYPTO_LOCK) - { - ssl_mutex_list[type]->lock(); - } - else - { - ssl_mutex_list[type]->unlock(); - } - } -} - + /**
+* @file llcrashlogger.cpp
+* @brief Crash logger implementation
+*
+* $LicenseInfo:firstyear=2003&license=viewerlgpl$
+* Second Life Viewer Source Code
+* Copyright (C) 2010, Linden Research, Inc.
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation;
+* version 2.1 of the License only.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*
+* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
+* $/LicenseInfo$
+*/
+
+#include <cstdio>
+#include <cstdlib>
+#include <sstream>
+#include <map>
+
+#include "llcrashlogger.h"
+#include "llcrashlock.h"
+#include "linden_common.h"
+#include "llstring.h"
+#include "indra_constants.h" // CRASH_BEHAVIOR_...
+#include "llerror.h"
+#include "llerrorcontrol.h"
+#include "lltimer.h"
+#include "lldir.h"
+#include "llfile.h"
+#include "llsdserialize.h"
+#include "llproxy.h"
+#include "llcorehttputil.h"
+#include "llhttpsdhandler.h"
+#include "httpcommon.h"
+#include "httpresponse.h"
+#include "llcleanup.h"
+
+#include <curl/curl.h>
+#include <openssl/crypto.h>
+
+bool gBreak = false;
+bool gSent = false;
+
+int LLCrashLogger::ssl_mutex_count = 0;
+LLCoreInt::HttpMutex ** LLCrashLogger::ssl_mutex_list = NULL;
+
+#define CRASH_UPLOAD_RETRIES 3 /* seconds */
+#define CRASH_UPLOAD_TIMEOUT 180 /* seconds */
+
+class LLCrashLoggerHandler : public LLHttpSDHandler
+{
+ LOG_CLASS(LLCrashLoggerHandler);
+public:
+ LLCrashLoggerHandler() {}
+
+protected:
+ virtual void onSuccess(LLCore::HttpResponse * response, const LLSD &content);
+ virtual void onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status);
+
+};
+
+void LLCrashLoggerHandler::onSuccess(LLCore::HttpResponse * response, const LLSD &content)
+{
+ LL_DEBUGS("CRASHREPORT") << "Request to " << response->getRequestURL() << "succeeded" << LL_ENDL;
+ gBreak = true;
+ gSent = true;
+}
+
+void LLCrashLoggerHandler::onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status)
+{
+ LL_WARNS("CRASHREPORT") << "Request to " << response->getRequestURL()
+ << " failed: " << status.toString() << LL_ENDL;
+ gBreak = true;
+}
+
+LLCrashLogger::LLCrashLogger() :
+ mCrashBehavior(CRASH_BEHAVIOR_ALWAYS_SEND),
+ mCrashInPreviousExec(false),
+ mCrashSettings("CrashSettings"),
+ mSentCrashLogs(false),
+ mCrashHost("")
+{
+}
+
+LLCrashLogger::~LLCrashLogger()
+{
+
+}
+
+// TRIM_SIZE must remain larger than LINE_SEARCH_SIZE.
+const int TRIM_SIZE = 128000;
+const int LINE_SEARCH_DIST = 500;
+const std::string SKIP_TEXT = "\n ...Skipping... \n";
+void trimSLLog(std::string& sllog)
+{
+ if(sllog.length() > TRIM_SIZE * 2)
+ {
+ std::string::iterator head = sllog.begin() + TRIM_SIZE;
+ std::string::iterator tail = sllog.begin() + sllog.length() - TRIM_SIZE;
+ std::string::iterator new_head = std::find(head, head - LINE_SEARCH_DIST, '\n');
+ if(new_head != head - LINE_SEARCH_DIST)
+ {
+ head = new_head;
+ }
+
+ std::string::iterator new_tail = std::find(tail, tail + LINE_SEARCH_DIST, '\n');
+ if(new_tail != tail + LINE_SEARCH_DIST)
+ {
+ tail = new_tail;
+ }
+
+ sllog.erase(head, tail);
+ sllog.insert(head, SKIP_TEXT.begin(), SKIP_TEXT.end());
+ }
+}
+
+std::string getStartupStateFromLog(std::string& sllog)
+{
+ std::string startup_state = "STATE_FIRST";
+ std::string startup_token = "Startup state changing from ";
+
+ int index = sllog.rfind(startup_token);
+ if (index < 0 || index + startup_token.length() > sllog.length()) {
+ return startup_state;
+ }
+
+ // find new line
+ char cur_char = sllog[index + startup_token.length()];
+ std::string::size_type newline_loc = index + startup_token.length();
+ while(cur_char != '\n' && newline_loc < sllog.length())
+ {
+ newline_loc++;
+ cur_char = sllog[newline_loc];
+ }
+
+ // get substring and find location of " to "
+ std::string state_line = sllog.substr(index, newline_loc - index);
+ std::string::size_type state_index = state_line.find(" to ");
+ startup_state = state_line.substr(state_index + 4, state_line.length() - state_index - 4);
+
+ return startup_state;
+}
+
+bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename )
+{
+ std::string db_file_name = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,filename);
+ llifstream log_file(db_file_name.c_str());
+
+ // Look for it in the given file
+ if (log_file.is_open())
+ {
+ LLSDSerialize::fromXML(dest, log_file);
+ log_file.close();
+ return true;
+ }
+ else
+ {
+ LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL;
+ }
+ return false;
+}
+
+void LLCrashLogger::mergeLogs( LLSD src_sd )
+{
+ LLSD::map_iterator iter = src_sd.beginMap();
+ LLSD::map_iterator end = src_sd.endMap();
+ for( ; iter != end; ++iter)
+ {
+ mDebugLog[iter->first] = iter->second;
+ }
+}
+
+bool LLCrashLogger::readMinidump(std::string minidump_path)
+{
+ size_t length=0;
+
+ llifstream minidump_stream(minidump_path.c_str(), std::ios_base::in | std::ios_base::binary);
+ if (minidump_stream.is_open())
+ {
+ minidump_stream.seekg(0, std::ios::end);
+ length = (size_t)minidump_stream.tellg();
+ LL_WARNS("CRASHREPORT") << "minidump length "<< length <<LL_ENDL;
+ minidump_stream.seekg(0, std::ios::beg);
+
+ LLSD::Binary data;
+ data.resize(length);
+
+ minidump_stream.read(reinterpret_cast<char *>(&(data[0])),length);
+ minidump_stream.close();
+
+ mCrashInfo["Minidump"] = data;
+ }
+ else
+ {
+ LL_WARNS("CRASHREPORT") << "failed to open minidump "<<minidump_path<<LL_ENDL;
+ }
+
+ return length > 0;
+}
+
+void LLCrashLogger::gatherFiles()
+{
+ updateApplication("Gathering logs...");
+
+ LLSD static_sd;
+ LLSD dynamic_sd;
+ //if we ever want to change the endpoint we send crashes to
+ //we can construct a file download ( a la feature table filename for example)
+ //containing the new endpoint
+ LLSD endpoint;
+ std::string grid;
+ std::string fqdn;
+
+ bool has_logs = readFromXML( static_sd, "static_debug_info.log" );
+ has_logs |= readFromXML( dynamic_sd, "dynamic_debug_info.log" );
+
+
+ if ( has_logs )
+ {
+ mDebugLog = static_sd;
+ mergeLogs(dynamic_sd);
+ mCrashInPreviousExec = mDebugLog["CrashNotHandled"].asBoolean();
+
+ mFileMap["SecondLifeLog"] = mDebugLog["SLLog"].asString();
+ mFileMap["SettingsXml"] = mDebugLog["SettingsFilename"].asString();
+ mFileMap["CrashHostUrl"] = loadCrashURLSetting();
+ if(mDebugLog.has("CAFilename"))
+ {
+ LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE,
+ LLCore::HttpRequest::GLOBAL_POLICY_ID, mDebugLog["CAFilename"].asString(), NULL);
+ }
+ else
+ {
+ LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE,
+ LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL);
+ }
+
+ LL_INFOS("CRASHREPORT") << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL;
+ LL_INFOS("CRASHREPORT") << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL;
+ }
+ else
+ {
+ // Figure out the filename of the second life log
+ LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE,
+ LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL);
+
+ mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"SecondLife.log");
+ mFileMap["SettingsXml"] = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS,"settings.xml");
+ }
+
+ if (!gDirUtilp->fileExists(mFileMap["SecondLifeLog"]) ) //We would prefer to get this from the per-run but here's our fallback.
+ {
+ mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,"SecondLife.old");
+ }
+
+ gatherPlatformSpecificFiles();
+
+
+ if ( has_logs && (mFileMap["CrashHostUrl"] != "") )
+ {
+ mCrashHost = mFileMap["CrashHostUrl"];
+ }
+
+ //default to agni, per product
+ mAltCrashHost = "http://viewercrashreport.agni.lindenlab.com/cgi-bin/viewercrashreceiver.py";
+
+ mCrashInfo["DebugLog"] = mDebugLog;
+ mFileMap["StatsLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"stats.log");
+
+ updateApplication("Encoding files...");
+
+ for(std::map<std::string, std::string>::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr)
+ {
+ std::string file = (*itr).second;
+ if (!file.empty())
+ {
+ LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL;
+ llifstream f(file.c_str());
+ if(f.is_open())
+ {
+ std::stringstream s;
+ s << f.rdbuf();
+
+ std::string crash_info = s.str();
+ if(itr->first == "SecondLifeLog")
+ {
+ if(!mCrashInfo["DebugLog"].has("StartupState"))
+ {
+ mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info);
+ }
+ trimSLLog(crash_info);
+ }
+
+ mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info));
+ }
+ else
+ {
+ LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL;
+ }
+ }
+ else
+ {
+ LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL;
+ }
+ }
+
+ std::string minidump_path;
+ // Add minidump as binary.
+ bool has_minidump = mDebugLog.has("MinidumpPath");
+
+ if (has_minidump)
+ {
+ minidump_path = mDebugLog["MinidumpPath"].asString();
+ has_minidump = readMinidump(minidump_path);
+ }
+ else
+ {
+ LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL;
+ }
+
+ if (!has_minidump) //Viewer was probably so hosed it couldn't write remaining data. Try brute force.
+ {
+ //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file.
+ typedef std::vector<std::string> vec;
+ std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"");
+ LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL;
+ vec file_vec = gDirUtilp->getFilesInDir(pathname);
+ for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter)
+ {
+ if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) )
+ {
+ std::string fullname = pathname + *iter;
+ llifstream fdat(fullname.c_str(), std::ifstream::binary);
+ if (fdat)
+ {
+ char buf[5];
+ fdat.read(buf,4);
+ fdat.close();
+ if (!strncmp(buf,"MDMP",4))
+ {
+ minidump_path = *iter;
+ has_minidump = readMinidump(fullname);
+ mDebugLog["MinidumpPath"] = fullname;
+ }
+ else
+ {
+ LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL;
+ }
+ }
+ else
+ {
+ LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL;
+ }
+ }
+ else
+ {
+ LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL;
+ }
+ }
+ }
+ else
+ {
+ LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL;
+ }
+}
+
+LLSD LLCrashLogger::constructPostData()
+{
+ return mCrashInfo;
+}
+
+const char* const CRASH_SETTINGS_FILE = "settings_crash_behavior.xml";
+
+std::string LLCrashLogger::loadCrashURLSetting()
+{
+
+ // First check user_settings (in the user's home dir)
+ std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, CRASH_SETTINGS_FILE);
+ if (! mCrashSettings.loadFromFile(filename))
+ {
+ // Next check app_settings (in the SL program dir)
+ std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, CRASH_SETTINGS_FILE);
+ mCrashSettings.loadFromFile(filename);
+ }
+
+ if (! mCrashSettings.controlExists("CrashHostUrl"))
+ {
+ return "";
+ }
+ else
+ {
+ return mCrashSettings.getString("CrashHostUrl");
+ }
+}
+
+bool LLCrashLogger::runCrashLogPost(std::string host, LLSD data, std::string msg, int retries, int timeout)
+{
+ LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest);
+ LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions);
+
+ httpOpts->setTimeout(timeout);
+ httpOpts->setSSLVerifyPeer(false);
+
+ for(int i = 0; i < retries; ++i)
+ {
+ updateApplication(llformat("%s, try %d...", msg.c_str(), i+1));
+
+ LL_INFOS("CRASHREPORT") << "POST crash data to " << host << LL_ENDL;
+ LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(httpRequest.get(), LLCore::HttpRequest::DEFAULT_POLICY_ID,
+ host, data, httpOpts, LLCore::HttpHeaders::ptr_t(), LLCore::HttpHandler::ptr_t(new LLCrashLoggerHandler));
+
+ if (handle == LLCORE_HTTP_HANDLE_INVALID)
+ {
+ LLCore::HttpStatus status = httpRequest->getStatus();
+ LL_WARNS("CRASHREPORT") << "Request POST failed to " << host << " with status of [" <<
+ status.getType() << "]\"" << status.toString() << "\"" << LL_ENDL;
+ return false;
+ }
+
+ while(!gBreak)
+ {
+ ms_sleep(250);
+ updateApplication(); // No new message, just pump the IO
+ httpRequest->update(0L);
+ }
+ if(gSent)
+ {
+ return gSent;
+ }
+
+ LL_WARNS("CRASHREPORT") << "Failed to send crash report to \"" << host << "\"" << LL_ENDL;
+ }
+ return gSent;
+}
+
+bool LLCrashLogger::sendCrashLog(std::string dump_dir)
+{
+
+ gDirUtilp->setDumpDir( dump_dir );
+
+ std::string dump_path = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,
+ "SecondLifeCrashReport");
+ std::string report_file = dump_path + ".log";
+
+ LL_DEBUGS("CRASHREPORT") << "sending " << report_file << LL_ENDL;
+
+ gatherFiles();
+
+ LLSD post_data;
+ post_data = constructPostData();
+
+ updateApplication("Sending reports...");
+
+ llofstream out_file(report_file.c_str());
+ LLSDSerialize::toPrettyXML(post_data, out_file);
+ out_file.flush();
+ out_file.close();
+
+ bool sent = false;
+
+ if(mCrashHost != "")
+ {
+ LL_WARNS("CRASHREPORT") << "Sending crash data to server from CrashHostUrl '" << mCrashHost << "'" << LL_ENDL;
+
+ std::string msg = "Using override crash server... ";
+ msg = msg+mCrashHost.c_str();
+ updateApplication(msg.c_str());
+
+ sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT);
+ }
+
+ if(!sent)
+ {
+ updateApplication("Using default server...");
+ sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT);
+ }
+
+ mSentCrashLogs = sent;
+
+ return sent;
+}
+
+bool LLCrashLogger::sendCrashLogs()
+{
+ LLSD locks = mKeyMaster.getProcessList();
+ LLSD newlocks = LLSD::emptyArray();
+
+ LLSD opts = getOptionData(PRIORITY_COMMAND_LINE);
+ LLSD rec;
+
+ if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") )
+ {
+ rec["pid"]=opts["pid"];
+ rec["dumpdir"]=opts["dumpdir"];
+ rec["procname"]=opts["procname"];
+ }
+
+ if (locks.isArray())
+ {
+ for (LLSD::array_iterator lock=locks.beginArray();
+ lock !=locks.endArray();
+ ++lock)
+ {
+ if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") )
+ {
+ if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) )
+ {
+ newlocks.append(*lock);
+ }
+ else
+ {
+ //TODO: This is a hack but I didn't want to include boost in another file or retest everything related to lldir
+ if (LLCrashLock::fileExists((*lock)["dumpdir"].asString()))
+ {
+ //the viewer cleans up the log directory on clean shutdown
+ //but is ignorant of the locking table.
+ if (!sendCrashLog((*lock)["dumpdir"].asString()))
+ {
+ newlocks.append(*lock); //Failed to send log so don't delete it.
+ }
+ else
+ {
+ mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString());
+ }
+ }
+ }
+ }
+ else
+ {
+ LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL;
+ }
+ }
+ }
+
+ if (rec)
+ {
+ newlocks.append(rec);
+ }
+
+ mKeyMaster.putProcessList(newlocks);
+ return true;
+}
+
+void LLCrashLogger::updateApplication(const std::string& message)
+{
+ if (!message.empty()) LL_INFOS("CRASHREPORT") << message << LL_ENDL;
+}
+
+bool LLCrashLogger::init()
+{
+ LL_DEBUGS("CRASHREPORT") << LL_ENDL;
+
+ LLCore::LLHttp::initialize();
+
+ // We assume that all the logs we're looking for reside on the current drive
+ gDirUtilp->initAppDirs("SecondLife");
+
+ LLError::initForApplication(gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, ""), gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, ""));
+
+ // Default to the product name "Second Life" (this is overridden by the -name argument)
+ mProductName = "Second Life";
+
+ // Rename current log file to ".old"
+ std::string old_log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log.old");
+ std::string log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log");
+
+#if LL_WINDOWS
+ LLAPRFile::remove(old_log_file);
+#endif
+
+ LLFile::rename(log_file.c_str(), old_log_file.c_str());
+
+ // Set the log file to crashreport.log
+ LLError::logToFile(log_file); //NOTE: Until this line, LL_INFOS LL_WARNS, etc are blown to the ether.
+
+ LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL;
+
+ // Handle locking
+ bool locked = mKeyMaster.requestMaster(); //Request master locking file. wait time is defaulted to 300S
+
+ while (!locked && mKeyMaster.isWaiting())
+ {
+ LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL;
+#if LL_WINDOWS
+ Sleep(1000);
+#else
+ ::sleep(1);
+#endif
+ locked = mKeyMaster.checkMaster();
+ }
+
+ if (!locked)
+ {
+ LL_WARNS("CRASHREPORT") << "Unable to get master lock. Another crash reporter may be hung." << LL_ENDL;
+ return false;
+ }
+
+ mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND,
+ "Controls behavior when viewer crashes "
+ "(0 = ask before sending crash report, "
+ "1 = always send crash report, "
+ "2 = never send crash report)");
+
+ init_curl();
+ LLCore::HttpRequest::createService();
+ LLCore::HttpRequest::startThread();
+
+ return true;
+}
+
+// For cleanup code common to all platforms.
+void LLCrashLogger::commonCleanup()
+{
+ term_curl();
+ LLError::logToFile(""); //close crashreport.log
+ SUBSYSTEM_CLEANUP(LLProxy);
+}
+
+void LLCrashLogger::init_curl()
+{
+ curl_global_init(CURL_GLOBAL_ALL);
+
+ ssl_mutex_count = CRYPTO_num_locks();
+ if (ssl_mutex_count > 0)
+ {
+ ssl_mutex_list = new LLCoreInt::HttpMutex *[ssl_mutex_count];
+
+ for (int i(0); i < ssl_mutex_count; ++i)
+ {
+ ssl_mutex_list[i] = new LLCoreInt::HttpMutex;
+ }
+
+ CRYPTO_set_locking_callback(ssl_locking_callback);
+ CRYPTO_THREADID_set_callback(ssl_thread_id_callback);
+ }
+}
+
+
+void LLCrashLogger::term_curl()
+{
+ CRYPTO_set_locking_callback(NULL);
+ for (int i(0); i < ssl_mutex_count; ++i)
+ {
+ delete ssl_mutex_list[i];
+ }
+ delete[] ssl_mutex_list;
+}
+
+
+void LLCrashLogger::ssl_thread_id_callback(CRYPTO_THREADID* pthreadid)
+{
+#if LL_WINDOWS
+ CRYPTO_THREADID_set_pointer(pthreadid, GetCurrentThread());
+#else
+ CRYPTO_THREADID_set_pointer(pthreadid, reinterpret_cast<void*>(pthread_self()));
+#endif
+}
+
+
+void LLCrashLogger::ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */)
+{
+ if (type >= 0 && type < ssl_mutex_count)
+ {
+ if (mode & CRYPTO_LOCK)
+ {
+ ssl_mutex_list[type]->lock();
+ }
+ else
+ {
+ ssl_mutex_list[type]->unlock();
+ }
+ }
+}
+
|