diff options
Diffstat (limited to 'indra/llcrashlogger/llcrashlogger.cpp')
-rw-r--r-- | indra/llcrashlogger/llcrashlogger.cpp | 1372 |
1 files changed, 686 insertions, 686 deletions
diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp index 46f7c40f06..e94aff2a37 100644 --- a/indra/llcrashlogger/llcrashlogger.cpp +++ b/indra/llcrashlogger/llcrashlogger.cpp @@ -1,686 +1,686 @@ - /**
-* @file llcrashlogger.cpp
-* @brief Crash logger implementation
-*
-* $LicenseInfo:firstyear=2003&license=viewerlgpl$
-* Second Life Viewer Source Code
-* Copyright (C) 2010, Linden Research, Inc.
-*
-* This library is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License as published by the Free Software Foundation;
-* version 2.1 of the License only.
-*
-* This library is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-* Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public
-* License along with this library; if not, write to the Free Software
-* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-*
-* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
-* $/LicenseInfo$
-*/
-
-#include <cstdio>
-#include <cstdlib>
-#include <sstream>
-#include <map>
-
-#include "llcrashlogger.h"
-#include "llcrashlock.h"
-#include "linden_common.h"
-#include "llstring.h"
-#include "indra_constants.h" // CRASH_BEHAVIOR_...
-#include "llerror.h"
-#include "llerrorcontrol.h"
-#include "lltimer.h"
-#include "lldir.h"
-#include "llfile.h"
-#include "llsdserialize.h"
-#include "llproxy.h"
-#include "llcorehttputil.h"
-#include "llhttpsdhandler.h"
-#include "httpcommon.h"
-#include "httpresponse.h"
-#include "llcleanup.h"
-
-#include <curl/curl.h>
-#include <openssl/crypto.h>
-
-bool gBreak = false;
-bool gSent = false;
-
-int LLCrashLogger::ssl_mutex_count = 0;
-LLCoreInt::HttpMutex ** LLCrashLogger::ssl_mutex_list = NULL;
-
-#define CRASH_UPLOAD_RETRIES 3 /* seconds */
-#define CRASH_UPLOAD_TIMEOUT 180 /* seconds */
-
-class LLCrashLoggerHandler : public LLHttpSDHandler
-{
- LOG_CLASS(LLCrashLoggerHandler);
-public:
- LLCrashLoggerHandler() {}
-
-protected:
- virtual void onSuccess(LLCore::HttpResponse * response, const LLSD &content);
- virtual void onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status);
-
-};
-
-void LLCrashLoggerHandler::onSuccess(LLCore::HttpResponse * response, const LLSD &content)
-{
- LL_DEBUGS("CRASHREPORT") << "Request to " << response->getRequestURL() << "succeeded" << LL_ENDL;
- gBreak = true;
- gSent = true;
-}
-
-void LLCrashLoggerHandler::onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status)
-{
- LL_WARNS("CRASHREPORT") << "Request to " << response->getRequestURL()
- << " failed: " << status.toString() << LL_ENDL;
- gBreak = true;
-}
-
-LLCrashLogger::LLCrashLogger() :
- mCrashBehavior(CRASH_BEHAVIOR_ALWAYS_SEND),
- mCrashInPreviousExec(false),
- mCrashSettings("CrashSettings"),
- mSentCrashLogs(false),
- mCrashHost("")
-{
-}
-
-LLCrashLogger::~LLCrashLogger()
-{
-
-}
-
-// TRIM_SIZE must remain larger than LINE_SEARCH_SIZE.
-const int TRIM_SIZE = 128000;
-const int LINE_SEARCH_DIST = 500;
-const std::string SKIP_TEXT = "\n ...Skipping... \n";
-void trimSLLog(std::string& sllog)
-{
- if(sllog.length() > TRIM_SIZE * 2)
- {
- std::string::iterator head = sllog.begin() + TRIM_SIZE;
- std::string::iterator tail = sllog.begin() + sllog.length() - TRIM_SIZE;
- std::string::iterator new_head = std::find(head, head - LINE_SEARCH_DIST, '\n');
- if(new_head != head - LINE_SEARCH_DIST)
- {
- head = new_head;
- }
-
- std::string::iterator new_tail = std::find(tail, tail + LINE_SEARCH_DIST, '\n');
- if(new_tail != tail + LINE_SEARCH_DIST)
- {
- tail = new_tail;
- }
-
- sllog.erase(head, tail);
- sllog.insert(head, SKIP_TEXT.begin(), SKIP_TEXT.end());
- }
-}
-
-std::string getStartupStateFromLog(std::string& sllog)
-{
- std::string startup_state = "STATE_FIRST";
- std::string startup_token = "Startup state changing from ";
-
- int index = sllog.rfind(startup_token);
- if (index < 0 || index + startup_token.length() > sllog.length()) {
- return startup_state;
- }
-
- // find new line
- char cur_char = sllog[index + startup_token.length()];
- std::string::size_type newline_loc = index + startup_token.length();
- while(cur_char != '\n' && newline_loc < sllog.length())
- {
- newline_loc++;
- cur_char = sllog[newline_loc];
- }
-
- // get substring and find location of " to "
- std::string state_line = sllog.substr(index, newline_loc - index);
- std::string::size_type state_index = state_line.find(" to ");
- startup_state = state_line.substr(state_index + 4, state_line.length() - state_index - 4);
-
- return startup_state;
-}
-
-bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename )
-{
- std::string db_file_name = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,filename);
- llifstream log_file(db_file_name.c_str());
-
- // Look for it in the given file
- if (log_file.is_open())
- {
- LLSDSerialize::fromXML(dest, log_file);
- log_file.close();
- return true;
- }
- else
- {
- LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL;
- }
- return false;
-}
-
-void LLCrashLogger::mergeLogs( LLSD src_sd )
-{
- LLSD::map_iterator iter = src_sd.beginMap();
- LLSD::map_iterator end = src_sd.endMap();
- for( ; iter != end; ++iter)
- {
- mDebugLog[iter->first] = iter->second;
- }
-}
-
-bool LLCrashLogger::readMinidump(std::string minidump_path)
-{
- size_t length=0;
-
- llifstream minidump_stream(minidump_path.c_str(), std::ios_base::in | std::ios_base::binary);
- if (minidump_stream.is_open())
- {
- minidump_stream.seekg(0, std::ios::end);
- length = (size_t)minidump_stream.tellg();
- LL_WARNS("CRASHREPORT") << "minidump length "<< length <<LL_ENDL;
- minidump_stream.seekg(0, std::ios::beg);
-
- LLSD::Binary data;
- data.resize(length);
-
- minidump_stream.read(reinterpret_cast<char *>(&(data[0])),length);
- minidump_stream.close();
-
- mCrashInfo["Minidump"] = data;
- }
- else
- {
- LL_WARNS("CRASHREPORT") << "failed to open minidump "<<minidump_path<<LL_ENDL;
- }
-
- return length > 0;
-}
-
-void LLCrashLogger::gatherFiles()
-{
- updateApplication("Gathering logs...");
-
- LLSD static_sd;
- LLSD dynamic_sd;
- //if we ever want to change the endpoint we send crashes to
- //we can construct a file download ( a la feature table filename for example)
- //containing the new endpoint
- LLSD endpoint;
- std::string grid;
- std::string fqdn;
-
- bool has_logs = readFromXML( static_sd, "static_debug_info.log" );
- has_logs |= readFromXML( dynamic_sd, "dynamic_debug_info.log" );
-
-
- if ( has_logs )
- {
- mDebugLog = static_sd;
- mergeLogs(dynamic_sd);
- mCrashInPreviousExec = mDebugLog["CrashNotHandled"].asBoolean();
-
- mFileMap["SecondLifeLog"] = mDebugLog["SLLog"].asString();
- mFileMap["SettingsXml"] = mDebugLog["SettingsFilename"].asString();
- mFileMap["CrashHostUrl"] = loadCrashURLSetting();
- if(mDebugLog.has("CAFilename"))
- {
- LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE,
- LLCore::HttpRequest::GLOBAL_POLICY_ID, mDebugLog["CAFilename"].asString(), NULL);
- }
- else
- {
- LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE,
- LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL);
- }
-
- LL_INFOS("CRASHREPORT") << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL;
- LL_INFOS("CRASHREPORT") << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL;
- }
- else
- {
- // Figure out the filename of the second life log
- LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE,
- LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL);
-
- mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"SecondLife.log");
- mFileMap["SettingsXml"] = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS,"settings.xml");
- }
-
- if (!gDirUtilp->fileExists(mFileMap["SecondLifeLog"]) ) //We would prefer to get this from the per-run but here's our fallback.
- {
- mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,"SecondLife.old");
- }
-
- gatherPlatformSpecificFiles();
-
-
- if ( has_logs && (mFileMap["CrashHostUrl"] != "") )
- {
- mCrashHost = mFileMap["CrashHostUrl"];
- }
-
- //default to agni, per product
- mAltCrashHost = "http://viewercrashreport.agni.lindenlab.com/cgi-bin/viewercrashreceiver.py";
-
- mCrashInfo["DebugLog"] = mDebugLog;
- mFileMap["StatsLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"stats.log");
-
- updateApplication("Encoding files...");
-
- for(std::map<std::string, std::string>::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr)
- {
- std::string file = (*itr).second;
- if (!file.empty())
- {
- LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL;
- llifstream f(file.c_str());
- if(f.is_open())
- {
- std::stringstream s;
- s << f.rdbuf();
-
- std::string crash_info = s.str();
- if(itr->first == "SecondLifeLog")
- {
- if(!mCrashInfo["DebugLog"].has("StartupState"))
- {
- mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info);
- }
- trimSLLog(crash_info);
- }
-
- mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info));
- }
- else
- {
- LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL;
- }
- }
- else
- {
- LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL;
- }
- }
-
- std::string minidump_path;
- // Add minidump as binary.
- bool has_minidump = mDebugLog.has("MinidumpPath");
-
- if (has_minidump)
- {
- minidump_path = mDebugLog["MinidumpPath"].asString();
- has_minidump = readMinidump(minidump_path);
- }
- else
- {
- LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL;
- }
-
- if (!has_minidump) //Viewer was probably so hosed it couldn't write remaining data. Try brute force.
- {
- //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file.
- typedef std::vector<std::string> vec;
- std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"");
- LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL;
- vec file_vec = gDirUtilp->getFilesInDir(pathname);
- for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter)
- {
- if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) )
- {
- std::string fullname = pathname + *iter;
- llifstream fdat(fullname.c_str(), std::ifstream::binary);
- if (fdat)
- {
- char buf[5];
- fdat.read(buf,4);
- fdat.close();
- if (!strncmp(buf,"MDMP",4))
- {
- minidump_path = *iter;
- has_minidump = readMinidump(fullname);
- mDebugLog["MinidumpPath"] = fullname;
- }
- else
- {
- LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL;
- }
- }
- else
- {
- LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL;
- }
- }
- else
- {
- LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL;
- }
- }
- }
- else
- {
- LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL;
- }
-}
-
-LLSD LLCrashLogger::constructPostData()
-{
- return mCrashInfo;
-}
-
-const char* const CRASH_SETTINGS_FILE = "settings_crash_behavior.xml";
-
-std::string LLCrashLogger::loadCrashURLSetting()
-{
-
- // First check user_settings (in the user's home dir)
- std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, CRASH_SETTINGS_FILE);
- if (! mCrashSettings.loadFromFile(filename))
- {
- // Next check app_settings (in the SL program dir)
- std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, CRASH_SETTINGS_FILE);
- mCrashSettings.loadFromFile(filename);
- }
-
- if (! mCrashSettings.controlExists("CrashHostUrl"))
- {
- return "";
- }
- else
- {
- return mCrashSettings.getString("CrashHostUrl");
- }
-}
-
-bool LLCrashLogger::runCrashLogPost(std::string host, LLSD data, std::string msg, int retries, int timeout)
-{
- LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest);
- LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions);
-
- httpOpts->setTimeout(timeout);
- httpOpts->setSSLVerifyPeer(false);
-
- for(int i = 0; i < retries; ++i)
- {
- updateApplication(llformat("%s, try %d...", msg.c_str(), i+1));
-
- LL_INFOS("CRASHREPORT") << "POST crash data to " << host << LL_ENDL;
- LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(httpRequest.get(), LLCore::HttpRequest::DEFAULT_POLICY_ID,
- host, data, httpOpts, LLCore::HttpHeaders::ptr_t(), LLCore::HttpHandler::ptr_t(new LLCrashLoggerHandler));
-
- if (handle == LLCORE_HTTP_HANDLE_INVALID)
- {
- LLCore::HttpStatus status = httpRequest->getStatus();
- LL_WARNS("CRASHREPORT") << "Request POST failed to " << host << " with status of [" <<
- status.getType() << "]\"" << status.toString() << "\"" << LL_ENDL;
- return false;
- }
-
- while(!gBreak)
- {
- ms_sleep(250);
- updateApplication(); // No new message, just pump the IO
- httpRequest->update(0L);
- }
- if(gSent)
- {
- return gSent;
- }
-
- LL_WARNS("CRASHREPORT") << "Failed to send crash report to \"" << host << "\"" << LL_ENDL;
- }
- return gSent;
-}
-
-bool LLCrashLogger::sendCrashLog(std::string dump_dir)
-{
-
- gDirUtilp->setDumpDir( dump_dir );
-
- std::string dump_path = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,
- "SecondLifeCrashReport");
- std::string report_file = dump_path + ".log";
-
- LL_DEBUGS("CRASHREPORT") << "sending " << report_file << LL_ENDL;
-
- gatherFiles();
-
- LLSD post_data;
- post_data = constructPostData();
-
- updateApplication("Sending reports...");
-
- llofstream out_file(report_file.c_str());
- LLSDSerialize::toPrettyXML(post_data, out_file);
- out_file.flush();
- out_file.close();
-
- bool sent = false;
-
- if(mCrashHost != "")
- {
- LL_WARNS("CRASHREPORT") << "Sending crash data to server from CrashHostUrl '" << mCrashHost << "'" << LL_ENDL;
-
- std::string msg = "Using override crash server... ";
- msg = msg+mCrashHost.c_str();
- updateApplication(msg.c_str());
-
- sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT);
- }
-
- if(!sent)
- {
- updateApplication("Using default server...");
- sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT);
- }
-
- mSentCrashLogs = sent;
-
- return sent;
-}
-
-bool LLCrashLogger::sendCrashLogs()
-{
- LLSD locks = mKeyMaster.getProcessList();
- LLSD newlocks = LLSD::emptyArray();
-
- LLSD opts = getOptionData(PRIORITY_COMMAND_LINE);
- LLSD rec;
-
- if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") )
- {
- rec["pid"]=opts["pid"];
- rec["dumpdir"]=opts["dumpdir"];
- rec["procname"]=opts["procname"];
- }
-
- if (locks.isArray())
- {
- for (LLSD::array_iterator lock=locks.beginArray();
- lock !=locks.endArray();
- ++lock)
- {
- if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") )
- {
- if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) )
- {
- newlocks.append(*lock);
- }
- else
- {
- //TODO: This is a hack but I didn't want to include boost in another file or retest everything related to lldir
- if (LLCrashLock::fileExists((*lock)["dumpdir"].asString()))
- {
- //the viewer cleans up the log directory on clean shutdown
- //but is ignorant of the locking table.
- if (!sendCrashLog((*lock)["dumpdir"].asString()))
- {
- newlocks.append(*lock); //Failed to send log so don't delete it.
- }
- else
- {
- mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString());
- }
- }
- }
- }
- else
- {
- LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL;
- }
- }
- }
-
- if (rec)
- {
- newlocks.append(rec);
- }
-
- mKeyMaster.putProcessList(newlocks);
- return true;
-}
-
-void LLCrashLogger::updateApplication(const std::string& message)
-{
- if (!message.empty()) LL_INFOS("CRASHREPORT") << message << LL_ENDL;
-}
-
-bool LLCrashLogger::init()
-{
- LL_DEBUGS("CRASHREPORT") << LL_ENDL;
-
- LLCore::LLHttp::initialize();
-
- // We assume that all the logs we're looking for reside on the current drive
- gDirUtilp->initAppDirs("SecondLife");
-
- LLError::initForApplication(gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, ""), gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, ""));
-
- // Default to the product name "Second Life" (this is overridden by the -name argument)
- mProductName = "Second Life";
-
- // Rename current log file to ".old"
- std::string old_log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log.old");
- std::string log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log");
-
-#if LL_WINDOWS
- LLAPRFile::remove(old_log_file);
-#endif
-
- LLFile::rename(log_file.c_str(), old_log_file.c_str());
-
- // Set the log file to crashreport.log
- LLError::logToFile(log_file); //NOTE: Until this line, LL_INFOS LL_WARNS, etc are blown to the ether.
-
- LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL;
-
- // Handle locking
- bool locked = mKeyMaster.requestMaster(); //Request master locking file. wait time is defaulted to 300S
-
- while (!locked && mKeyMaster.isWaiting())
- {
- LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL;
-#if LL_WINDOWS
- Sleep(1000);
-#else
- ::sleep(1);
-#endif
- locked = mKeyMaster.checkMaster();
- }
-
- if (!locked)
- {
- LL_WARNS("CRASHREPORT") << "Unable to get master lock. Another crash reporter may be hung." << LL_ENDL;
- return false;
- }
-
- mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND,
- "Controls behavior when viewer crashes "
- "(0 = ask before sending crash report, "
- "1 = always send crash report, "
- "2 = never send crash report)");
-
- init_curl();
- LLCore::HttpRequest::createService();
- LLCore::HttpRequest::startThread();
-
- return true;
-}
-
-// For cleanup code common to all platforms.
-void LLCrashLogger::commonCleanup()
-{
- term_curl();
- LLError::logToFile(""); //close crashreport.log
- SUBSYSTEM_CLEANUP(LLProxy);
-}
-
-void LLCrashLogger::init_curl()
-{
- curl_global_init(CURL_GLOBAL_ALL);
-
- ssl_mutex_count = CRYPTO_num_locks();
- if (ssl_mutex_count > 0)
- {
- ssl_mutex_list = new LLCoreInt::HttpMutex *[ssl_mutex_count];
-
- for (int i(0); i < ssl_mutex_count; ++i)
- {
- ssl_mutex_list[i] = new LLCoreInt::HttpMutex;
- }
-
- CRYPTO_set_locking_callback(ssl_locking_callback);
- CRYPTO_THREADID_set_callback(ssl_thread_id_callback);
- }
-}
-
-
-void LLCrashLogger::term_curl()
-{
- CRYPTO_set_locking_callback(NULL);
- for (int i(0); i < ssl_mutex_count; ++i)
- {
- delete ssl_mutex_list[i];
- }
- delete[] ssl_mutex_list;
-}
-
-
-void LLCrashLogger::ssl_thread_id_callback(CRYPTO_THREADID* pthreadid)
-{
-#if LL_WINDOWS
- CRYPTO_THREADID_set_pointer(pthreadid, GetCurrentThread());
-#else
- CRYPTO_THREADID_set_pointer(pthreadid, reinterpret_cast<void*>(pthread_self()));
-#endif
-}
-
-
-void LLCrashLogger::ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */)
-{
- if (type >= 0 && type < ssl_mutex_count)
- {
- if (mode & CRYPTO_LOCK)
- {
- ssl_mutex_list[type]->lock();
- }
- else
- {
- ssl_mutex_list[type]->unlock();
- }
- }
-}
-
+ /** +* @file llcrashlogger.cpp +* @brief Crash logger implementation +* +* $LicenseInfo:firstyear=2003&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2010, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA +* $/LicenseInfo$ +*/ + +#include <cstdio> +#include <cstdlib> +#include <sstream> +#include <map> + +#include "llcrashlogger.h" +#include "llcrashlock.h" +#include "linden_common.h" +#include "llstring.h" +#include "indra_constants.h" // CRASH_BEHAVIOR_... +#include "llerror.h" +#include "llerrorcontrol.h" +#include "lltimer.h" +#include "lldir.h" +#include "llfile.h" +#include "llsdserialize.h" +#include "llproxy.h" +#include "llcorehttputil.h" +#include "llhttpsdhandler.h" +#include "httpcommon.h" +#include "httpresponse.h" +#include "llcleanup.h" + +#include <curl/curl.h> +#include <openssl/crypto.h> + +bool gBreak = false; +bool gSent = false; + +int LLCrashLogger::ssl_mutex_count = 0; +LLCoreInt::HttpMutex ** LLCrashLogger::ssl_mutex_list = NULL; + +#define CRASH_UPLOAD_RETRIES 3 /* seconds */ +#define CRASH_UPLOAD_TIMEOUT 180 /* seconds */ + +class LLCrashLoggerHandler : public LLHttpSDHandler +{ + LOG_CLASS(LLCrashLoggerHandler); +public: + LLCrashLoggerHandler() {} + +protected: + virtual void onSuccess(LLCore::HttpResponse * response, const LLSD &content); + virtual void onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status); + +}; + +void LLCrashLoggerHandler::onSuccess(LLCore::HttpResponse * response, const LLSD &content) +{ + LL_DEBUGS("CRASHREPORT") << "Request to " << response->getRequestURL() << "succeeded" << LL_ENDL; + gBreak = true; + gSent = true; +} + +void LLCrashLoggerHandler::onFailure(LLCore::HttpResponse * response, LLCore::HttpStatus status) +{ + LL_WARNS("CRASHREPORT") << "Request to " << response->getRequestURL() + << " failed: " << status.toString() << LL_ENDL; + gBreak = true; +} + +LLCrashLogger::LLCrashLogger() : + mCrashBehavior(CRASH_BEHAVIOR_ALWAYS_SEND), + mCrashInPreviousExec(false), + mCrashSettings("CrashSettings"), + mSentCrashLogs(false), + mCrashHost("") +{ +} + +LLCrashLogger::~LLCrashLogger() +{ + +} + +// TRIM_SIZE must remain larger than LINE_SEARCH_SIZE. +const int TRIM_SIZE = 128000; +const int LINE_SEARCH_DIST = 500; +const std::string SKIP_TEXT = "\n ...Skipping... \n"; +void trimSLLog(std::string& sllog) +{ + if(sllog.length() > TRIM_SIZE * 2) + { + std::string::iterator head = sllog.begin() + TRIM_SIZE; + std::string::iterator tail = sllog.begin() + sllog.length() - TRIM_SIZE; + std::string::iterator new_head = std::find(head, head - LINE_SEARCH_DIST, '\n'); + if(new_head != head - LINE_SEARCH_DIST) + { + head = new_head; + } + + std::string::iterator new_tail = std::find(tail, tail + LINE_SEARCH_DIST, '\n'); + if(new_tail != tail + LINE_SEARCH_DIST) + { + tail = new_tail; + } + + sllog.erase(head, tail); + sllog.insert(head, SKIP_TEXT.begin(), SKIP_TEXT.end()); + } +} + +std::string getStartupStateFromLog(std::string& sllog) +{ + std::string startup_state = "STATE_FIRST"; + std::string startup_token = "Startup state changing from "; + + int index = sllog.rfind(startup_token); + if (index < 0 || index + startup_token.length() > sllog.length()) { + return startup_state; + } + + // find new line + char cur_char = sllog[index + startup_token.length()]; + std::string::size_type newline_loc = index + startup_token.length(); + while(cur_char != '\n' && newline_loc < sllog.length()) + { + newline_loc++; + cur_char = sllog[newline_loc]; + } + + // get substring and find location of " to " + std::string state_line = sllog.substr(index, newline_loc - index); + std::string::size_type state_index = state_line.find(" to "); + startup_state = state_line.substr(state_index + 4, state_line.length() - state_index - 4); + + return startup_state; +} + +bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename ) +{ + std::string db_file_name = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,filename); + llifstream log_file(db_file_name.c_str()); + + // Look for it in the given file + if (log_file.is_open()) + { + LLSDSerialize::fromXML(dest, log_file); + log_file.close(); + return true; + } + else + { + LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL; + } + return false; +} + +void LLCrashLogger::mergeLogs( LLSD src_sd ) +{ + LLSD::map_iterator iter = src_sd.beginMap(); + LLSD::map_iterator end = src_sd.endMap(); + for( ; iter != end; ++iter) + { + mDebugLog[iter->first] = iter->second; + } +} + +bool LLCrashLogger::readMinidump(std::string minidump_path) +{ + size_t length=0; + + llifstream minidump_stream(minidump_path.c_str(), std::ios_base::in | std::ios_base::binary); + if (minidump_stream.is_open()) + { + minidump_stream.seekg(0, std::ios::end); + length = (size_t)minidump_stream.tellg(); + LL_WARNS("CRASHREPORT") << "minidump length "<< length <<LL_ENDL; + minidump_stream.seekg(0, std::ios::beg); + + LLSD::Binary data; + data.resize(length); + + minidump_stream.read(reinterpret_cast<char *>(&(data[0])),length); + minidump_stream.close(); + + mCrashInfo["Minidump"] = data; + } + else + { + LL_WARNS("CRASHREPORT") << "failed to open minidump "<<minidump_path<<LL_ENDL; + } + + return length > 0; +} + +void LLCrashLogger::gatherFiles() +{ + updateApplication("Gathering logs..."); + + LLSD static_sd; + LLSD dynamic_sd; + //if we ever want to change the endpoint we send crashes to + //we can construct a file download ( a la feature table filename for example) + //containing the new endpoint + LLSD endpoint; + std::string grid; + std::string fqdn; + + bool has_logs = readFromXML( static_sd, "static_debug_info.log" ); + has_logs |= readFromXML( dynamic_sd, "dynamic_debug_info.log" ); + + + if ( has_logs ) + { + mDebugLog = static_sd; + mergeLogs(dynamic_sd); + mCrashInPreviousExec = mDebugLog["CrashNotHandled"].asBoolean(); + + mFileMap["SecondLifeLog"] = mDebugLog["SLLog"].asString(); + mFileMap["SettingsXml"] = mDebugLog["SettingsFilename"].asString(); + mFileMap["CrashHostUrl"] = loadCrashURLSetting(); + if(mDebugLog.has("CAFilename")) + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, + LLCore::HttpRequest::GLOBAL_POLICY_ID, mDebugLog["CAFilename"].asString(), NULL); + } + else + { + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, + LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL); + } + + LL_INFOS("CRASHREPORT") << "Using log file from debug log " << mFileMap["SecondLifeLog"] << LL_ENDL; + LL_INFOS("CRASHREPORT") << "Using settings file from debug log " << mFileMap["SettingsXml"] << LL_ENDL; + } + else + { + // Figure out the filename of the second life log + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, + LLCore::HttpRequest::GLOBAL_POLICY_ID, gDirUtilp->getCAFile(), NULL); + + mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"SecondLife.log"); + mFileMap["SettingsXml"] = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS,"settings.xml"); + } + + if (!gDirUtilp->fileExists(mFileMap["SecondLifeLog"]) ) //We would prefer to get this from the per-run but here's our fallback. + { + mFileMap["SecondLifeLog"] = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,"SecondLife.old"); + } + + gatherPlatformSpecificFiles(); + + + if ( has_logs && (mFileMap["CrashHostUrl"] != "") ) + { + mCrashHost = mFileMap["CrashHostUrl"]; + } + + //default to agni, per product + mAltCrashHost = "http://viewercrashreport.agni.lindenlab.com/cgi-bin/viewercrashreceiver.py"; + + mCrashInfo["DebugLog"] = mDebugLog; + mFileMap["StatsLog"] = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"stats.log"); + + updateApplication("Encoding files..."); + + for(std::map<std::string, std::string>::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr) + { + std::string file = (*itr).second; + if (!file.empty()) + { + LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL; + llifstream f(file.c_str()); + if(f.is_open()) + { + std::stringstream s; + s << f.rdbuf(); + + std::string crash_info = s.str(); + if(itr->first == "SecondLifeLog") + { + if(!mCrashInfo["DebugLog"].has("StartupState")) + { + mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info); + } + trimSLLog(crash_info); + } + + mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info)); + } + else + { + LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL; + } + } + else + { + LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL; + } + } + + std::string minidump_path; + // Add minidump as binary. + bool has_minidump = mDebugLog.has("MinidumpPath"); + + if (has_minidump) + { + minidump_path = mDebugLog["MinidumpPath"].asString(); + has_minidump = readMinidump(minidump_path); + } + else + { + LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL; + } + + if (!has_minidump) //Viewer was probably so hosed it couldn't write remaining data. Try brute force. + { + //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file. + typedef std::vector<std::string> vec; + std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,""); + LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL; + vec file_vec = gDirUtilp->getFilesInDir(pathname); + for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter) + { + if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) ) + { + std::string fullname = pathname + *iter; + llifstream fdat(fullname.c_str(), std::ifstream::binary); + if (fdat) + { + char buf[5]; + fdat.read(buf,4); + fdat.close(); + if (!strncmp(buf,"MDMP",4)) + { + minidump_path = *iter; + has_minidump = readMinidump(fullname); + mDebugLog["MinidumpPath"] = fullname; + } + else + { + LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL; + } + } + else + { + LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL; + } + } + else + { + LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL; + } + } + } + else + { + LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL; + } +} + +LLSD LLCrashLogger::constructPostData() +{ + return mCrashInfo; +} + +const char* const CRASH_SETTINGS_FILE = "settings_crash_behavior.xml"; + +std::string LLCrashLogger::loadCrashURLSetting() +{ + + // First check user_settings (in the user's home dir) + std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, CRASH_SETTINGS_FILE); + if (! mCrashSettings.loadFromFile(filename)) + { + // Next check app_settings (in the SL program dir) + std::string filename = gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, CRASH_SETTINGS_FILE); + mCrashSettings.loadFromFile(filename); + } + + if (! mCrashSettings.controlExists("CrashHostUrl")) + { + return ""; + } + else + { + return mCrashSettings.getString("CrashHostUrl"); + } +} + +bool LLCrashLogger::runCrashLogPost(std::string host, LLSD data, std::string msg, int retries, int timeout) +{ + LLCore::HttpRequest::ptr_t httpRequest(new LLCore::HttpRequest); + LLCore::HttpOptions::ptr_t httpOpts(new LLCore::HttpOptions); + + httpOpts->setTimeout(timeout); + httpOpts->setSSLVerifyPeer(false); + + for(int i = 0; i < retries; ++i) + { + updateApplication(llformat("%s, try %d...", msg.c_str(), i+1)); + + LL_INFOS("CRASHREPORT") << "POST crash data to " << host << LL_ENDL; + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(httpRequest.get(), LLCore::HttpRequest::DEFAULT_POLICY_ID, + host, data, httpOpts, LLCore::HttpHeaders::ptr_t(), LLCore::HttpHandler::ptr_t(new LLCrashLoggerHandler)); + + if (handle == LLCORE_HTTP_HANDLE_INVALID) + { + LLCore::HttpStatus status = httpRequest->getStatus(); + LL_WARNS("CRASHREPORT") << "Request POST failed to " << host << " with status of [" << + status.getType() << "]\"" << status.toString() << "\"" << LL_ENDL; + return false; + } + + while(!gBreak) + { + ms_sleep(250); + updateApplication(); // No new message, just pump the IO + httpRequest->update(0L); + } + if(gSent) + { + return gSent; + } + + LL_WARNS("CRASHREPORT") << "Failed to send crash report to \"" << host << "\"" << LL_ENDL; + } + return gSent; +} + +bool LLCrashLogger::sendCrashLog(std::string dump_dir) +{ + + gDirUtilp->setDumpDir( dump_dir ); + + std::string dump_path = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, + "SecondLifeCrashReport"); + std::string report_file = dump_path + ".log"; + + LL_DEBUGS("CRASHREPORT") << "sending " << report_file << LL_ENDL; + + gatherFiles(); + + LLSD post_data; + post_data = constructPostData(); + + updateApplication("Sending reports..."); + + llofstream out_file(report_file.c_str()); + LLSDSerialize::toPrettyXML(post_data, out_file); + out_file.flush(); + out_file.close(); + + bool sent = false; + + if(mCrashHost != "") + { + LL_WARNS("CRASHREPORT") << "Sending crash data to server from CrashHostUrl '" << mCrashHost << "'" << LL_ENDL; + + std::string msg = "Using override crash server... "; + msg = msg+mCrashHost.c_str(); + updateApplication(msg.c_str()); + + sent = runCrashLogPost(mCrashHost, post_data, std::string("Sending to server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); + } + + if(!sent) + { + updateApplication("Using default server..."); + sent = runCrashLogPost(mAltCrashHost, post_data, std::string("Sending to default server"), CRASH_UPLOAD_RETRIES, CRASH_UPLOAD_TIMEOUT); + } + + mSentCrashLogs = sent; + + return sent; +} + +bool LLCrashLogger::sendCrashLogs() +{ + LLSD locks = mKeyMaster.getProcessList(); + LLSD newlocks = LLSD::emptyArray(); + + LLSD opts = getOptionData(PRIORITY_COMMAND_LINE); + LLSD rec; + + if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") ) + { + rec["pid"]=opts["pid"]; + rec["dumpdir"]=opts["dumpdir"]; + rec["procname"]=opts["procname"]; + } + + if (locks.isArray()) + { + for (LLSD::array_iterator lock=locks.beginArray(); + lock !=locks.endArray(); + ++lock) + { + if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") ) + { + if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) ) + { + newlocks.append(*lock); + } + else + { + //TODO: This is a hack but I didn't want to include boost in another file or retest everything related to lldir + if (LLCrashLock::fileExists((*lock)["dumpdir"].asString())) + { + //the viewer cleans up the log directory on clean shutdown + //but is ignorant of the locking table. + if (!sendCrashLog((*lock)["dumpdir"].asString())) + { + newlocks.append(*lock); //Failed to send log so don't delete it. + } + else + { + mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString()); + } + } + } + } + else + { + LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL; + } + } + } + + if (rec) + { + newlocks.append(rec); + } + + mKeyMaster.putProcessList(newlocks); + return true; +} + +void LLCrashLogger::updateApplication(const std::string& message) +{ + if (!message.empty()) LL_INFOS("CRASHREPORT") << message << LL_ENDL; +} + +bool LLCrashLogger::init() +{ + LL_DEBUGS("CRASHREPORT") << LL_ENDL; + + LLCore::LLHttp::initialize(); + + // We assume that all the logs we're looking for reside on the current drive + gDirUtilp->initAppDirs("SecondLife"); + + LLError::initForApplication(gDirUtilp->getExpandedFilename(LL_PATH_USER_SETTINGS, ""), gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS, "")); + + // Default to the product name "Second Life" (this is overridden by the -name argument) + mProductName = "Second Life"; + + // Rename current log file to ".old" + std::string old_log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log.old"); + std::string log_file = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, "crashreport.log"); + +#if LL_WINDOWS + LLAPRFile::remove(old_log_file); +#endif + + LLFile::rename(log_file.c_str(), old_log_file.c_str()); + + // Set the log file to crashreport.log + LLError::logToFile(log_file); //NOTE: Until this line, LL_INFOS LL_WARNS, etc are blown to the ether. + + LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL; + + // Handle locking + bool locked = mKeyMaster.requestMaster(); //Request master locking file. wait time is defaulted to 300S + + while (!locked && mKeyMaster.isWaiting()) + { + LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL; +#if LL_WINDOWS + Sleep(1000); +#else + ::sleep(1); +#endif + locked = mKeyMaster.checkMaster(); + } + + if (!locked) + { + LL_WARNS("CRASHREPORT") << "Unable to get master lock. Another crash reporter may be hung." << LL_ENDL; + return false; + } + + mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND, + "Controls behavior when viewer crashes " + "(0 = ask before sending crash report, " + "1 = always send crash report, " + "2 = never send crash report)"); + + init_curl(); + LLCore::HttpRequest::createService(); + LLCore::HttpRequest::startThread(); + + return true; +} + +// For cleanup code common to all platforms. +void LLCrashLogger::commonCleanup() +{ + term_curl(); + LLError::logToFile(""); //close crashreport.log + SUBSYSTEM_CLEANUP(LLProxy); +} + +void LLCrashLogger::init_curl() +{ + curl_global_init(CURL_GLOBAL_ALL); + + ssl_mutex_count = CRYPTO_num_locks(); + if (ssl_mutex_count > 0) + { + ssl_mutex_list = new LLCoreInt::HttpMutex *[ssl_mutex_count]; + + for (int i(0); i < ssl_mutex_count; ++i) + { + ssl_mutex_list[i] = new LLCoreInt::HttpMutex; + } + + CRYPTO_set_locking_callback(ssl_locking_callback); + CRYPTO_THREADID_set_callback(ssl_thread_id_callback); + } +} + + +void LLCrashLogger::term_curl() +{ + CRYPTO_set_locking_callback(NULL); + for (int i(0); i < ssl_mutex_count; ++i) + { + delete ssl_mutex_list[i]; + } + delete[] ssl_mutex_list; +} + + +void LLCrashLogger::ssl_thread_id_callback(CRYPTO_THREADID* pthreadid) +{ +#if LL_WINDOWS + CRYPTO_THREADID_set_pointer(pthreadid, GetCurrentThread()); +#else + CRYPTO_THREADID_set_pointer(pthreadid, reinterpret_cast<void*>(pthread_self())); +#endif +} + + +void LLCrashLogger::ssl_locking_callback(int mode, int type, const char * /* file */, int /* line */) +{ + if (type >= 0 && type < ssl_mutex_count) + { + if (mode & CRYPTO_LOCK) + { + ssl_mutex_list[type]->lock(); + } + else + { + ssl_mutex_list[type]->unlock(); + } + } +} + |