MAINT-6322 fix merge error that prevented crash dumps from being located for upload (and add better logging)

3 files changed, 136 insertions, 38 deletions

diff --git a/indra/llcommon/llapp.cpp b/indra/llcommon/llapp.cpp
index 5a40845e7d..eb0699ad41 100644
--- a/indra/llcommon/llapp.cpp
+++ b/indra/llcommon/llapp.cpp
@@ -929,7 +929,7 @@ bool unix_minidump_callback(const google_breakpad::MinidumpDescriptor& minidump_
 
 	strncpy(path, minidump_desc.path(), remaining);
 	
-	LL_INFOS() << "generated minidump: " << LLApp::instance()->getMiniDumpFilename() << LL_ENDL;
+	LL_INFOS("CRASHREPORT") << "generated minidump: " << LLApp::instance()->getMiniDumpFilename() << LL_ENDL;
 	LLApp::runErrorHandler();
 	
 #ifndef LL_RELEASE_FOR_DOWNLOAD
@@ -975,7 +975,7 @@ bool unix_post_minidump_callback(const char *dump_dir,
 		strncpy(path, ".dmp", remaining);
 	}
 	
-	LL_INFOS() << "generated minidump: " << LLApp::instance()->getMiniDumpFilename() << LL_ENDL;
+	LL_INFOS("CRASHREPORT") << "generated minidump: " << LLApp::instance()->getMiniDumpFilename() << LL_ENDL;
 	LLApp::runErrorHandler();
 	
 #ifndef LL_RELEASE_FOR_DOWNLOAD
@@ -1019,7 +1019,7 @@ bool windows_post_minidump_callback(const wchar_t* dump_path,
 		strncpy(path, ".dmp", remaining);
 	}
 
-	LL_INFOS() << "generated minidump: " << LLApp::instance()->getMiniDumpFilename() << LL_ENDL;
+	LL_INFOS("CRASHREPORT") << "generated minidump: " << LLApp::instance()->getMiniDumpFilename() << LL_ENDL;
    // *NOTE:Mani - this code is stolen from LLApp, where its never actually used.
 	//OSMessageBox("Attach Debugger Now", "Error", OSMB_OK);
    // *TODO: Translate the signals/exceptions into cross-platform stuff
diff --git a/indra/llcrashlogger/llcrashlogger.cpp b/indra/llcrashlogger/llcrashlogger.cpp
index 8e0a725954..4bfc2d40dd 100644
--- a/indra/llcrashlogger/llcrashlogger.cpp
+++ b/indra/llcrashlogger/llcrashlogger.cpp
@@ -162,6 +162,10 @@ bool LLCrashLogger::readFromXML(LLSD& dest, const std::string& filename )
         log_file.close();
         return true;
     }
+    else
+    {
+        LL_WARNS("CRASHREPORT") << "Failed to open " << db_file_name << LL_ENDL;
+    }
     return false;
 }
 
@@ -194,6 +198,11 @@ bool LLCrashLogger::readMinidump(std::string minidump_path)
 		
 		mCrashInfo["Minidump"] = data;
 	}
+    else
+    {
+        LL_WARNS("CRASHREPORT") << "failed to open minidump "<<minidump_path<<LL_ENDL;
+    }
+    
 	return (length>0?true:false);
 }
 
@@ -270,27 +279,36 @@ void LLCrashLogger::gatherFiles()
 
 	for(std::map<std::string, std::string>::iterator itr = mFileMap.begin(); itr != mFileMap.end(); ++itr)
 	{
-		std::ifstream f((*itr).second.c_str());
-		if(f.is_open())
+        std::string file = (*itr).second;
+        if (!file.empty())
         {
-            std::stringstream s;
-            s << f.rdbuf();
-
-            std::string crash_info = s.str();
-            if(itr->first == "SecondLifeLog")
+            LL_DEBUGS("CRASHREPORT") << "trying to read " << itr->first << ": " << file << LL_ENDL;
+            std::ifstream f(file.c_str());
+            if(f.is_open())
             {
-                if(!mCrashInfo["DebugLog"].has("StartupState"))
+                std::stringstream s;
+                s << f.rdbuf();
+
+                std::string crash_info = s.str();
+                if(itr->first == "SecondLifeLog")
                 {
-                    mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info);
+                    if(!mCrashInfo["DebugLog"].has("StartupState"))
+                    {
+                        mCrashInfo["DebugLog"]["StartupState"] = getStartupStateFromLog(crash_info);
+                    }
+                    trimSLLog(crash_info);
                 }
-                trimSLLog(crash_info);
-            }
 
-            mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info));
+                mCrashInfo[(*itr).first] = LLStringFn::strip_invalid_xml(rawstr_to_utf8(crash_info));
+            }
+            else
+            {
+                LL_WARNS("CRASHREPORT") << "Failed to open file " << file << LL_ENDL;
+            }
         }
         else
         {
-            LL_WARNS("CRASHREPORT") << "Can't find file " << (*itr).second << LL_ENDL;
+            LL_DEBUGS("CRASHREPORT") << "empty file in list for " << itr->first << LL_ENDL;
         }
 	}
 	
@@ -301,20 +319,21 @@ void LLCrashLogger::gatherFiles()
 	if (has_minidump)
 	{
 		minidump_path = mDebugLog["MinidumpPath"].asString();
-	}
-
-	if (has_minidump)
-	{
 		has_minidump = readMinidump(minidump_path);
 	}
+    else
+    {
+        LL_WARNS("CRASHREPORT") << "DebugLog does not have MinidumpPath" << LL_ENDL;
+    }
 
     if (!has_minidump)  //Viewer was probably so hosed it couldn't write remaining data.  Try brute force.
     {
-       //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file.
+        //Look for a filename at least 30 characters long in the dump dir which contains the characters MDMP as the first 4 characters in the file.
         typedef std::vector<std::string> vec;
         std::string pathname = gDirUtilp->getExpandedFilename(LL_PATH_DUMP,"");
+        LL_WARNS("CRASHREPORT") << "Searching for minidump in " << pathname << LL_ENDL;
         vec file_vec = gDirUtilp->getFilesInDir(pathname);
-        for(vec::const_iterator iter=file_vec.begin(); iter!=file_vec.end(); ++iter)
+        for(vec::const_iterator iter=file_vec.begin(); !has_minidump && iter!=file_vec.end(); ++iter)
         {
             if ( ( iter->length() > 30 ) && (iter->rfind(".dmp") == (iter->length()-4) ) )
             {
@@ -330,15 +349,27 @@ void LLCrashLogger::gatherFiles()
                         minidump_path = *iter;
                         has_minidump = readMinidump(fullname);
 						mDebugLog["MinidumpPath"] = fullname;
-						if (has_minidump) 
-						{
-							break;
-						}
+                    }
+                    else
+                    {
+                        LL_DEBUGS("CRASHREPORT") << "MDMP not found in " << fullname << LL_ENDL;
                     }
                 }
+                else
+                {
+                    LL_DEBUGS("CRASHREPORT") << "failed to open " << fullname << LL_ENDL;
+                }
             }
+            else
+            {
+                LL_DEBUGS("CRASHREPORT") << "Name does not match minidump name pattern " << *iter << LL_ENDL;
+            }            
         }
     }
+    else
+    {
+        LL_WARNS("CRASHREPORT") << "readMinidump returned no minidump" << LL_ENDL;
+    }
 }
 
 LLSD LLCrashLogger::constructPostData()
@@ -458,22 +489,63 @@ bool LLCrashLogger::sendCrashLog(std::string dump_dir)
 
 bool LLCrashLogger::sendCrashLogs()
 {
-    
+    LLSD locks = mKeyMaster.getProcessList();
+    LLSD newlocks = LLSD::emptyArray();
+
 	LLSD opts = getOptionData(PRIORITY_COMMAND_LINE);
     LLSD rec;
 
-	if ( opts.has("dumpdir") )
+	if ( opts.has("pid") && opts.has("dumpdir") && opts.has("procname") )
     {
         rec["pid"]=opts["pid"];
         rec["dumpdir"]=opts["dumpdir"];
         rec["procname"]=opts["procname"];
     }
-    else
+	
+    if (locks.isArray())
     {
-        return false;
-    }       
+        for (LLSD::array_iterator lock=locks.beginArray();
+             lock !=locks.endArray();
+             ++lock)
+        {
+            if ( (*lock).has("pid") && (*lock).has("dumpdir") && (*lock).has("procname") )
+            {
+                if ( mKeyMaster.isProcessAlive( (*lock)["pid"].asInteger(), (*lock)["procname"].asString() ) )
+                {
+                    newlocks.append(*lock);
+                }
+                else
+                {
+					//TODO:  This is a hack but I didn't want to include boost in another file or retest everything related to lldir 
+                    if (LLCrashLock::fileExists((*lock)["dumpdir"].asString()))
+                    {
+                        //the viewer cleans up the log directory on clean shutdown
+                        //but is ignorant of the locking table. 
+                        if (!sendCrashLog((*lock)["dumpdir"].asString()))
+                        {
+                            newlocks.append(*lock);    //Failed to send log so don't delete it.
+                        }
+                        else
+                        {
+                            mKeyMaster.cleanupProcess((*lock)["dumpdir"].asString());
+                        }
+                    }
+				}
+            }
+            else
+            {
+                LL_INFOS() << "Discarding corrupted entry from lock table." << LL_ENDL;
+            }
+        }
+    }
 
-    return sendCrashLog(rec["dumpdir"].asString());
+    if (rec)
+    {
+        newlocks.append(rec);
+    }
+    
+    mKeyMaster.putProcessList(newlocks);
+    return true;
 }
 
 void LLCrashLogger::updateApplication(const std::string& message)
@@ -510,6 +582,26 @@ bool LLCrashLogger::init()
 
     LL_INFOS("CRASHREPORT") << "Crash reporter file rotation complete." << LL_ENDL;
 
+    // Handle locking
+    bool locked = mKeyMaster.requestMaster();  //Request master locking file.  wait time is defaulted to 300S
+    
+    while (!locked && mKeyMaster.isWaiting())
+    {
+		LL_INFOS("CRASHREPORT") << "Waiting for lock." << LL_ENDL;
+#if LL_WINDOWS
+		Sleep(1000);
+#else
+        sleep(1);
+#endif 
+        locked = mKeyMaster.checkMaster();
+    }
+    
+    if (!locked)
+    {
+        LL_WARNS("CRASHREPORT") << "Unable to get master lock.  Another crash reporter may be hung." << LL_ENDL;
+        return false;
+    }
+
     mCrashSettings.declareS32("CrashSubmitBehavior", CRASH_BEHAVIOR_ALWAYS_SEND,
 							  "Controls behavior when viewer crashes "
 							  "(0 = ask before sending crash report, "
diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp
index aa152ffdd6..e7d2b19186 100644
--- a/indra/newview/llappviewer.cpp
+++ b/indra/newview/llappviewer.cpp
@@ -3613,9 +3613,9 @@ void getFileList()
 
 void LLAppViewer::handleViewerCrash()
 {
-	LL_INFOS() << "Handle viewer crash entry." << LL_ENDL;
+	LL_INFOS("CRASHREPORT") << "Handle viewer crash entry." << LL_ENDL;
 
-	LL_INFOS() << "Last render pool type: " << LLPipeline::sCurRenderPoolType << LL_ENDL ;
+	LL_INFOS("CRASHREPORT") << "Last render pool type: " << LLPipeline::sCurRenderPoolType << LL_ENDL ;
 
 	LLMemory::logMemoryInfo(true) ;
 
@@ -3723,30 +3723,36 @@ void LLAppViewer::handleViewerCrash()
 #endif 
 
 	char *minidump_file = pApp->getMiniDumpFilename();
-
+    LL_DEBUGS("CRASHREPORT") << "minidump file name " << minidump_file << LL_ENDL;
 	if(minidump_file && minidump_file[0] != 0)
 	{
 		gDebugInfo["Dynamic"]["MinidumpPath"] = minidump_file;
 	}
-#ifdef LL_WINDOWS
 	else
 	{
+#ifdef LL_WINDOWS
 		getFileList();
+#else
+        LL_WARNS("CRASHREPORT") << "no minidump file?" << LL_ENDL;
+#endif        
 	}
-#endif
     gDebugInfo["Dynamic"]["CrashType"]="crash";
 	
 	if (gMessageSystem && gDirUtilp)
 	{
 		std::string filename;
 		filename = gDirUtilp->getExpandedFilename(LL_PATH_DUMP, "stats.log");
+        LL_DEBUGS("CRASHREPORT") << "recording stats " << filename << LL_ENDL;
 		llofstream file(filename.c_str(), std::ios_base::binary);
 		if(file.good())
 		{
-			LL_INFOS() << "Handle viewer crash generating stats log." << LL_ENDL;
 			gMessageSystem->summarizeLogs(file);
 			file.close();
 		}
+        else
+        {
+            LL_WARNS("CRASHREPORT") << "problem recording stats" << LL_ENDL;
+        }        
 	}
 
 	if (gMessageSystem)