implemented EXT-2582 (Save IM chat history as plain text file)

updated 1.23 chat log format - in multilined messages subsequent lines are prepended with a space
multilined messages of an old format are not supported, each line is considered as a separate message
the parser depends on current used date and time format

--HG--
branch : product-engine

4 files changed, 336 insertions, 16 deletions

diff --git a/indra/newview/llimview.cpp b/indra/newview/llimview.cpp
index c5c412d4ee..5481ca97cd 100644
--- a/indra/newview/llimview.cpp
+++ b/indra/newview/llimview.cpp
@@ -83,19 +83,16 @@
 #include "llfirstuse.h"
 #include "llagentui.h"
 
+const static std::string IM_TIME("time");
+const static std::string IM_TEXT("message");
+const static std::string IM_FROM("from");
+const static std::string IM_FROM_ID("from_id");
+
 //
 // Globals
 //
 LLIMMgr* gIMMgr = NULL;
 
-//
-// Statics
-//
-// *FIXME: make these all either UIStrings or Strings
-
-const static std::string IM_SEPARATOR(": ");
-
-
 void toast_callback(const LLSD& msg){
 	// do not show toast in busy mode or it goes from agent
 	if (gAgent.getBusy() || gAgent.getID() == msg["from_id"])
@@ -193,7 +190,13 @@ LLIMModel::LLIMSession::LLIMSession(const LLUUID& session_id, const std::string&
 	}
 
 	if ( gSavedPerAccountSettings.getBOOL("LogShowHistory") )
-		LLLogChat::loadHistory(mName, &chatFromLogFile, (void *)this);
+	{
+		std::list<LLSD> chat_history;
+
+		//involves parsing of a chat history
+		LLLogChat::loadAllHistory(mName, chat_history);
+		addMessagesFromHistory(chat_history);
+	}
 }
 
 void LLIMModel::LLIMSession::onVoiceChannelStateChanged(const LLVoiceChannel::EState& old_state, const LLVoiceChannel::EState& new_state)
@@ -303,6 +306,30 @@ void LLIMModel::LLIMSession::addMessage(const std::string& from, const LLUUID& f
 	}
 }
 
+void LLIMModel::LLIMSession::addMessagesFromHistory(const std::list<LLSD>& history)
+{
+	std::list<LLSD>::const_iterator it = history.begin();
+	while (it != history.end())
+	{
+		const LLSD& msg = *it;
+
+		std::string from = msg[IM_FROM];
+		LLUUID from_id = LLUUID::null;
+		if (msg[IM_FROM_ID].isUndefined())
+		{
+			gCacheName->getUUID(from, from_id);
+		}
+
+
+		std::string timestamp = msg[IM_TIME];
+		std::string text = msg[IM_TEXT];
+
+		addMessage(from, from_id, text, timestamp);
+
+		it++;
+	}
+}
+
 void LLIMModel::LLIMSession::chatFromLogFile(LLLogChat::ELogLineType type, const LLSD& msg, void* userdata)
 {
 	if (!userdata) return;
diff --git a/indra/newview/llimview.h b/indra/newview/llimview.h
index b2d3c6b921..66f92c83a5 100644
--- a/indra/newview/llimview.h
+++ b/indra/newview/llimview.h
@@ -60,6 +60,8 @@ public:
 		virtual ~LLIMSession();
 
 		void sessionInitReplyReceived(const LLUUID& new_session_id);
+
+		void addMessagesFromHistory(const std::list<LLSD>& history);
 		void addMessage(const std::string& from, const LLUUID& from_id, const std::string& utf8_text, const std::string& time);
 		void onVoiceChannelStateChanged(const LLVoiceChannel::EState& old_state, const LLVoiceChannel::EState& new_state);
 		static void chatFromLogFile(LLLogChat::ELogLineType type, const LLSD& msg, void* userdata);
diff --git a/indra/newview/lllogchat.cpp b/indra/newview/lllogchat.cpp
index 9caa863bd8..33fd3e3bf1 100644
--- a/indra/newview/lllogchat.cpp
+++ b/indra/newview/lllogchat.cpp
@@ -32,15 +32,59 @@
 
 #include "llviewerprecompiledheaders.h"
 
+#include "llagent.h"
+#include "llagentui.h"
 #include "lllogchat.h"
-#include "llappviewer.h"
 #include "llfloaterchat.h"
 #include "lltrans.h"
 #include "llviewercontrol.h"
-#include "llsdserialize.h"
+
+#include <boost/algorithm/string/trim.hpp>
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/regex.hpp>
+#include <boost/regex/v4/match_results.hpp>
 
 const S32 LOG_RECALL_SIZE = 2048;
 
+const static std::string IM_TIME("time");
+const static std::string IM_TEXT("message");
+const static std::string IM_FROM("from");
+const static std::string IM_FROM_ID("from_id");
+const static std::string IM_SEPARATOR(": ");
+
+const static std::string NEW_LINE("\n");
+const static std::string NEW_LINE_SPACE_PREFIX("\n ");
+const static std::string TWO_SPACES("  ");
+const static std::string MULTI_LINE_PREFIX(" ");
+
+//viewer 1.23 may have used "You" for Agent's entries
+const static std::string YOU("You");
+
+/**
+ *  Chat log lines - timestamp and name are optional but message text is mandatory.
+ *
+ *  Typical plain text chat log lines:
+ *
+ *  SuperCar: You aren't the owner
+ *  [2:59]  SuperCar: You aren't the owner
+ *  [2009/11/20 3:00]  SuperCar: You aren't the owner
+ *  Katar Ivercourt is Offline
+ *  [3:00]  Katar Ivercourt is Offline
+ *  [2009/11/20 3:01]  Corba ProductEngine is Offline
+ */
+const static boost::regex TIMESTAMP_AND_STUFF("^(\\[\\d{4}/\\d{1,2}/\\d{1,2}\\s+\\d{1,2}:\\d{2}\\]\\s+|\\[\\d{1,2}:\\d{2}\\]\\s+)?(.*)$");
+
+/**
+ *  Regular expression suitable to match names like
+ *  "You", "Second Life", "Igor ProductEngine", "Object", "Mega House"
+ */
+const static boost::regex NAME_AND_TEXT("(You:|Second Life:|[^\\s:]+\\s*[:]{1}|\\S+\\s+[^\\s:]+[:]{1})?(\\s*)(.*)");
+
+const static int IDX_TIMESTAMP = 1;
+const static int IDX_STUFF = 2;
+const static int IDX_NAME = 1;
+const static int IDX_TEXT = 3;
+
 //static
 std::string LLLogChat::makeLogFileName(std::string filename)
 {
@@ -118,7 +162,7 @@ void LLLogChat::saveHistory(const std::string& filename,
 	item["from_id"]	= from_id;
 	item["message"]	= line;
 
-	file << LLSDOStreamer <LLSDNotationFormatter>(item) << std::endl;
+	file << LLChatLogFormatter(item) << std::endl;
 
 	file.close();
 }
@@ -154,9 +198,6 @@ void LLLogChat::loadHistory(const std::string& filename, void (*callback)(ELogLi
 			}
 		}
 
-		// the parser's destructor is protected so we cannot create in the stack.
-		LLPointer<LLSDParser> parser = new LLSDNotationParser();
-
 		while ( fgets(buffer, LOG_RECALL_SIZE, fptr)  && !feof(fptr) ) 
 		{
 			len = strlen(buffer) - 1;		/*Flawfinder: ignore*/
@@ -167,7 +208,8 @@ void LLLogChat::loadHistory(const std::string& filename, void (*callback)(ELogLi
 				LLSD item;
 				std::string line(buffer);
 				std::istringstream iss(line);
-				if (parser->parse(iss, item, line.length()) == LLSDParser::PARSE_FAILURE)
+				
+				if (!LLChatLogParser::parse(line, item))
 				{
 					item["message"]	= line;
 					callback(LOG_LINE, item, userdata);
@@ -187,3 +229,197 @@ void LLLogChat::loadHistory(const std::string& filename, void (*callback)(ELogLi
 		fclose(fptr);
 	}
 }
+
+void append_to_last_message(std::list<LLSD>& messages, const std::string& line)
+{
+	if (!messages.size()) return;
+
+	std::string im_text = messages.back()[IM_TEXT].asString();
+	im_text.append(line);
+	messages.back()[IM_TEXT] = im_text;
+}
+
+void LLLogChat::loadAllHistory(const std::string& session_name, std::list<LLSD>& messages)
+{
+	if (session_name.empty())
+	{
+		llwarns << "Session name is Empty!" << llendl;
+		return ;
+	}
+
+	LLFILE* fptr = LLFile::fopen(makeLogFileName(session_name), "r");		/*Flawfinder: ignore*/
+	if (!fptr) return;	//No previous conversation with this name.
+
+	char buffer[LOG_RECALL_SIZE];		/*Flawfinder: ignore*/
+	char *bptr;
+	S32 len;
+	bool firstline = TRUE;
+
+	if (fseek(fptr, (LOG_RECALL_SIZE - 1) * -1  , SEEK_END))
+	{	//File is smaller than recall size.  Get it all.
+		firstline = FALSE;
+		if (fseek(fptr, 0, SEEK_SET))
+		{
+			fclose(fptr);
+			return;
+		}
+	}
+
+	while (fgets(buffer, LOG_RECALL_SIZE, fptr)  && !feof(fptr)) 
+	{
+		len = strlen(buffer) - 1;		/*Flawfinder: ignore*/
+		for (bptr = (buffer + len); (*bptr == '\n' || *bptr == '\r') && bptr>buffer; bptr--)	*bptr='\0';
+		
+		if (firstline)
+		{
+			firstline = FALSE;
+			continue;
+		}
+
+		std::string line(buffer);
+
+		//updated 1.23 plaint text log format requires a space added before subsequent lines in a multilined message
+		if (' ' == line[0])
+		{
+			line.erase(0, MULTI_LINE_PREFIX.length());
+			append_to_last_message(messages, '\n' + line);
+		}
+		else if (0 == len && ('\n' == line[0] || '\r' == line[0]))
+		{
+			//to support old format's multilined messages with new lines used to divide paragraphs
+			append_to_last_message(messages, line);
+		}
+		else
+		{
+			LLSD item;
+			if (!LLChatLogParser::parse(line, item))
+			{
+				item[IM_TEXT] = line;
+			}
+			messages.push_back(item);
+		}
+	}
+	fclose(fptr);
+}
+
+//*TODO mark object's names in a special way so that they will be distinguishable form avatar name 
+//which are more strict by its nature (only firstname and secondname)
+//Example, an object's name can be writen like "Object <actual_object's_name>"
+void LLChatLogFormatter::format(const LLSD& im, std::ostream& ostr) const
+{
+	if (!im.isMap())
+	{
+		llwarning("invalid LLSD type of an instant message", 0);
+		return;
+	}
+
+	if (im[IM_TIME].isDefined())
+	{
+		std::string timestamp = im[IM_TIME].asString();
+		boost::trim(timestamp);
+		ostr << '[' << timestamp << ']' << TWO_SPACES;
+	}
+
+	//*TODO mark object's names in a special way so that they will be distinguishable form avatar name 
+	//which are more strict by its nature (only firstname and secondname)
+	//Example, an object's name can be writen like "Object <actual_object's_name>"
+	if (im[IM_FROM].isDefined())
+	{
+		std::string from = im[IM_FROM].asString();
+		boost::trim(from);
+		if (from.size())
+		{
+			ostr << from << IM_SEPARATOR;
+		}
+	}
+	
+	if (im[IM_TEXT].isDefined())
+	{
+		std::string im_text = im[IM_TEXT].asString();
+
+		//multilined text will be saved with prepended spaces
+		boost::replace_all(im_text, NEW_LINE, NEW_LINE_SPACE_PREFIX);
+		ostr << im_text;
+	}
+}
+
+bool LLChatLogParser::parse(std::string& raw, LLSD& im)
+{
+	if (!raw.length()) return false;
+	
+	im = LLSD::emptyMap();
+
+	//matching a timestamp
+	boost::match_results<std::string::const_iterator> matches;
+	if (!boost::regex_match(raw, matches, TIMESTAMP_AND_STUFF)) return false;
+	
+	bool has_timestamp = matches[IDX_TIMESTAMP].matched;
+	if (has_timestamp)
+	{
+		//timestamp was successfully parsed
+		std::string timestamp = matches[IDX_TIMESTAMP];
+		boost::trim(timestamp);
+		timestamp.erase(0, 1);
+		timestamp.erase(timestamp.length()-1, 1);
+		im[IM_TIME] = timestamp;	
+	}
+	else
+	{
+		//timestamp is optional
+		im[IM_TIME] = "";
+	}
+
+	bool has_stuff = matches[IDX_STUFF].matched;
+	if (!has_stuff)
+	{
+		return false;  //*TODO should return false or not?
+	}
+
+	//matching a name and a text
+	std::string stuff = matches[IDX_STUFF];
+	boost::match_results<std::string::const_iterator> name_and_text;
+	if (!boost::regex_match(stuff, name_and_text, NAME_AND_TEXT)) return false;
+	
+	bool has_name = name_and_text[IDX_NAME].matched;
+	std::string name = name_and_text[IDX_NAME];
+
+	//we don't need a name/text separator
+	if (has_name && name.length() && name[name.length()-1] == ':')
+	{
+		name.erase(name.length()-1, 1);
+	}
+
+	if (!has_name || name == SYSTEM_FROM)
+	{
+		//name is optional too
+		im[IM_FROM] = SYSTEM_FROM;
+		im[IM_FROM_ID] = LLUUID::null;
+	}
+
+	if (!has_name)
+	{
+		//text is mandatory
+		im[IM_TEXT] = stuff;
+		return true; //parse as a message from Second Life
+	}
+	
+	bool has_text = name_and_text[IDX_TEXT].matched;
+	if (!has_text) return false;
+
+	//for parsing logs created in very old versions of a viewer
+	if (name == "You")
+	{
+		std::string agent_name;
+		LLAgentUI::buildFullname(agent_name);
+		im[IM_FROM] = agent_name;
+		im[IM_FROM_ID] = gAgentID;
+	}
+	else
+	{
+		im[IM_FROM] = name;
+	}
+	
+
+	im[IM_TEXT] = name_and_text[IDX_TEXT];
+	return true;  //parsed name and message text, maybe have a timestamp too
+}
diff --git a/indra/newview/lllogchat.h b/indra/newview/lllogchat.h
index e252cd7d41..3d3f5c4458 100644
--- a/indra/newview/lllogchat.h
+++ b/indra/newview/lllogchat.h
@@ -51,11 +51,66 @@ public:
 				const LLUUID& from_id,
 				const std::string& line);
 
+	/** @deprecated @see loadAllHistory() */
 	static void loadHistory(const std::string& filename, 
 		                    void (*callback)(ELogLineType, const LLSD&, void*), 
 							void* userdata);
+
+	static void loadAllHistory(const std::string& session_name, std::list<LLSD>& messages);
 private:
 	static std::string cleanFileName(std::string filename);
 };
 
+/**
+ * Formatter for the plain text chat log files
+ */
+class LLChatLogFormatter
+{
+public:
+	LLChatLogFormatter(const LLSD& im) : mIM(im) {}
+	virtual ~LLChatLogFormatter() {};
+
+	friend std::ostream& operator<<(std::ostream& str, const LLChatLogFormatter& formatter)
+	{
+		formatter.format(formatter.mIM, str);
+		return str;
+	}
+
+protected:
+
+	/**
+	 * Format an instant message to a stream
+	 * Timestamps and sender names are required
+	 * New lines of multilined messages are prepended with a space
+	 */
+	void format(const LLSD& im, std::ostream& ostr) const;
+
+	LLSD mIM;
+};
+
+/**
+ * Parser for the plain text chat log files
+ */
+class LLChatLogParser
+{
+public:
+
+	 /**
+	 * Parse a line from the plain text chat log file
+	 * General plain text log format is like: "[timestamp]  [name]: [message]"
+	 * [timestamp] and [name] are optional
+	 * Examples of plain text chat log lines:
+	 * "[2009/11/20 2:53]  Igor ProductEngine: howdy"
+	 * "Igor ProductEngine: howdy"
+	 * "Dserduk ProductEngine is Online"
+	 *
+	 * @return false if failed to parse mandatory data - message text
+	 */
+	static bool parse(std::string& raw, LLSD& im);
+
+protected:
+	LLChatLogParser();
+	virtual ~LLChatLogParser() {};
+};
+
 #endif