merge changes for STORM-1831

4 files changed, 430 insertions, 231 deletions

diff --git a/indra/llui/llkeywords.cpp b/indra/llui/llkeywords.cpp
index 26d27d1f34..6aeaf4798b 100755
--- a/indra/llui/llkeywords.cpp
+++ b/indra/llui/llkeywords.cpp
@@ -1,25 +1,25 @@
-/** 
+/**
  * @file llkeywords.cpp
  * @brief Keyword list for LSL
  *
  * $LicenseInfo:firstyear=2000&license=viewerlgpl$
  * Second Life Viewer Source Code
  * Copyright (C) 2010, Linden Research, Inc.
- * 
+ *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation;
  * version 2.1 of the License only.
- * 
+ *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
+ *
  * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  * $/LicenseInfo$
  */
@@ -30,11 +30,9 @@
 #include <fstream>
 
 #include "llkeywords.h"
+#include "llsdserialize.h"
 #include "lltexteditor.h"
 #include "llstl.h"
-#include <boost/tokenizer.hpp>
-
-const U32 KEYWORD_FILE_CURRENT_VERSION = 2;
 
 inline BOOL LLKeywordToken::isHead(const llwchar* s) const
 {
@@ -53,10 +51,6 @@ inline BOOL LLKeywordToken::isHead(const llwchar* s) const
 	return res;
 }
 
-LLKeywords::LLKeywords() : mLoaded(FALSE)
-{
-}
-
 inline BOOL LLKeywordToken::isTail(const llwchar* s) const
 {
 	BOOL res = TRUE;
@@ -73,6 +67,11 @@ inline BOOL LLKeywordToken::isTail(const llwchar* s) const
 	return res;
 }
 
+LLKeywords::LLKeywords() : mLoaded(FALSE)
+{
+	//setFilenameSyntax( gDirUtilp->getExpandedFilename(LL_PATH_APP_SETTINGS,"keywords_lsl_default.xml") );
+}
+
 LLKeywords::~LLKeywords()
 {
 	std::for_each(mWordTokenMap.begin(), mWordTokenMap.end(), DeletePairedPointer());
@@ -80,180 +79,338 @@ LLKeywords::~LLKeywords()
 	std::for_each(mDelimiterTokenList.begin(), mDelimiterTokenList.end(), DeletePointer());
 }
 
-BOOL LLKeywords::loadFromFile( const std::string& filename )
-{
-	mLoaded = FALSE;
 
-	////////////////////////////////////////////////////////////
-	// File header
 
-	const S32 BUFFER_SIZE = 1024;
-	char	buffer[BUFFER_SIZE];	/* Flawfinder: ignore */
+void LLKeywords::addColorGroup(const std::string key_in, const LLColor4 color)
+{
+	WStringMapIndex key ( utf8str_to_wstring(key_in) );
+	mColorGroupMap[key] = color;
+}
 
-	llifstream file;
-	file.open(filename);	/* Flawfinder: ignore */
-	if( file.fail() )
+// Add the token as described
+void LLKeywords::addToken(LLKeywordToken::TOKEN_TYPE type,
+						  const std::string& key_in,
+						  const LLColor4& color,
+						  const std::string& tool_tip_in,
+						  const std::string& delimiter_in)
+{
+	std::string tip_text = tool_tip_in;
+	LLStringUtil::replaceString(tip_text, "\\n", "\n" );
+	LLStringUtil::replaceString(tip_text, "\t", " " );
+	if (tip_text =="")
 	{
-		llinfos << "LLKeywords::loadFromFile()  Unable to open file: " << filename << llendl;
-		return mLoaded;
+		tip_text = "[no info]";
 	}
+	LLWString tool_tip = utf8str_to_wstring(tip_text);
 
-	// Identifying string
-	file >> buffer;
-	if( strcmp( buffer, "llkeywords" ) )
+	LLWString key = utf8str_to_wstring(key_in);
+	LLWString delimiter = utf8str_to_wstring(delimiter_in);
+	switch(type)
 	{
-		llinfos << filename << " does not appear to be a keyword file" << llendl;
-		return mLoaded;
-	}
+	case LLKeywordToken::TT_CONSTANT:
+	case LLKeywordToken::TT_CONTROL:
+	case LLKeywordToken::TT_EVENT:
+	case LLKeywordToken::TT_FUNCTION:
+	case LLKeywordToken::TT_LABEL:
+	case LLKeywordToken::TT_SECTION:
+	case LLKeywordToken::TT_TYPE:
+	case LLKeywordToken::TT_WORD:
+		mWordTokenMap[key] = new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null);
+		break;
 
-	// Check file version
-	file >> buffer;
-	U32	version_num;
-	file >> version_num;
-	if( strcmp(buffer, "version") || version_num != (U32)KEYWORD_FILE_CURRENT_VERSION )
-	{
-		llinfos << filename << " does not appear to be a version " << KEYWORD_FILE_CURRENT_VERSION << " keyword file" << llendl;
-		return mLoaded;
-	}
+	case LLKeywordToken::TT_LINE:
+		mLineTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null));
+		break;
 
-	// start of line (SOL)
-	std::string SOL_COMMENT("#");
-	std::string SOL_WORD("[word ");
-	std::string SOL_LINE("[line ");
-	std::string SOL_ONE_SIDED_DELIMITER("[one_sided_delimiter ");
-	std::string SOL_TWO_SIDED_DELIMITER("[two_sided_delimiter ");
-	std::string SOL_DOUBLE_QUOTATION_MARKS("[double_quotation_marks ");
+	case LLKeywordToken::TT_TWO_SIDED_DELIMITER:
+	case LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS:
+	case LLKeywordToken::TT_ONE_SIDED_DELIMITER:
+		mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
+		break;
 
-	LLColor3 cur_color( 1, 0, 0 );
-	LLKeywordToken::TOKEN_TYPE cur_type = LLKeywordToken::WORD;
+	default:
+		llassert(0);
+	}
+}
 
-	while (!file.eof())
+std::string LLKeywords::getArguments(LLSD& arguments)
+{
+	std::string args = "";
+	if (arguments.isMap())
 	{
-		buffer[0] = 0;
-		file.getline( buffer, BUFFER_SIZE );
-		std::string line(buffer);
-		if( line.find(SOL_COMMENT) == 0 )
-		{
-			continue;
-		}
-		else if( line.find(SOL_WORD) == 0 )
-		{
-			cur_color = readColor( line.substr(SOL_WORD.size()) );
-			cur_type = LLKeywordToken::WORD;
-			continue;
-		}
-		else if( line.find(SOL_LINE) == 0 )
-		{
-			cur_color = readColor( line.substr(SOL_LINE.size()) );
-			cur_type = LLKeywordToken::LINE;
-			continue;
-		}
-		else if( line.find(SOL_TWO_SIDED_DELIMITER) == 0 )
+		int count = arguments.size();
+		LLSD::map_iterator argsIt = arguments.beginMap();
+		for ( ; argsIt != arguments.endMap(); ++argsIt)
 		{
-			cur_color = readColor( line.substr(SOL_TWO_SIDED_DELIMITER.size()) );
-			cur_type = LLKeywordToken::TWO_SIDED_DELIMITER;
-			continue;
-		}
-		else if( line.find(SOL_DOUBLE_QUOTATION_MARKS) == 0 )
-		{
-			cur_color = readColor( line.substr(SOL_DOUBLE_QUOTATION_MARKS.size()) );
-			cur_type = LLKeywordToken::DOUBLE_QUOTATION_MARKS;
-			continue;
-		}
-		else if( line.find(SOL_ONE_SIDED_DELIMITER) == 0 )	
-		{
-			cur_color = readColor( line.substr(SOL_ONE_SIDED_DELIMITER.size()) );
-			cur_type = LLKeywordToken::ONE_SIDED_DELIMITER;
-			continue;
+				args += argsIt->second.get("type").asString() + " " + argsIt->first;
+				if (count-- > 1)
+				{
+					args += ", ";
+				}
 		}
+	}
+	else if (!arguments.isUndefined())
+	{
+		LL_WARNS("Arguments") << "Not a map! Invalid LLSD passed to function.\n" << arguments << LL_ENDL;
+	}
+	return args == "" ? " void " : args;
+}
 
-		std::string token_buffer( line );
-		LLStringUtil::trim(token_buffer);
-		
-		typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
-		boost::char_separator<char> sep_word("", " \t");
-		tokenizer word_tokens(token_buffer, sep_word);
-		tokenizer::iterator token_word_iter = word_tokens.begin();
+std::string LLKeywords::getAttribute(const std::string& key)
+{
+	attribute_iterator_t it = mAttributes.find(key);
+	return (it != mAttributes.end()) ? it->second : "";
+}
 
-		if( !token_buffer.empty() && token_word_iter != word_tokens.end() )
-		{
-			// first word is the keyword or a left delimiter
-			std::string keyword = (*token_word_iter);
-			LLStringUtil::trim(keyword);
+LLColor4 LLKeywords::getColorGroup(const std::string key_in)
+{
+	std::string ColourGroup = "Black";
+	if (key_in == "constants-float") {
+		ColourGroup = "SyntaxLslConstantFloat";
+	} else if (key_in == "constants-integer") {
+		ColourGroup = "SyntaxLslConstantInteger";
+	} else if (key_in == "constants-key") {
+		ColourGroup = "SyntaxLslConstantKey";
+	} else if (key_in == "constants-string") {
+		ColourGroup = "SyntaxLslConstantRotation";
+	} else if (key_in == "constants-string") {
+		ColourGroup = "SyntaxLslConstantString";
+	} else if (key_in == "constants-vector") {
+		ColourGroup = "SyntaxLslConstantVector";
+	} else if (key_in == "controls") {
+		ColourGroup = "SyntaxLslControlFlow";
+	} else if (key_in == "events") {
+		ColourGroup = "SyntaxLslEvent";
+	} else if (key_in == "functions") {
+		ColourGroup = "SyntaxLslFunction";
+	} else if (key_in == "types") {
+		ColourGroup = "SyntaxLslDataType";
+	} else if (key_in == "sections") {
+		ColourGroup = "SyntaxLslSection";
+	} else if (key_in == "misc-double_quotation_marks") {
+		ColourGroup = "SyntaxLslStringLiteral";
+	} else if (key_in == "misc-comments_1_sided") {
+		ColourGroup = "SyntaxLslComment1Sided";
+	} else if (key_in == "misc-comments_2_sided") {
+		ColourGroup = "SyntaxLslComment2Sided";	
+	}
 
-			// second word may be a right delimiter
-			std::string delimiter;
-			if (cur_type == LLKeywordToken::TWO_SIDED_DELIMITER)
+	return LLUIColorTable::instance().getColor(ColourGroup);
+}
+
+bool LLKeywords::initialise(LLSD SyntaxXML)
+{
+	mSyntax = SyntaxXML;
+	mLoaded = true;
+	return mLoaded;
+}
+
+///**
+// * @brief Start processing the colour LLSD from its beginning.
+// *
+// */
+//std::string LLKeywords::processColors()
+//{
+//	return processColors(mColors, "");
+//}
+
+///**
+// * @brief	Recursively process the colour LLSD from an arbitrary level.
+// * @desc	Process the supplied LLSD for colour data. The strPrefix is a string
+// *			of hyphen separated keys from previous levels.
+// */
+//std::string LLKeywords::processColors(LLSD &settings, const std::string strPrefix)
+//{
+//	if (settings.isMap() || (! settings.isMap() && strPrefix != "") )
+//	{
+//		LLSD llsd_map = settings;
+
+//		LLSD::map_iterator my_iter = llsd_map.beginMap();
+//		for ( ; my_iter != llsd_map.endMap(); ++my_iter)
+//		{
+//			std::string strGroup = strPrefix;
+//			const LLSD::String& key = my_iter->first;
+//			LLSD& value = my_iter->second;
+//
+//			if (key == "color")
+//			{
+//				if (value.isMap() || value.isArray())
+//				{
+//					addColorGroup(strGroup, readColor(value));
+//				}
+//				else
+//				{
+//					LL_WARNS("Invalid Color") << "Invalid Color Entry - first: '" << key << "' second: '" << value << "'" << LL_ENDL;
+//				}
+//			}
+//			else if (value.isMap())
+//			{
+//				strGroup += (strGroup.length() == 0) ? my_iter->first : "-" + my_iter->first;
+//				strGroup = processColors(value, strGroup);
+//			}
+//			else
+//			{
+//				LL_WARNS("Invalid Color") << "Invalid Color Entry - first: '" << key << "' second: '" << value << "'" << LL_ENDL;
+//			}
+//		}
+//	}
+//	return strPrefix;
+//}
+//*/
+
+void LLKeywords::processTokens()
+{
+	// Add 'standard' stuff: Quotes, Comments, Strings, Labels, etc. before processing the LLSD
+	std::string delimiter;
+	addToken(LLKeywordToken::TT_LABEL, "@", getColorGroup("misc-flow-label"), "Label\nTarget for jump statement", delimiter );
+	addToken(LLKeywordToken::TT_ONE_SIDED_DELIMITER, "//", getColorGroup("misc-comments_1_sided"), "Comment (single-line)\nNon-functional commentary or disabled code", delimiter );
+	addToken(LLKeywordToken::TT_TWO_SIDED_DELIMITER, "/*", getColorGroup("misc-comments_2_sided"), "Comment (multi-line)\nNon-functional commentary or disabled code", "*/" );
+	addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "\"", getColorGroup("misc-double_quotation_marks"), "String literal", "\"" );
+
+	LLSD::map_iterator outerIt = mSyntax.beginMap();
+	for ( ; outerIt != mSyntax.endMap(); ++outerIt)
+	{
+		if (outerIt->first == "misc")
+		{
+			if (outerIt->second.isMap())
 			{
-				while (delimiter.length() == 0 && ++token_word_iter != word_tokens.end())
+				LLSD::map_iterator innerIt = outerIt->second.beginMap();
+				for ( ; innerIt != outerIt->second.endMap(); ++innerIt)
 				{
-					delimiter = *token_word_iter;
-					LLStringUtil::trim(delimiter);
+					processTokensGroup(innerIt->second, "misc-" + innerIt->first);
 				}
 			}
-			else if (cur_type == LLKeywordToken::DOUBLE_QUOTATION_MARKS)
-			{
-				// Closing delimiter is identical to the opening one.
-				delimiter = keyword;
-			}
-
-			// following words are tooltip
-			std::string tool_tip;
-			while (++token_word_iter != word_tokens.end())
+			else
 			{
-				tool_tip += (*token_word_iter);
+				LL_ERRS("LSL-Tokens-Processing") << "Map for misc entries is missing! Ignoring." << LL_ENDL;
 			}
-			LLStringUtil::trim(tool_tip);
-			
-			if( !tool_tip.empty() )
+		}
+		else
+		{
+			if (outerIt->second.isMap())
 			{
-				// Replace : with \n for multi-line tool tips.
-				LLStringUtil::replaceChar( tool_tip, ':', '\n' );
-				addToken(cur_type, keyword, cur_color, tool_tip, delimiter );
+				processTokensGroup(outerIt->second, outerIt->first);
 			}
 			else
 			{
-				addToken(cur_type, keyword, cur_color, LLStringUtil::null, delimiter );
+				LL_ERRS("LSL-Tokens-Processing") << "Map for " + outerIt->first + " entries is missing! Ignoring." << LL_ENDL;
 			}
 		}
 	}
-
-	file.close();
-
-	mLoaded = TRUE;
-	return mLoaded;
+	LL_INFOS("") << LL_ENDL;
 }
 
-// Add the token as described
-void LLKeywords::addToken(LLKeywordToken::TOKEN_TYPE type,
-						  const std::string& key_in,
-						  const LLColor3& color,
-						  const std::string& tool_tip_in,
-						  const std::string& delimiter_in)
+void LLKeywords::processTokensGroup(LLSD& Tokens, const std::string Group)
 {
-	LLWString key = utf8str_to_wstring(key_in);
-	LLWString tool_tip = utf8str_to_wstring(tool_tip_in);
-	LLWString delimiter = utf8str_to_wstring(delimiter_in);
-	switch(type)
+	LLColor4 Color;
+	LL_INFOS("Tokens") << "Group: '" << Group << "', using colour: '" << Color << "'" << LL_ENDL;
+
+	LLKeywordToken::TOKEN_TYPE token_type = LLKeywordToken::TT_UNKNOWN;
+	// If a new token type is added here, it must also be added to the 'addToken' method
+	if (Group == "constants")
 	{
-	case LLKeywordToken::WORD:
-		mWordTokenMap[key] = new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null);
-		break;
+		token_type = LLKeywordToken::TT_CONSTANT;
+	}
+	else if (Group == "controls")
+	{
+		token_type = LLKeywordToken::TT_CONTROL;
+	}
+	else if (Group == "events")
+	{
+		token_type = LLKeywordToken::TT_EVENT;
+	}
+	else if (Group == "functions")
+	{
+		token_type = LLKeywordToken::TT_FUNCTION;
+	}
+	else if (Group == "label")
+	{
+		token_type = LLKeywordToken::TT_LABEL;
+	}
+	else if (Group == "types")
+	{
+		token_type = LLKeywordToken::TT_TYPE;
+	}
 
-	case LLKeywordToken::LINE:
-		mLineTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, LLWStringUtil::null));
-		break;
+	if (Tokens.isMap())
+	{
+		LLSD::map_iterator outerIt = Tokens.beginMap();
+		for ( ; outerIt != Tokens.endMap(); ++outerIt)
+		{
+			Color = getColorGroup(Group);
+			if (outerIt->second.isMap())
+			{
+				mAttributes.clear();
+				bool deprecated = false;
+				LLSD arguments = LLSD ();
+				LLSD::map_iterator innerIt = outerIt->second.beginMap();
+				for ( ; innerIt != outerIt->second.endMap(); ++innerIt)
+				{
+					if (innerIt->first == "arguments")
+					{ 
+						if (innerIt->second.isMap())
+						{
+							arguments = innerIt->second;
+						}
+					}
+					else
+					{
+						mAttributes[innerIt->first] = innerIt->second.asString();
+					}
+				}
 
-	case LLKeywordToken::TWO_SIDED_DELIMITER:
-	case LLKeywordToken::DOUBLE_QUOTATION_MARKS:
-	case LLKeywordToken::ONE_SIDED_DELIMITER:
-		mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
-		break;
+				std::string tooltip = "";
+				if (token_type == LLKeywordToken::TT_CONSTANT)
+				{
+					Color = getColorGroup(Group + "-" + getAttribute("type"));
+					tooltip = "Type: " + getAttribute("type") + ", Value: " + getAttribute("value");
+				}
+				else if (token_type == LLKeywordToken::TT_EVENT)
+				{
+					tooltip = outerIt->first + "(" + getArguments(arguments) + ")";
+				}
+				else if (token_type == LLKeywordToken::TT_FUNCTION)
+				{
+					tooltip = getAttribute("return") + " " + outerIt->first + "(" + getArguments(arguments) + ");";
+					tooltip += "\nEnergy: ";
+					tooltip += getAttribute("energy") == "" ? "0.0" : getAttribute("energy");
+					if (getAttribute("sleep") != "")
+					{
+						tooltip += ", Sleep: " + getAttribute("sleep");
+					}
+				}
 
-	default:
-		llassert(0);
+				if (getAttribute("tooltip") != "")
+				{
+					if (tooltip != "")
+					{
+						tooltip += "\n";
+					}
+					tooltip += getAttribute("tooltip");
+				}
+
+				deprecated = getAttribute("deprecated") == "true" ? true : false;
+				if (deprecated)
+				{
+					Color = getColorGroup("deprecated");
+				}
+				addToken(token_type, outerIt->first, Color, tooltip);
+			}
+		}
+	}
+	else if (Tokens.isArray())	// Currently nothing should need this, but it's here for completeness
+	{
+		for (int count = 0; count < Tokens.size(); ++count)
+		{
+			addToken(token_type, Tokens[count], Color, "");
+		}
+	}
+	else
+	{
+		LL_INFOS("Tokens") << "Invalid map/array passed: '" << Tokens << "'" << LL_ENDL;
 	}
 }
+
 LLKeywords::WStringMapIndex::WStringMapIndex(const WStringMapIndex& other)
 {
 	if(other.mOwner)
@@ -298,13 +455,13 @@ bool LLKeywords::WStringMapIndex::operator<(const LLKeywords::WStringMapIndex &o
 {
 	// NOTE: Since this is only used to organize a std::map, it doesn't matter if it uses correct collate order or not.
 	// The comparison only needs to strictly order all possible strings, and be stable.
-	
+
 	bool result = false;
 	const llwchar* self_iter = mData;
 	const llwchar* self_end = mData + mLength;
 	const llwchar* other_iter = other.mData;
 	const llwchar* other_end = other.mData + other.mLength;
-	
+
 	while(true)
 	{
 		if(other_iter >= other_end)
@@ -319,7 +476,7 @@ bool LLKeywords::WStringMapIndex::operator<(const LLKeywords::WStringMapIndex &o
 		{
 			// self is shorter than other.
 			result = true;
-			break; 
+			break;
 		}
 		else if(*self_iter != *other_iter)
 		{
@@ -331,11 +488,11 @@ bool LLKeywords::WStringMapIndex::operator<(const LLKeywords::WStringMapIndex &o
 		self_iter++;
 		other_iter++;
 	}
-	
+
 	return result;
 }
 
-LLColor3 LLKeywords::readColor( const std::string& s )
+LLColor4 LLKeywords::readColor( const std::string& s )
 {
 	F32 r, g, b;
 	r = g = b = 0.0f;
@@ -344,7 +501,23 @@ LLColor3 LLKeywords::readColor( const std::string& s )
 	{
 		llinfos << " poorly formed color in keyword file" << llendl;
 	}
-	return LLColor3( r, g, b );
+	return LLColor4( r, g, b, 1.f);
+}
+
+LLColor4 LLKeywords::readColor(LLSD& sd)
+{
+	if (sd.isArray())
+	{
+		return LLColor4 (sd, 1.f);
+	}
+	else if (sd.isMap())
+	{
+		return LLColor4 ( sd.get("x").asReal(), sd.get("y").asReal(), sd.get("z").asReal(), 1.f );
+	}
+	else
+	{
+		return LLColor4::black;
+	}
 }
 
 LLFastTimer::DeclareTimer FTM_SYNTAX_COLORING("Syntax Coloring");
@@ -360,10 +533,10 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 	{
 		return;
 	}
-	
+
 	S32 text_len = wtext.size() + 1;
 
-	seg_list->push_back( new LLNormalTextSegment( defaultColor, 0, text_len, editor ) ); 
+	seg_list->push_back( new LLNormalTextSegment( defaultColor, 0, text_len, editor ) );
 
 	const llwchar* base = wtext.c_str();
 	const llwchar* cur = base;
@@ -393,7 +566,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 				continue;
 			}
 
-			// cur is now at the first non-whitespace character of a new line	
+			// cur is now at the first non-whitespace character of a new line
 
 			// Line start tokens
 			{
@@ -411,7 +584,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 							cur++;
 						}
 						S32 seg_end = cur - base;
-						
+
 						//create segments from seg_start to seg_end
 						insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, defaultColor, editor);
 						line_done = TRUE; // to break out of second loop.
@@ -456,14 +629,14 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 
 					seg_start = cur - base;
 					cur += cur_delimiter->getLengthHead();
-					
+
 					LLKeywordToken::TOKEN_TYPE type = cur_delimiter->getType();
-					if( type == LLKeywordToken::TWO_SIDED_DELIMITER || type == LLKeywordToken::DOUBLE_QUOTATION_MARKS )
+					if( type == LLKeywordToken::TT_TWO_SIDED_DELIMITER || type == LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS )
 					{
 						while( *cur && !cur_delimiter->isTail(cur))
 						{
 							// Check for an escape sequence.
-							if (type == LLKeywordToken::DOUBLE_QUOTATION_MARKS && *cur == '\\')
+							if (type == LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS && *cur == '\\')
 							{
 								// Count the number of backslashes.
 								S32 num_backslashes = 0;
@@ -510,7 +683,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 					}
 					else
 					{
-						llassert( cur_delimiter->getType() == LLKeywordToken::ONE_SIDED_DELIMITER );
+						llassert( cur_delimiter->getType() == LLKeywordToken::TT_ONE_SIDED_DELIMITER );
 						// Left side is the delimiter.  Right side is eol or eof.
 						while( *cur && ('\n' != *cur) )
 						{
@@ -556,7 +729,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 
 						insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, defaultColor, editor);
 					}
-					cur += seg_len; 
+					cur += seg_len;
 					continue;
 				}
 			}
@@ -572,7 +745,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 void LLKeywords::insertSegments(const LLWString& wtext, std::vector<LLTextSegmentPtr>& seg_list, LLKeywordToken* cur_token, S32 text_len, S32 seg_start, S32 seg_end, const LLColor4 &defaultColor, LLTextEditor& editor )
 {
 	std::string::size_type pos = wtext.find('\n',seg_start);
-	
+
 	while (pos!=-1 && pos < (std::string::size_type)seg_end)
 	{
 		if (pos!=seg_start)
@@ -651,7 +824,7 @@ void LLKeywords::dump()
 
 void LLKeywordToken::dump()
 {
-	llinfos << "[" << 
+	llinfos << "[" <<
 		mColor.mV[VX] << ", " <<
 		mColor.mV[VY] << ", " <<
 		mColor.mV[VZ] << "] [" <<
diff --git a/indra/llui/llkeywords.h b/indra/llui/llkeywords.h
index ac34015393..074cedfc63 100755
--- a/indra/llui/llkeywords.h
+++ b/indra/llui/llkeywords.h
@@ -1,25 +1,25 @@
-/** 
+/**
  * @file llkeywords.h
  * @brief Keyword list for LSL
  *
  * $LicenseInfo:firstyear=2001&license=viewerlgpl$
  * Second Life Viewer Source Code
  * Copyright (C) 2010, Linden Research, Inc.
- * 
+ *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation;
  * version 2.1 of the License only.
- * 
+ *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
+ *
  * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  * $/LicenseInfo$
  */
@@ -28,8 +28,10 @@
 #define LL_LLKEYWORDS_H
 
 
+#include "lldir.h"
 #include "llstring.h"
 #include "v3color.h"
+#include "v4color.h"
 #include <map>
 #include <list>
 #include <deque>
@@ -41,26 +43,35 @@ typedef LLPointer<LLTextSegment> LLTextSegmentPtr;
 class LLKeywordToken
 {
 public:
-	/** 
+	/**
 	 * @brief Types of tokens/delimters being parsed.
 	 *
 	 * @desc Tokens/delimiters that need to be identified/highlighted. All are terminated if an EOF is encountered.
-	 * - WORD are keywords in the normal sense, i.e. constants, events, etc.
-	 * - LINE are for entire lines (currently only flow control labels use this).
-	 * - ONE_SIDED_DELIMITER are for open-ended delimiters which are terminated by EOL.
-	 * - TWO_SIDED_DELIMITER are for delimiters that end with a different delimiter than they open with.
-	 * - DOUBLE_QUOTATION_MARKS are for delimiting areas using the same delimiter to open and close.
+	 * - TT_WORD are keywords in the normal sense, i.e. constants, events, etc.
+	 * - TT_LINE are for entire lines (currently only flow control labels use this).
+	 * - TT_ONE_SIDED_DELIMITER are for open-ended delimiters which are terminated by EOL.
+	 * - TT_TWO_SIDED_DELIMITER are for delimiters that end with a different delimiter than they open with.
+	 * - TT_DOUBLE_QUOTATION_MARKS are for delimiting areas using the same delimiter to open and close.
 	 */
 	enum TOKEN_TYPE
 	{
-		WORD,
-		LINE,
-		TWO_SIDED_DELIMITER,
-		ONE_SIDED_DELIMITER,
-		DOUBLE_QUOTATION_MARKS
+		TT_UNKNOWN,
+		TT_WORD,
+		TT_LINE,
+		TT_TWO_SIDED_DELIMITER,
+		TT_ONE_SIDED_DELIMITER,
+		TT_DOUBLE_QUOTATION_MARKS,
+		// Following constants are more specific versions of the preceding ones
+		TT_CONSTANT,						// WORD
+		TT_CONTROL,							// WORD
+		TT_EVENT,							// WORD
+		TT_FUNCTION,						// WORD
+		TT_LABEL,							// LINE
+		TT_SECTION,							// WORD
+		TT_TYPE								// WORD
 	};
 
-	LLKeywordToken( TOKEN_TYPE type, const LLColor3& color, const LLWString& token, const LLWString& tool_tip, const LLWString& delimiter  ) 
+	LLKeywordToken( TOKEN_TYPE type, const LLColor4& color, const LLWString& token, const LLWString& tool_tip, const LLWString& delimiter  )
 		:
 		mType( type ),
 		mToken( token ),
@@ -75,7 +86,7 @@ public:
 	BOOL				isHead(const llwchar* s) const;
 	BOOL				isTail(const llwchar* s) const;
 	const LLWString&	getToken() const		{ return mToken; }
-	const LLColor3&		getColor() const		{ return mColor; }
+	const LLColor4&		getColor() const		{ return mColor; }
 	TOKEN_TYPE			getType()  const		{ return mType; }
 	const LLWString&	getToolTip() const		{ return mToolTip; }
 	const LLWString&	getDelimiter() const	{ return mDelimiter; }
@@ -87,7 +98,7 @@ public:
 private:
 	TOKEN_TYPE	mType;
 	LLWString	mToken;
-	LLColor3	mColor;
+	LLColor4	mColor;
 	LLWString	mToolTip;
 	LLWString	mDelimiter;
 };
@@ -98,18 +109,26 @@ public:
 	LLKeywords();
 	~LLKeywords();
 
+	void		addColorGroup(const std::string key_in, const LLColor4 color);
+	void		clearLoaded() { mLoaded = false; }
+	LLColor4	getColorGroup(const std::string key_in);
+	BOOL		loadFromFile();
 	BOOL		loadFromFile(const std::string& filename);
-	BOOL		isLoaded() const	{ return mLoaded; }
+	bool		isLoaded() const	{ return mLoaded; }
 
 	void		findSegments(std::vector<LLTextSegmentPtr> *seg_list, const LLWString& text, const LLColor4 &defaultColor, class LLTextEditor& editor );
+	bool		initialise(LLSD SyntaxXML);
+	std::string	processColors();
+	std::string	processColors(LLSD &data, const std::string strGroup);
+	void		processTokens();
 
 	// Add the token as described
 	void addToken(LLKeywordToken::TOKEN_TYPE type,
 					const std::string& key,
-					const LLColor3& color,
+					const LLColor4& color,
 					const std::string& tool_tip = LLStringUtil::null,
 					const std::string& delimiter = LLStringUtil::null);
-	
+
 	// This class is here as a performance optimization.
 	// The word token map used to be defined as std::map<LLWString, LLKeywordToken*>.
 	// This worked, but caused a performance bottleneck due to memory allocation and string copies
@@ -133,6 +152,9 @@ public:
 		const llwchar *mData;
 		size_t mLength;
 		bool mOwner;
+
+
+		LLColor4			mColor;
 	};
 
 	typedef std::map<WStringMapIndex, LLKeywordToken*> word_token_map_t;
@@ -140,20 +162,38 @@ public:
 	keyword_iterator_t begin() const { return mWordTokenMap.begin(); }
 	keyword_iterator_t end() const { return mWordTokenMap.end(); }
 
+	typedef std::map<WStringMapIndex, LLColor4> group_color_map_t;
+	typedef group_color_map_t::const_iterator color_iterator_t;
+	group_color_map_t	mColorGroupMap;
+
 #ifdef _DEBUG
 	void		dump();
 #endif
 
-private:
-	LLColor3	readColor(const std::string& s);
+protected:
+	void		processTokensGroup(LLSD& Tokens, const std::string Group);
+	LLColor4	readColor(const std::string& s);
+	LLColor4	readColor(LLSD& sd);
 	void		insertSegment(std::vector<LLTextSegmentPtr>& seg_list, LLTextSegmentPtr new_segment, S32 text_len, const LLColor4 &defaultColor, class LLTextEditor& editor);
 	void		insertSegments(const LLWString& wtext, std::vector<LLTextSegmentPtr>& seg_list, LLKeywordToken* token, S32 text_len, S32 seg_start, S32 seg_end, const LLColor4 &defaultColor, LLTextEditor& editor);
+	BOOL		loadIntoLLSD( const std::string& filename, LLSD& data );
 
-	BOOL		mLoaded;
+	LLSD		mColors;
+	bool		mLoaded;
+	LLSD		mSyntax;
 	word_token_map_t mWordTokenMap;
 	typedef std::deque<LLKeywordToken*> token_list_t;
 	token_list_t mLineTokenList;
 	token_list_t mDelimiterTokenList;
+
+	typedef  std::map<std::string, std::string> element_attributes_t;
+	typedef element_attributes_t::const_iterator attribute_iterator_t;
+	element_attributes_t mAttributes;
+	std::string	getAttribute(const std::string& key);
+
+	std::string	getArguments(LLSD& args);
+
+private:
 };
 
 #endif  // LL_LLKEYWORDS_H
diff --git a/indra/llui/lltexteditor.cpp b/indra/llui/lltexteditor.cpp
index 62140dd9d6..303bc48933 100755
--- a/indra/llui/lltexteditor.cpp
+++ b/indra/llui/lltexteditor.cpp
@@ -2513,29 +2513,19 @@ BOOL LLTextEditor::tryToRevertToPristineState()
 
 
 static LLFastTimer::DeclareTimer FTM_SYNTAX_HIGHLIGHTING("Syntax Highlighting");
-void LLTextEditor::loadKeywords(const std::string& filename,
-								const std::vector<std::string>& funcs,
-								const std::vector<std::string>& tooltips,
-								const LLColor3& color)
+void LLTextEditor::loadKeywords()
 {
 	LLFastTimer ft(FTM_SYNTAX_HIGHLIGHTING);
-	if(mKeywords.loadFromFile(filename))
-	{
-		S32 count = llmin(funcs.size(), tooltips.size());
-		for(S32 i = 0; i < count; i++)
-		{
-			std::string name = utf8str_trim(funcs[i]);
-			mKeywords.addToken(LLKeywordToken::WORD, name, color, tooltips[i] );
-		}
-		segment_vec_t segment_list;
-		mKeywords.findSegments(&segment_list, getWText(), mDefaultColor.get(), *this);
+	mKeywords.processTokens();
 
-		mSegments.clear();
-		segment_set_t::iterator insert_it = mSegments.begin();
-		for (segment_vec_t::iterator list_it = segment_list.begin(); list_it != segment_list.end(); ++list_it)
-		{
-			insert_it = mSegments.insert(insert_it, *list_it);
-		}
+	segment_vec_t segment_list;
+	mKeywords.findSegments(&segment_list, getWText(), mDefaultColor.get(), *this);
+
+	mSegments.clear();
+	segment_set_t::iterator insert_it = mSegments.begin();
+	for (segment_vec_t::iterator list_it = segment_list.begin(); list_it != segment_list.end(); ++list_it)
+	{
+		insert_it = mSegments.insert(insert_it, *list_it);
 	}
 }
 
diff --git a/indra/llui/lltexteditor.h b/indra/llui/lltexteditor.h
index d3b7bc0eb7..d7f6c7f643 100755
--- a/indra/llui/lltexteditor.h
+++ b/indra/llui/lltexteditor.h
@@ -1,25 +1,25 @@
-/** 
+/**
  * @file lltexteditor.h
  * @brief LLTextEditor base class
  *
  * $LicenseInfo:firstyear=2001&license=viewerlgpl$
  * Second Life Viewer Source Code
  * Copyright (C) 2010, Linden Research, Inc.
- * 
+ *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation;
  * version 2.1 of the License only.
- * 
+ *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- * 
+ *
  * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA
  * $/LicenseInfo$
  */
@@ -133,7 +133,7 @@ public:
 	virtual BOOL	canCopy() const;
 	virtual void	paste();
 	virtual BOOL	canPaste() const;
- 
+
 	virtual void	updatePrimary();
 	virtual void	copyPrimary();
 	virtual void	pastePrimary();
@@ -151,7 +151,7 @@ public:
 	void			selectNext(const std::string& search_text_in, BOOL case_insensitive, BOOL wrap = TRUE);
 	BOOL			replaceText(const std::string& search_text, const std::string& replace_text, BOOL case_insensitive, BOOL wrap = TRUE);
 	void			replaceTextAll(const std::string& search_text, const std::string& replace_text, BOOL case_insensitive);
-	
+
 	// Undo/redo stack
 	void			blockUndo();
 
@@ -188,13 +188,14 @@ public:
 
 	void			getCurrentLineAndColumn( S32* line, S32* col, BOOL include_wordwrap );
 
-	void			loadKeywords(const std::string& filename,
-								 const std::vector<std::string>& funcs,
-								 const std::vector<std::string>& tooltips,
-								 const LLColor3& func_color);
+	LLKeywords		mKeywords;
+	void			loadKeywords();
 	LLKeywords::keyword_iterator_t keywordsBegin()	{ return mKeywords.begin(); }
 	LLKeywords::keyword_iterator_t keywordsEnd()	{ return mKeywords.end(); }
 
+	void			loadKeywords(const std::string& filename_keywords,
+								const std::string& filename_colors);
+
 	// Hacky methods to make it into a word-wrapping, potentially scrolling,
 	// read-only text box.
 	void			setCommitOnFocusLost(BOOL b)			{ mCommitOnFocusLost = b; }
@@ -218,7 +219,7 @@ protected:
 	void			drawPreeditMarker();
 
 	void 			assignEmbedded(const std::string &s);
-	
+
 	void			removeCharOrTab();
 
 	void			indentSelectedLines( S32 spaces );
@@ -237,12 +238,12 @@ protected:
 	S32				nextWordPos(S32 cursorPos) const;
 
 	void			autoIndent();
-	
+
 	void			findEmbeddedItemSegments(S32 start, S32 end);
 	void			getSegmentsInRange(segment_vec_t& segments, S32 start, S32 end, bool include_partial) const;
 
 	virtual llwchar	pasteEmbeddedItem(llwchar ext_char) { return ext_char; }
-	
+
 
 	// Here's the method that takes and applies text commands.
 	S32 			execute(TextCmd* cmd);
@@ -256,7 +257,7 @@ protected:
 	S32 			removeChar(S32 pos);
 	S32				insert(S32 pos, const LLWString &wstr, bool group_with_next_op, LLTextSegmentPtr segment);
 	S32				remove(S32 pos, S32 length, bool group_with_next_op);
-	
+
 	void			focusLostHelper();
 	void			updateAllowingLanguageInput();
 	BOOL			hasPreeditString() const;
@@ -274,14 +275,14 @@ protected:
 	//
 	// Protected data
 	//
-	// Probably deserves serious thought to hiding as many of these 
+	// Probably deserves serious thought to hiding as many of these
 	// as possible behind protected accessor methods.
 	//
 
 	// Use these to determine if a click on an embedded item is a drag or not.
 	S32				mMouseDownX;
 	S32				mMouseDownY;
-	
+
 	LLWString			mPreeditWString;
 	LLWString			mPreeditOverwrittenWString;
 	std::vector<S32> 	mPreeditPositions;
@@ -308,11 +309,6 @@ private:
 
 	void			onKeyStroke();
 
-	//
-	// Data
-	//
-	LLKeywords		mKeywords;
-
 	// Concrete TextCmd sub-classes used by the LLTextEditor base class
 	class TextCmdInsert;
 	class TextCmdAddChar;