Merge pull request #5235 from secondlife/harold/speed_up_slua_highlighting

Simplify `--[[` Lua comment syntax highlighting, speed up SLua highlighting

2 files changed, 76 insertions, 175 deletions

diff --git a/indra/llui/llkeywords.cpp b/indra/llui/llkeywords.cpp
index aa5aceddfa..aeabf43021 100644
--- a/indra/llui/llkeywords.cpp
+++ b/indra/llui/llkeywords.cpp
@@ -28,7 +28,6 @@
 
 #include <iostream>
 #include <fstream>
-#include <regex>
 
 #include "llkeywords.h"
 #include "llsdserialize.h"
@@ -63,8 +62,6 @@ LLKeywords::~LLKeywords()
     mLineTokenList.clear();
     std::for_each(mDelimiterTokenList.begin(), mDelimiterTokenList.end(), DeletePointer());
     mDelimiterTokenList.clear();
-    std::for_each(mRegexTokenList.begin(), mRegexTokenList.end(), DeletePointer());
-    mRegexTokenList.clear();
 }
 
 // Add the token as described
@@ -105,13 +102,10 @@ void LLKeywords::addToken(LLKeywordToken::ETokenType type,
     case LLKeywordToken::TT_TWO_SIDED_DELIMITER:
     case LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS:
     case LLKeywordToken::TT_ONE_SIDED_DELIMITER:
+    case LLKeywordToken::TT_LONG_BRACKET:
         mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
         break;
 
-    case LLKeywordToken::TT_REGEX_MATCH:
-        mRegexTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter));
-        break;
-
     default:
         llassert(0);
     }
@@ -229,13 +223,14 @@ void LLKeywords::processTokens()
     if (mLuauLanguage)
     {
         addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "\'", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal", "\'");
+        // TODO: Might be nice to add a special case for this so we can still highlight expressions in `{}`s
         addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "`", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal", "`");
         // Add Lua-style comments
         addToken(LLKeywordToken::TT_ONE_SIDED_DELIMITER, "--", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style single-line)\nNon-functional commentary or disabled code", delimiter);
-        // Add Lua multi-line comments
-        addToken(LLKeywordToken::TT_REGEX_MATCH, "^--\\[(=*)\\[", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style multi-line)\nNon-functional commentary or disabled code", "\\]\\1\\]");
-        // Add Lua multi-line strings
-        addToken(LLKeywordToken::TT_REGEX_MATCH, "^\\[(=*)\\[", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal (Lua-style multi-line)", "\\]\\1\\]");
+        // Add Lua multi-line comments (long brackets)
+        addToken(LLKeywordToken::TT_LONG_BRACKET, "--[", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style multi-line)\nNon-functional commentary or disabled code", delimiter);
+        // Add Lua multi-line strings (long brackets)
+        addToken(LLKeywordToken::TT_LONG_BRACKET, "[", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal (Lua-style multi-line)", delimiter);
     }
     else
     {
@@ -265,20 +260,6 @@ void LLKeywords::processTokens()
         }
     }
 
-    // Pre-compile all regex patterns for tokens in mRegexTokenList
-    for (LLKeywordToken* regex_token : mRegexTokenList)
-    {
-        std::string start_pattern(regex_token->getToken().begin(), regex_token->getToken().end());
-        try
-        {
-            regex_token->setCompiledRegex(new std::regex(start_pattern));
-        }
-        catch (const std::regex_error& e)
-        {
-            LL_WARNS() << "Regex error in start pattern: " << e.what() << " in pattern: " << start_pattern << LL_ENDL;
-        }
-    }
-
     LL_INFOS("SyntaxLSL") << "Finished processing tokens." << LL_ENDL;
 }
 
@@ -522,10 +503,6 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 
     seg_list->push_back( new LLNormalTextSegment( style, 0, text_len, editor ) );
 
-    std::string text_to_search;
-    text_to_search.reserve(wtext.size());
-
-    bool  has_regex = !mRegexTokenList.empty();
     auto& delimiters = mDelimiterTokenList;
 
     const llwchar* base = wtext.c_str();
@@ -608,151 +585,98 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
 
         while( *cur && *cur != '\n' )
         {
-            // Check for regex matches first
-            bool regex_matched = false;
-            if (has_regex)
+            // Check against delimiters
             {
-                S32 seg_start = (S32)(cur - base);
-
-                text_to_search.assign(wtext.begin() + seg_start, wtext.end());
+                S32 seg_start = 0;
+                LLKeywordToken* cur_delimiter = NULL;
+                for (auto* delimiter : delimiters)
+                {
+                    if( delimiter->isHead( cur ) )
+                    {
+                        cur_delimiter = delimiter;
+                        break;
+                    }
+                }
 
-                for (LLKeywordToken* regex_token : mRegexTokenList)
+                if( cur_delimiter )
                 {
-                    std::regex* compiled_regex = regex_token->getCompiledRegex();
+                    S32 between_delimiters = 0;
+                    S32 seg_end = 0;
+
+                    seg_start = (S32)(cur - base);
+
+                    LLKeywordToken::ETokenType type = cur_delimiter->getType();
 
-                    // If we have a pre-compiled regex, use it
-                    if (compiled_regex)
+                    // Handle Lua long brackets specially - need to verify full pattern
+                    if (type == LLKeywordToken::TT_LONG_BRACKET)
                     {
-                        std::string end_pattern(regex_token->getDelimiter().begin(), regex_token->getDelimiter().end());
+                        const llwchar* p = cur + cur_delimiter->getLengthHead();  // after --[ or [
+
+                        // Count equals signs
+                        S32 level = 0;
+                        while (*p == '=')
+                        {
+                            level++;
+                            p++;
+                        }
 
-                        try
+                        // Must have second [
+                        if (*p == '[')
                         {
-                            std::smatch start_match;
+                            p++;  // skip the second [
+                            const llwchar* content_start = p;
 
-                            if (std::regex_search(text_to_search, start_match, *compiled_regex) && !start_match.empty())
+                            // Build the closing pattern: ] + level equals + ]
+                            // Search for it in the remaining text
+                            while (*p)
                             {
-                                if (start_match.position() == 0) // Match starts at current position
+                                if (*p == ']')
                                 {
-                                    // Calculate segment boundaries for start pattern
-                                    S32 start_match_length = static_cast<S32>(start_match.str().length());
-                                    S32 start_seg_end = seg_start + start_match_length;
-
-                                    if (end_pattern.empty())
+                                    // Check if this is our closing bracket
+                                    const llwchar* close_check = p + 1;
+                                    S32 close_equals = 0;
+                                    while (*close_check == '=')
                                     {
-                                        // If no end pattern is provided, treat the entire regex match as a single segment
-                                        // Move cursor past the matched segment
-                                        cur = base + start_seg_end;
-
-                                        // Insert the matched segment
-                                        insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, start_seg_end, style, editor);
+                                        close_equals++;
+                                        close_check++;
                                     }
-                                    else
-                                    {   // TODO: better optimization for this part
-
-                                        // Look for the end pattern after the start pattern
-                                        std::string remaining_text = text_to_search.substr(start_match_length);
-
-                                        // Process end pattern - replace any capture group references
-                                        std::string actual_end_pattern = end_pattern;
-
-                                        // Handle capture groups in the end pattern (replace \1, \2, etc. with their matched content)
-                                        for (size_t i = 1; i < start_match.size(); ++i)
-                                        {
-                                            std::string capture = start_match[i].str();
-                                            std::string placeholder = "\\" + std::to_string(i);
-
-                                            // Replace all occurrences of the placeholder with the captured content
-                                            size_t pos = 0;
-                                            while ((pos = actual_end_pattern.find(placeholder, pos)) != std::string::npos)
-                                            {
-                                                actual_end_pattern.replace(pos, placeholder.length(), capture);
-                                                pos += capture.length();
-                                            }
-                                        }
-
-                                        try
-                                        {
-                                            std::regex end_regex_pattern(actual_end_pattern);
-                                            std::smatch end_match;
-
-                                            S32 seg_end = start_seg_end;
-
-                                            if (std::regex_search(remaining_text, end_match, end_regex_pattern) && !end_match.empty())
-                                            {
-                                                // Calculate position of end match relative to the original text
-                                                S32 end_match_position = static_cast<S32>(end_match.position());
-                                                S32 end_match_length = static_cast<S32>(end_match.str().length());
-
-                                                // Calculate the total length including both patterns and text between
-                                                seg_end += end_match_position + end_match_length;
-                                            }
-                                            else
-                                            {
-                                                // End pattern not found, treat everything up to EOF as the segment
-                                                seg_end += static_cast<S32>(remaining_text.length());
-                                            }
-
-                                            // Move cursor past the entire matched segment (start + content + end)
-                                            cur = base + seg_end;
-
-                                            // Insert the matched segment
-                                            insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, seg_end, style, editor);
-                                        }
-                                        catch (const std::regex_error& e)
-                                        {
-                                            LL_WARNS() << "Regex error in end pattern: " << e.what() << " in pattern: " << actual_end_pattern << LL_ENDL;
-                                            // Fall back to treating the start match as the entire segment
-                                            cur = base + start_seg_end;
-                                            insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, start_seg_end, style, editor);
-                                        }
+                                    if (close_equals == level && *close_check == ']')
+                                    {
+                                        // Found the matching close
+                                        seg_end = (S32)(close_check + 1 - base);
+                                        cur = close_check + 1;
+                                        insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor);
+                                        break;
                                     }
-
-                                    regex_matched = true;
-                                    break;
                                 }
+                                p++;
+                            }
+
+                            if (!*p)
+                            {
+                                // No closing found, highlight to end of file
+                                seg_end = text_len - 1;
+                                cur = base + seg_end;
+                                insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor);
                             }
+                            continue;
                         }
-                        catch (const std::regex_error& e)
+                        else
                         {
-                            LL_WARNS() << "Error using compiled regex: " << e.what() << LL_ENDL;
+                            // Not a valid long bracket (e.g., --[abc), skip this delimiter
+                            cur_delimiter = NULL;
                         }
                     }
-                    else
-                    {
-                        // Skip tokens that aren't pre-compiled
-                        LL_WARNS() << "Skipping regex token due to missing pre-compiled pattern: "
-                                   << wstring_to_utf8str(regex_token->getToken()) << LL_ENDL;
-                    }
-                }
-
-                if (regex_matched)
-                {
-                    continue;
-                }
-            }
 
-            // Check against delimiters
-            {
-                S32 seg_start = 0;
-                LLKeywordToken* cur_delimiter = NULL;
-                for (auto* delimiter : delimiters)
-                {
-                    if( delimiter->isHead( cur ) )
+                    if (!cur_delimiter)
                     {
-                        cur_delimiter = delimiter;
-                        break;
+                        // Long bracket validation failed, continue to next character
+                        cur++;
+                        continue;
                     }
-                }
 
-                if( cur_delimiter )
-                {
-                    S32 between_delimiters = 0;
-                    S32 seg_end = 0;
-
-                    seg_start = (S32)(cur - base);
                     cur += cur_delimiter->getLengthHead();
 
-                    LLKeywordToken::ETokenType type = cur_delimiter->getType();
                     if(type == LLKeywordToken::TT_TWO_SIDED_DELIMITER || type == LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS)
                     {
                         while( *cur && !cur_delimiter->isTail(cur))
@@ -1009,14 +933,6 @@ void LLKeywords::dump()
         LLKeywordToken* delimiter_token = *iter;
         delimiter_token->dump();
     }
-
-    LL_INFOS() << "LLKeywords::sRegexTokenList" << LL_ENDL;
-    for (token_list_t::iterator iter = mRegexTokenList.begin();
-         iter != mRegexTokenList.end(); ++iter)
-    {
-        LLKeywordToken* regex_token = *iter;
-        regex_token->dump();
-    }
 }
 
 void LLKeywordToken::dump()
diff --git a/indra/llui/llkeywords.h b/indra/llui/llkeywords.h
index f5da50a053..9dbe26a66d 100644
--- a/indra/llui/llkeywords.h
+++ b/indra/llui/llkeywords.h
@@ -36,7 +36,6 @@
 #include <map>
 #include <list>
 #include <deque>
-#include <regex>
 #include "llpointer.h"
 
 class LLTextSegment;
@@ -54,11 +53,7 @@ public:
      * - TT_ONE_SIDED_DELIMITER are for open-ended delimiters which are terminated by EOL.
      * - TT_TWO_SIDED_DELIMITER are for delimiters that end with a different delimiter than they open with.
      * - TT_DOUBLE_QUOTATION_MARKS are for delimiting areas using the same delimiter to open and close.
-     * - TT_REGEX_MATCH are for pattern-based matching using regular expressions.
-     *      For TT_REGEX_MATCH: mToken contains the start pattern, mDelimiter contains the end pattern (if any).
-     *      If mDelimiter is empty, the entire match is considered one segment.
-     *      If mDelimiter contains capture group references (e.g. \1, \2), these will be replaced with
-     *      the corresponding capture groups from the start pattern match.
+     * - TT_LONG_BRACKET are for Lua tokens that use brackets with counted equals signs.
      */
     typedef enum e_token_type
     {
@@ -68,7 +63,7 @@ public:
         TT_TWO_SIDED_DELIMITER,
         TT_ONE_SIDED_DELIMITER,
         TT_DOUBLE_QUOTATION_MARKS,
-        TT_REGEX_MATCH,
+        TT_LONG_BRACKET,                    // Lua long brackets: --[=*[ or [=*[
         // Following constants are more specific versions of the preceding ones
         TT_CONSTANT,                        // WORD
         TT_CONTROL,                         // WORD
@@ -85,18 +80,12 @@ public:
         mToken( token ),
         mColor( color ),
         mToolTip( tool_tip ),
-        mDelimiter( delimiter ),     // right delimiter
-        mCompiledRegex( nullptr )
+        mDelimiter( delimiter )     // right delimiter
     {
     }
 
     ~LLKeywordToken()
     {
-        if (mCompiledRegex)
-        {
-            delete mCompiledRegex;
-            mCompiledRegex = nullptr;
-        }
     }
 
     S32                 getLengthHead() const   { return static_cast<S32>(mToken.size()); }
@@ -108,8 +97,6 @@ public:
     ETokenType          getType()  const        { return mType; }
     const LLWString&    getToolTip() const      { return mToolTip; }
     const LLWString&    getDelimiter() const    { return mDelimiter; }
-    std::regex*         getCompiledRegex() const { return mCompiledRegex; }
-    void                setCompiledRegex(std::regex* regex) { mCompiledRegex = regex; }
 
 #ifdef _DEBUG
     void        dump();
@@ -121,7 +108,6 @@ private:
     LLUIColor    mColor;
     LLWString   mToolTip;
     LLWString   mDelimiter;
-    std::regex* mCompiledRegex;
 };
 
 class LLKeywords
@@ -214,7 +200,6 @@ protected:
     typedef std::deque<LLKeywordToken*> token_list_t;
     token_list_t mLineTokenList;
     token_list_t mDelimiterTokenList;
-    token_list_t mRegexTokenList;
 
     typedef  std::map<std::string, std::string, std::less<>> element_attributes_t;
     typedef element_attributes_t::const_iterator attribute_iterator_t;