From 579f3f52f04ac544d4e222b94aaeb1b297ee69b4 Mon Sep 17 00:00:00 2001 From: Harold Cindy <120691094+HaroldCindy@users.noreply.github.com> Date: Wed, 7 Jan 2026 17:22:34 -0800 Subject: Simplify `--[[` Lua comment syntax highlighting The regex strategy was causing exponential slowdown as the file got larger, use our own state management where we have control over how lookahead works without intermediary allocs --- indra/llui/llkeywords.cpp | 230 +++++++++++++++------------------------------- indra/llui/llkeywords.h | 21 +---- 2 files changed, 76 insertions(+), 175 deletions(-) diff --git a/indra/llui/llkeywords.cpp b/indra/llui/llkeywords.cpp index aa5aceddfa..aeabf43021 100644 --- a/indra/llui/llkeywords.cpp +++ b/indra/llui/llkeywords.cpp @@ -28,7 +28,6 @@ #include #include -#include #include "llkeywords.h" #include "llsdserialize.h" @@ -63,8 +62,6 @@ LLKeywords::~LLKeywords() mLineTokenList.clear(); std::for_each(mDelimiterTokenList.begin(), mDelimiterTokenList.end(), DeletePointer()); mDelimiterTokenList.clear(); - std::for_each(mRegexTokenList.begin(), mRegexTokenList.end(), DeletePointer()); - mRegexTokenList.clear(); } // Add the token as described @@ -105,13 +102,10 @@ void LLKeywords::addToken(LLKeywordToken::ETokenType type, case LLKeywordToken::TT_TWO_SIDED_DELIMITER: case LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS: case LLKeywordToken::TT_ONE_SIDED_DELIMITER: + case LLKeywordToken::TT_LONG_BRACKET: mDelimiterTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter)); break; - case LLKeywordToken::TT_REGEX_MATCH: - mRegexTokenList.push_front(new LLKeywordToken(type, color, key, tool_tip, delimiter)); - break; - default: llassert(0); } @@ -229,13 +223,14 @@ void LLKeywords::processTokens() if (mLuauLanguage) { addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "\'", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal", "\'"); + // TODO: Might be nice to add a special case for this so we can still highlight expressions in `{}`s addToken(LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS, "`", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal", "`"); // Add Lua-style comments addToken(LLKeywordToken::TT_ONE_SIDED_DELIMITER, "--", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style single-line)\nNon-functional commentary or disabled code", delimiter); - // Add Lua multi-line comments - addToken(LLKeywordToken::TT_REGEX_MATCH, "^--\\[(=*)\\[", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style multi-line)\nNon-functional commentary or disabled code", "\\]\\1\\]"); - // Add Lua multi-line strings - addToken(LLKeywordToken::TT_REGEX_MATCH, "^\\[(=*)\\[", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal (Lua-style multi-line)", "\\]\\1\\]"); + // Add Lua multi-line comments (long brackets) + addToken(LLKeywordToken::TT_LONG_BRACKET, "--[", LLUIColorTable::instance().getColor("SyntaxLslComment"), "Comment (Lua-style multi-line)\nNon-functional commentary or disabled code", delimiter); + // Add Lua multi-line strings (long brackets) + addToken(LLKeywordToken::TT_LONG_BRACKET, "[", LLUIColorTable::instance().getColor("SyntaxLslStringLiteral"), "String literal (Lua-style multi-line)", delimiter); } else { @@ -265,20 +260,6 @@ void LLKeywords::processTokens() } } - // Pre-compile all regex patterns for tokens in mRegexTokenList - for (LLKeywordToken* regex_token : mRegexTokenList) - { - std::string start_pattern(regex_token->getToken().begin(), regex_token->getToken().end()); - try - { - regex_token->setCompiledRegex(new std::regex(start_pattern)); - } - catch (const std::regex_error& e) - { - LL_WARNS() << "Regex error in start pattern: " << e.what() << " in pattern: " << start_pattern << LL_ENDL; - } - } - LL_INFOS("SyntaxLSL") << "Finished processing tokens." << LL_ENDL; } @@ -522,10 +503,6 @@ void LLKeywords::findSegments(std::vector* seg_list, const LLW seg_list->push_back( new LLNormalTextSegment( style, 0, text_len, editor ) ); - std::string text_to_search; - text_to_search.reserve(wtext.size()); - - bool has_regex = !mRegexTokenList.empty(); auto& delimiters = mDelimiterTokenList; const llwchar* base = wtext.c_str(); @@ -608,151 +585,98 @@ void LLKeywords::findSegments(std::vector* seg_list, const LLW while( *cur && *cur != '\n' ) { - // Check for regex matches first - bool regex_matched = false; - if (has_regex) + // Check against delimiters { - S32 seg_start = (S32)(cur - base); - - text_to_search.assign(wtext.begin() + seg_start, wtext.end()); + S32 seg_start = 0; + LLKeywordToken* cur_delimiter = NULL; + for (auto* delimiter : delimiters) + { + if( delimiter->isHead( cur ) ) + { + cur_delimiter = delimiter; + break; + } + } - for (LLKeywordToken* regex_token : mRegexTokenList) + if( cur_delimiter ) { - std::regex* compiled_regex = regex_token->getCompiledRegex(); + S32 between_delimiters = 0; + S32 seg_end = 0; + + seg_start = (S32)(cur - base); - // If we have a pre-compiled regex, use it - if (compiled_regex) + LLKeywordToken::ETokenType type = cur_delimiter->getType(); + + // Handle Lua long brackets specially - need to verify full pattern + if (type == LLKeywordToken::TT_LONG_BRACKET) { - std::string end_pattern(regex_token->getDelimiter().begin(), regex_token->getDelimiter().end()); + const llwchar* p = cur + cur_delimiter->getLengthHead(); // after --[ or [ + + // Count equals signs + S32 level = 0; + while (*p == '=') + { + level++; + p++; + } - try + // Must have second [ + if (*p == '[') { - std::smatch start_match; + p++; // skip the second [ + const llwchar* content_start = p; - if (std::regex_search(text_to_search, start_match, *compiled_regex) && !start_match.empty()) + // Build the closing pattern: ] + level equals + ] + // Search for it in the remaining text + while (*p) { - if (start_match.position() == 0) // Match starts at current position + if (*p == ']') { - // Calculate segment boundaries for start pattern - S32 start_match_length = static_cast(start_match.str().length()); - S32 start_seg_end = seg_start + start_match_length; - - if (end_pattern.empty()) + // Check if this is our closing bracket + const llwchar* close_check = p + 1; + S32 close_equals = 0; + while (*close_check == '=') { - // If no end pattern is provided, treat the entire regex match as a single segment - // Move cursor past the matched segment - cur = base + start_seg_end; - - // Insert the matched segment - insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, start_seg_end, style, editor); + close_equals++; + close_check++; } - else - { // TODO: better optimization for this part - - // Look for the end pattern after the start pattern - std::string remaining_text = text_to_search.substr(start_match_length); - - // Process end pattern - replace any capture group references - std::string actual_end_pattern = end_pattern; - - // Handle capture groups in the end pattern (replace \1, \2, etc. with their matched content) - for (size_t i = 1; i < start_match.size(); ++i) - { - std::string capture = start_match[i].str(); - std::string placeholder = "\\" + std::to_string(i); - - // Replace all occurrences of the placeholder with the captured content - size_t pos = 0; - while ((pos = actual_end_pattern.find(placeholder, pos)) != std::string::npos) - { - actual_end_pattern.replace(pos, placeholder.length(), capture); - pos += capture.length(); - } - } - - try - { - std::regex end_regex_pattern(actual_end_pattern); - std::smatch end_match; - - S32 seg_end = start_seg_end; - - if (std::regex_search(remaining_text, end_match, end_regex_pattern) && !end_match.empty()) - { - // Calculate position of end match relative to the original text - S32 end_match_position = static_cast(end_match.position()); - S32 end_match_length = static_cast(end_match.str().length()); - - // Calculate the total length including both patterns and text between - seg_end += end_match_position + end_match_length; - } - else - { - // End pattern not found, treat everything up to EOF as the segment - seg_end += static_cast(remaining_text.length()); - } - - // Move cursor past the entire matched segment (start + content + end) - cur = base + seg_end; - - // Insert the matched segment - insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, seg_end, style, editor); - } - catch (const std::regex_error& e) - { - LL_WARNS() << "Regex error in end pattern: " << e.what() << " in pattern: " << actual_end_pattern << LL_ENDL; - // Fall back to treating the start match as the entire segment - cur = base + start_seg_end; - insertSegments(wtext, *seg_list, regex_token, text_len, seg_start, start_seg_end, style, editor); - } + if (close_equals == level && *close_check == ']') + { + // Found the matching close + seg_end = (S32)(close_check + 1 - base); + cur = close_check + 1; + insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor); + break; } - - regex_matched = true; - break; } + p++; + } + + if (!*p) + { + // No closing found, highlight to end of file + seg_end = text_len - 1; + cur = base + seg_end; + insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor); } + continue; } - catch (const std::regex_error& e) + else { - LL_WARNS() << "Error using compiled regex: " << e.what() << LL_ENDL; + // Not a valid long bracket (e.g., --[abc), skip this delimiter + cur_delimiter = NULL; } } - else - { - // Skip tokens that aren't pre-compiled - LL_WARNS() << "Skipping regex token due to missing pre-compiled pattern: " - << wstring_to_utf8str(regex_token->getToken()) << LL_ENDL; - } - } - if (regex_matched) - { - continue; - } - } - - // Check against delimiters - { - S32 seg_start = 0; - LLKeywordToken* cur_delimiter = NULL; - for (auto* delimiter : delimiters) - { - if( delimiter->isHead( cur ) ) + if (!cur_delimiter) { - cur_delimiter = delimiter; - break; + // Long bracket validation failed, continue to next character + cur++; + continue; } - } - if( cur_delimiter ) - { - S32 between_delimiters = 0; - S32 seg_end = 0; - - seg_start = (S32)(cur - base); cur += cur_delimiter->getLengthHead(); - LLKeywordToken::ETokenType type = cur_delimiter->getType(); if(type == LLKeywordToken::TT_TWO_SIDED_DELIMITER || type == LLKeywordToken::TT_DOUBLE_QUOTATION_MARKS) { while( *cur && !cur_delimiter->isTail(cur)) @@ -1009,14 +933,6 @@ void LLKeywords::dump() LLKeywordToken* delimiter_token = *iter; delimiter_token->dump(); } - - LL_INFOS() << "LLKeywords::sRegexTokenList" << LL_ENDL; - for (token_list_t::iterator iter = mRegexTokenList.begin(); - iter != mRegexTokenList.end(); ++iter) - { - LLKeywordToken* regex_token = *iter; - regex_token->dump(); - } } void LLKeywordToken::dump() diff --git a/indra/llui/llkeywords.h b/indra/llui/llkeywords.h index f5da50a053..9dbe26a66d 100644 --- a/indra/llui/llkeywords.h +++ b/indra/llui/llkeywords.h @@ -36,7 +36,6 @@ #include #include #include -#include #include "llpointer.h" class LLTextSegment; @@ -54,11 +53,7 @@ public: * - TT_ONE_SIDED_DELIMITER are for open-ended delimiters which are terminated by EOL. * - TT_TWO_SIDED_DELIMITER are for delimiters that end with a different delimiter than they open with. * - TT_DOUBLE_QUOTATION_MARKS are for delimiting areas using the same delimiter to open and close. - * - TT_REGEX_MATCH are for pattern-based matching using regular expressions. - * For TT_REGEX_MATCH: mToken contains the start pattern, mDelimiter contains the end pattern (if any). - * If mDelimiter is empty, the entire match is considered one segment. - * If mDelimiter contains capture group references (e.g. \1, \2), these will be replaced with - * the corresponding capture groups from the start pattern match. + * - TT_LONG_BRACKET are for Lua tokens that use brackets with counted equals signs. */ typedef enum e_token_type { @@ -68,7 +63,7 @@ public: TT_TWO_SIDED_DELIMITER, TT_ONE_SIDED_DELIMITER, TT_DOUBLE_QUOTATION_MARKS, - TT_REGEX_MATCH, + TT_LONG_BRACKET, // Lua long brackets: --[=*[ or [=*[ // Following constants are more specific versions of the preceding ones TT_CONSTANT, // WORD TT_CONTROL, // WORD @@ -85,18 +80,12 @@ public: mToken( token ), mColor( color ), mToolTip( tool_tip ), - mDelimiter( delimiter ), // right delimiter - mCompiledRegex( nullptr ) + mDelimiter( delimiter ) // right delimiter { } ~LLKeywordToken() { - if (mCompiledRegex) - { - delete mCompiledRegex; - mCompiledRegex = nullptr; - } } S32 getLengthHead() const { return static_cast(mToken.size()); } @@ -108,8 +97,6 @@ public: ETokenType getType() const { return mType; } const LLWString& getToolTip() const { return mToolTip; } const LLWString& getDelimiter() const { return mDelimiter; } - std::regex* getCompiledRegex() const { return mCompiledRegex; } - void setCompiledRegex(std::regex* regex) { mCompiledRegex = regex; } #ifdef _DEBUG void dump(); @@ -121,7 +108,6 @@ private: LLUIColor mColor; LLWString mToolTip; LLWString mDelimiter; - std::regex* mCompiledRegex; }; class LLKeywords @@ -214,7 +200,6 @@ protected: typedef std::deque token_list_t; token_list_t mLineTokenList; token_list_t mDelimiterTokenList; - token_list_t mRegexTokenList; typedef std::map> element_attributes_t; typedef element_attributes_t::const_iterator attribute_iterator_t; -- cgit v1.3