#5278 Move syntax tokenization off UI thread; apply results on main thread

1 files changed, 69 insertions, 40 deletions

diff --git a/indra/llui/llkeywords.cpp b/indra/llui/llkeywords.cpp
index 2168ba2382..0ada54df06 100644
--- a/indra/llui/llkeywords.cpp
+++ b/indra/llui/llkeywords.cpp
@@ -498,41 +498,25 @@ bool LLKeywords::WStringMapIndex::operator<(const LLKeywords::WStringMapIndex &o
 
 LLTrace::BlockTimerStatHandle FTM_SYNTAX_COLORING("Syntax Coloring");
 
-// Walk through a string, applying the rules specified by the keyword token list and
-// create a list of color segments.
-void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLWString& wtext, LLTextEditor& editor, LLStyleConstSP style)
+void LLKeywords::collectSegmentOps(segment_ops_t& ops, const LLWString& wtext, bool disable_syntax_highlighting) const
 {
-    LL_RECORD_BLOCK_TIME(FTM_SYNTAX_COLORING);
+    ops.clear();
 
-    if( wtext.empty() )
+    if (wtext.empty())
     {
         return;
     }
 
-    // Clear the segment list
-    seg_list->clear();
-    // Reserve capacity for segments based on an estimated average of 8 characters per segment.
-    constexpr size_t AVERAGE_SEGMENT_LENGTH = 8;
-    seg_list->reserve(wtext.size() / AVERAGE_SEGMENT_LENGTH);
-
-    S32 text_len = static_cast<S32>(wtext.size()) + 1;
-
-    seg_list->push_back( new LLNormalTextSegment( style, 0, text_len, editor ) );
-
-    static LLCachedControl<bool> sDisableSyntaxHighlighting(gSavedSettings, "ScriptEditorDisableSyntaxHighlight", false);
-    const bool disable_syntax_highlighting = sDisableSyntaxHighlighting;
-
     const llwchar* base = wtext.c_str();
     const llwchar* cur = base;
+
     while( *cur )
     {
         if( *cur == '\n' || cur == base )
         {
             if( *cur == '\n' )
             {
-                LLTextSegmentPtr text_segment = new LLLineBreakTextSegment(style, (S32)(cur - base));
-                text_segment->setToken( 0 );
-                insertSegment( *seg_list, text_segment, text_len, style, editor);
+                ops.push_back({SegmentOp::OP_LINE_BREAK, (S32)(cur - base), 0, nullptr});
                 cur++;
                 if( !*cur || *cur == '\n' )
                 {
@@ -570,8 +554,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                             }
                             S32 seg_end = (S32)(cur - base);
 
-                            //create segments from seg_start to seg_end
-                            insertSegments(wtext, *seg_list,cur_token, text_len, seg_start, seg_end, style, editor);
+                            ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_end, cur_token});
                             line_done = true; // to break out of second loop.
                             break;
                         }
@@ -646,7 +629,6 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                         if (*p == '[')
                         {
                             p++;  // skip the second [
-                            const llwchar* content_start = p;
 
                             // Build the closing pattern: ] + level equals + ]
                             // Search for it in the remaining text
@@ -667,7 +649,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                                         // Found the matching close
                                         seg_end = (S32)(close_check + 1 - base);
                                         cur = close_check + 1;
-                                        insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor);
+                                        ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_end, cur_delimiter});
                                         break;
                                     }
                                 }
@@ -677,9 +659,9 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                             if (!*p)
                             {
                                 // No closing found, highlight to end of file
-                                seg_end = text_len - 1;
+                                seg_end = static_cast<S32>(wtext.size());
                                 cur = base + seg_end;
-                                insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor);
+                                ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_end, cur_delimiter});
                             }
                             continue;
                         }
@@ -761,7 +743,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                         seg_end = seg_start + between_delimiters + cur_delimiter->getLengthHead();
                     }
 
-                    insertSegments(wtext, *seg_list, cur_delimiter, text_len, seg_start, seg_end, style, editor);
+                    ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_end, cur_delimiter});
                     // Note: we don't increment cur, since the end of one delimited seg may be immediately
                     // followed by the start of another one.
                     continue;
@@ -794,13 +776,12 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                     S32 seg_end = seg_start + seg_len;
 
                     // First try to match the whole token (including dots for Lua namespaces)
-                    word_token_map_t::iterator map_iter = mWordTokenMap.find(WStringMapIndex(word_start, seg_len));
+                    word_token_map_t::const_iterator map_iter = mWordTokenMap.find(WStringMapIndex(word_start, seg_len));
 
                     if (map_iter != mWordTokenMap.end())
                     {
                         // Found a match for the complete token (including any namespace)
-                        LLKeywordToken* cur_token = map_iter->second;
-                        insertSegments(wtext, *seg_list, cur_token, text_len, seg_start, seg_end, style, editor);
+                        ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_end, map_iter->second});
                     }
                     else if (namespace_dots > 0 && mLuauLanguage)
                     {
@@ -815,8 +796,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                             if (map_iter != mWordTokenMap.end())
                             {
                                 // Found a match for the namespace prefix, highlight just that part
-                                LLKeywordToken* cur_token = map_iter->second;
-                                insertSegments(wtext, *seg_list, cur_token, text_len, seg_start, seg_start + prefix_len, style, editor);
+                                ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_start + prefix_len, map_iter->second});
 
                                 // Now try to match the function part (after the dot)
                                 const llwchar* func_part = last_dot + 1;
@@ -830,13 +810,7 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
                                     if (map_iter != mWordTokenMap.end())
                                     {
                                         // Found a match for the function part
-                                        LLKeywordToken* cur_token = map_iter->second;
-                                        insertSegments(wtext, *seg_list, cur_token, text_len, seg_start, seg_end, style, editor);
-                                    }
-                                    else
-                                    {
-                                        // No token found, continue without incrementing cur
-                                        // since we already advanced it while collecting the word
+                                        ops.push_back({SegmentOp::OP_TOKEN, seg_start, seg_end, map_iter->second});
                                     }
                                 }
                             }
@@ -854,6 +828,61 @@ void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLW
     }
 }
 
+void LLKeywords::applySegmentOps(std::vector<LLTextSegmentPtr> *seg_list,
+                                 const LLWString& wtext,
+                                 const segment_ops_t& ops,
+                                 LLTextEditor& editor,
+                                 LLStyleConstSP style)
+{
+    if (wtext.empty())
+    {
+        return;
+    }
+
+    // Clear the segment list
+    seg_list->clear();
+    // Reserve capacity for segments based on an estimated average of 8 characters per segment.
+    constexpr size_t AVERAGE_SEGMENT_LENGTH = 8;
+    seg_list->reserve(wtext.size() / AVERAGE_SEGMENT_LENGTH);
+
+    S32 text_len = static_cast<S32>(wtext.size()) + 1;
+
+    seg_list->push_back( new LLNormalTextSegment( style, 0, text_len, editor ) );
+
+    for (const auto& op : ops)
+    {
+        if (op.type == SegmentOp::OP_LINE_BREAK)
+        {
+            LLTextSegmentPtr text_segment = new LLLineBreakTextSegment(style, op.start);
+            text_segment->setToken( 0 );
+            insertSegment( *seg_list, text_segment, text_len, style, editor);
+        }
+        else
+        {
+            insertSegments(wtext, *seg_list, op.token, text_len, op.start, op.end, style, editor);
+        }
+    }
+}
+
+// Walk through a string, applying the rules specified by the keyword token list and
+// create a list of color segments.
+void LLKeywords::findSegments(std::vector<LLTextSegmentPtr>* seg_list, const LLWString& wtext, LLTextEditor& editor, LLStyleConstSP style)
+{
+    LL_RECORD_BLOCK_TIME(FTM_SYNTAX_COLORING);
+
+    if( wtext.empty() )
+    {
+        return;
+    }
+
+    static LLCachedControl<bool> sDisableSyntaxHighlighting(gSavedSettings, "ScriptEditorDisableSyntaxHighlight", false);
+    const bool disable_syntax_highlighting = sDisableSyntaxHighlighting;
+
+    segment_ops_t ops;
+    collectSegmentOps(ops, wtext, disable_syntax_highlighting);
+    applySegmentOps(seg_list, wtext, ops, editor, style);
+}
+
 void LLKeywords::insertSegments(const LLWString& wtext, std::vector<LLTextSegmentPtr>& seg_list, LLKeywordToken* cur_token, S32 text_len, S32 seg_start, S32 seg_end, LLStyleConstSP style, LLTextEditor& editor )
 {
     std::string::size_type pos = wtext.find('\n',seg_start);