#5278 Move syntax tokenization off UI thread; apply results on main thread

1 files changed, 254 insertions, 21 deletions

diff --git a/indra/newview/llscripteditor.cpp b/indra/newview/llscripteditor.cpp
index d171d785a7..871210ea6d 100644
--- a/indra/newview/llscripteditor.cpp
+++ b/indra/newview/llscripteditor.cpp
@@ -31,6 +31,173 @@
 #include "llsyntaxid.h"
 #include "lllocalcliprect.h"
 #include "llviewercontrol.h"
+#include "llworkerthread.h"
+#include <utility>
+
+static LLWorkerThread& getSyntaxWorkerThread()
+{
+    static LLWorkerThread* sThread = new LLWorkerThread("SyntaxParse", true);
+    return *sThread;
+}
+
+class LLScriptEditorSyntaxWorker final : public LLWorkerClass
+{
+public:
+    struct Request
+    {
+        LLWString       text;
+        S32             text_generation = 0;
+        U32             keywords_generation = 0;
+        bool            disable_highlight = false;
+        LLKeywords*     keywords = nullptr;
+    };
+
+    struct Result
+    {
+        LLWString                   text;
+        LLKeywords::segment_ops_t   ops;
+        S32                         text_generation = 0;
+        U32                         keywords_generation = 0;
+        bool                        disable_highlight = false;
+        LLKeywords*                 keywords = nullptr;
+    };
+
+    LLScriptEditorSyntaxWorker(LLWorkerThread* thread, LLScriptEditor* editor)
+    :   LLWorkerClass(thread, "ScriptEditorSyntax"),
+        mEditor(editor),
+        mHasPending(false),
+        mDropResults(false)
+    {
+    }
+
+    void queueRequest(const Request& request)
+    {
+        bool should_add = false;
+        {
+            LLMutexLock lock(&mRequestMutex);
+            mPendingRequest = request;
+            mHasPending = true;
+            should_add = (!haveWork() && !isWorking());
+        }
+        if (should_add)
+        {
+            addWork(0);
+        }
+    }
+
+    void pump()
+    {
+        checkWork();
+        bool should_add = false;
+        {
+            LLMutexLock lock(&mRequestMutex);
+            should_add = (!haveWork() && !isWorking() && mHasPending);
+        }
+        if (should_add)
+        {
+            addWork(0);
+        }
+    }
+
+    void waitForIdle(bool drop_results)
+    {
+        mDropResults = drop_results;
+        {
+            LLMutexLock lock(&mRequestMutex);
+            mHasPending = false;
+        }
+        if (mRequestHandle != LLWorkerThread::nullHandle())
+        {
+            mWorkerThread->waitForResult(mRequestHandle, false);
+            checkWork();
+        }
+        mDropResults = false;
+    }
+
+    void shutdown()
+    {
+        mEditor = nullptr;
+        waitForIdle(true);
+        scheduleDelete();
+    }
+
+private:
+    void startWork(S32 param) override
+    {
+        LLMutexLock lock(&mRequestMutex);
+        if (!mHasPending)
+        {
+            return;
+        }
+        mActiveRequest = mPendingRequest;
+        mHasPending = false;
+    }
+
+    bool doWork(S32 param) override
+    {
+        Request request;
+        {
+            LLMutexLock lock(&mRequestMutex);
+            request = mActiveRequest;
+        }
+
+        Result result;
+        result.text = request.text;
+        result.text_generation = request.text_generation;
+        result.keywords_generation = request.keywords_generation;
+        result.disable_highlight = request.disable_highlight;
+        result.keywords = request.keywords;
+
+        if (request.keywords)
+        {
+            request.keywords->collectSegmentOps(result.ops, request.text, request.disable_highlight);
+        }
+
+        {
+            LLMutexLock lock(&mResultMutex);
+            mResult = std::move(result);
+        }
+        return true;
+    }
+
+    void endWork(S32 param, bool aborted) override
+    {
+        if (aborted || mDropResults || !mEditor)
+        {
+            return;
+        }
+
+        Result result;
+        {
+            LLMutexLock lock(&mResultMutex);
+            result = std::move(mResult);
+        }
+
+        if (mEditor->getTextGeneration() != result.text_generation)
+        {
+            return;
+        }
+        if (mEditor->getKeywordsGeneration() != result.keywords_generation)
+        {
+            return;
+        }
+        if (&mEditor->getKeywords() != result.keywords)
+        {
+            return;
+        }
+
+        mEditor->applySyntaxSegments(result.text, result.ops);
+    }
+
+    LLScriptEditor* mEditor;
+    LLMutex mRequestMutex;
+    LLMutex mResultMutex;
+    Request mPendingRequest;
+    Request mActiveRequest;
+    Result mResult;
+    bool mHasPending;
+    bool mDropResults;
+};
 
 const S32   UI_TEXTEDITOR_LINE_NUMBER_MARGIN = 32;
 
@@ -46,7 +213,13 @@ LLScriptEditor::LLScriptEditor(const Params& p)
 :   LLTextEditor(p)
 ,   mShowLineNumbers(p.show_line_numbers),
     mUseDefaultFontSize(p.default_font_size),
-    mLuauLanguage(false)
+    mLuauLanguage(false),
+    mKeywordsGeneration(0),
+    mLastQueuedTextGeneration(-1),
+    mLastQueuedKeywordsGeneration(0),
+    mLastQueuedDisableHighlight(false),
+    mLastQueuedKeywords(nullptr),
+    mSyntaxWorker(nullptr)
 {
     if (mShowLineNumbers)
     {
@@ -55,6 +228,16 @@ LLScriptEditor::LLScriptEditor(const Params& p)
     }
 }
 
+LLScriptEditor::~LLScriptEditor()
+{
+    if (mSyntaxWorker)
+    {
+        mSyntaxWorker->shutdown();
+        getSyntaxWorkerThread().update(0.f);
+        mSyntaxWorker = nullptr;
+    }
+}
+
 bool LLScriptEditor::postBuild()
 {
     gSavedSettings.getControl("LSLFontSizeName")->getCommitSignal()->connect(boost::bind(&LLScriptEditor::onFontSizeChange, this));
@@ -154,43 +337,93 @@ void LLScriptEditor::initKeywords(bool luau_language)
 void LLScriptEditor::loadKeywords()
 {
     LL_PROFILE_ZONE_SCOPED;
+    ensureSyntaxWorker();
+    mSyntaxWorker->waitForIdle(true);
     getKeywords().processTokens();
+    ++mKeywordsGeneration;
 
-    LLStyleConstSP style = new LLStyle(LLStyle::Params().font(getScriptFont()).color(mDefaultColor.get()));
-
-    segment_vec_t segment_list;
-    getKeywords().findSegments(&segment_list, getWText(), *this, style);
-
-    mSegments.clear();
-    segment_set_t::iterator insert_it = mSegments.begin();
-    for (segment_vec_t::iterator list_it = segment_list.begin(); list_it != segment_list.end(); ++list_it)
+    LLKeywords::segment_ops_t ops;
+    const LLWString& text = getWText();
+    const llwchar* base = text.c_str();
+    for (const llwchar* cur = base; *cur; ++cur)
     {
-        insert_it = mSegments.insert(insert_it, *list_it);
+        if (*cur == '\n')
+        {
+            ops.push_back({LLKeywords::SegmentOp::OP_LINE_BREAK, (S32)(cur - base), 0, nullptr});
+        }
     }
+    applySyntaxSegments(text, ops);
+    queueSyntaxParse();
 }
 
 void LLScriptEditor::updateSegments()
 {
-    if (mReflowIndex < S32_MAX && getKeywords().isLoaded() && mParseOnTheFly)
+    if (getKeywords().isLoaded() && mParseOnTheFly)
     {
         LL_PROFILE_ZONE_SCOPED;
-
-        LLStyleConstSP style = new LLStyle(LLStyle::Params().font(getScriptFont()).color(mDefaultColor.get()));
-
-        // HACK:  No non-ascii keywords for now
-        segment_vec_t segment_list;
-        getKeywords().findSegments(&segment_list, getWText(), *this, style);
-
-        clearSegments();
-        for (segment_vec_t::iterator list_it = segment_list.begin(); list_it != segment_list.end(); ++list_it)
+        ensureSyntaxWorker();
+        mSyntaxWorker->pump();
+        if (mReflowIndex < S32_MAX)
         {
-            insertSegment(*list_it);
+            queueSyntaxParse();
         }
     }
 
     LLTextBase::updateSegments();
 }
 
+void LLScriptEditor::ensureSyntaxWorker()
+{
+    if (!mSyntaxWorker)
+    {
+        mSyntaxWorker = new LLScriptEditorSyntaxWorker(&getSyntaxWorkerThread(), this);
+    }
+}
+
+void LLScriptEditor::queueSyntaxParse()
+{
+    static LLCachedControl<bool> sDisableSyntaxHighlighting(gSavedSettings, "ScriptEditorDisableSyntaxHighlight", false);
+    const bool disable_syntax_highlighting = sDisableSyntaxHighlighting;
+
+    LLKeywords* keywords = &getKeywords();
+    const S32 text_generation = getTextGeneration();
+
+    if (mLastQueuedTextGeneration == text_generation
+        && mLastQueuedKeywordsGeneration == mKeywordsGeneration
+        && mLastQueuedDisableHighlight == disable_syntax_highlighting
+        && mLastQueuedKeywords == keywords)
+    {
+        return;
+    }
+
+    mLastQueuedTextGeneration = text_generation;
+    mLastQueuedKeywordsGeneration = mKeywordsGeneration;
+    mLastQueuedDisableHighlight = disable_syntax_highlighting;
+    mLastQueuedKeywords = keywords;
+
+    LLScriptEditorSyntaxWorker::Request request;
+    request.text = getWText();
+    request.text_generation = text_generation;
+    request.keywords_generation = mKeywordsGeneration;
+    request.disable_highlight = disable_syntax_highlighting;
+    request.keywords = keywords;
+    mSyntaxWorker->queueRequest(request);
+}
+
+void LLScriptEditor::applySyntaxSegments(const LLWString& text, const LLKeywords::segment_ops_t& ops)
+{
+    LLStyleConstSP style = new LLStyle(LLStyle::Params().font(getScriptFont()).color(mDefaultColor.get()));
+
+    segment_vec_t segment_list;
+    getKeywords().applySegmentOps(&segment_list, text, ops, *this, style);
+
+    clearSegments();
+    for (segment_vec_t::iterator list_it = segment_list.begin(); list_it != segment_list.end(); ++list_it)
+    {
+        insertSegment(*list_it);
+    }
+}
+
 void LLScriptEditor::clearSegments()
 {
     if (!mSegments.empty())