Merge 'release/luau-scripting' into lua-leap for Emoji release.

6 files changed, 212 insertions, 7 deletions

diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt
index 990d661e38..aed9ee080b 100644
--- a/indra/llcommon/CMakeLists.txt
+++ b/indra/llcommon/CMakeLists.txt
@@ -3,6 +3,7 @@
 project(llcommon)
 
 include(00-Common)
+include(ICU4C)
 include(LLCommon)
 include(bugsplat)
 include(Linking)
@@ -286,6 +287,7 @@ target_link_libraries(
         ll::uriparser
         ll::oslibraries
         ll::tracy
+        ll::icu4c
     )
 
 target_include_directories(llcommon INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/indra/llcommon/llcoros.h b/indra/llcommon/llcoros.h
index 966ce03296..fd878f20ad 100644
--- a/indra/llcommon/llcoros.h
+++ b/indra/llcommon/llcoros.h
@@ -92,7 +92,7 @@ class LL_COMMON_API LLCoros: public LLSingleton<LLCoros>
     LLSINGLETON(LLCoros);
     ~LLCoros();
 
-    void cleanupSingleton();
+    void cleanupSingleton() override;
 public:
     /// The viewer's use of the term "coroutine" became deeply embedded before
     /// the industry term "fiber" emerged to distinguish userland threads from
diff --git a/indra/llcommon/llsingleton.h b/indra/llcommon/llsingleton.h
index 51ef514cf7..cbe5ab6406 100644
--- a/indra/llcommon/llsingleton.h
+++ b/indra/llcommon/llsingleton.h
@@ -802,7 +802,7 @@ public:
 private:                                                                \
     /* implement LLSingleton pure virtual method whose sole purpose */  \
     /* is to remind people to use this macro */                         \
-    virtual void you_must_use_LLSINGLETON_macro() {}                    \
+    virtual void you_must_use_LLSINGLETON_macro() override {}                    \
     friend class LLSingleton<DERIVED_CLASS>;                            \
     DERIVED_CLASS(__VA_ARGS__)
 
diff --git a/indra/llcommon/llstring.cpp b/indra/llcommon/llstring.cpp
index f6629803ee..82dc7c9f80 100644
--- a/indra/llcommon/llstring.cpp
+++ b/indra/llcommon/llstring.cpp
@@ -30,6 +30,7 @@
 #include "llerror.h"
 #include "llfasttimer.h"
 #include "llsd.h"
+#include <unicode/uchar.h>
 #include <vector>
 
 #if LL_WINDOWS
@@ -338,8 +339,6 @@ S32 wchar_utf8_length(const llwchar wc)
 {
 	if (wc < 0x80)
 	{
-		// This case will also catch negative values which are
-		// technically invalid.
 		return 1;
 	}
 	else if (wc < 0x800)
@@ -364,6 +363,30 @@ S32 wchar_utf8_length(const llwchar wc)
 	}
 }
 
+std::string wchar_utf8_preview(const llwchar wc)
+{
+    std::ostringstream oss;
+    oss << std::hex << std::uppercase << (U32)wc;
+
+    U8 out_bytes[8];
+    U32 size = (U32)wchar_to_utf8chars(wc, (char*)out_bytes);
+
+    if (size > 1)
+    {
+        oss << " [";
+        for (U32 i = 0; i < size; ++i)
+        {
+            if (i)
+            {
+                oss << ", ";
+            }
+            oss << (int)out_bytes[i];
+        }
+        oss << "]";
+    }
+
+    return oss.str();
+}
 
 S32 wstring_utf8_length(const LLWString& wstr)
 {
@@ -600,6 +623,7 @@ std::string mbcsstring_makeASCII(const std::string& wstr)
 	}
 	return out_str;
 }
+
 std::string utf8str_removeCRLF(const std::string& utf8str)
 {
 	if (0 == utf8str.length())
@@ -621,6 +645,119 @@ std::string utf8str_removeCRLF(const std::string& utf8str)
 	return out;
 }
 
+llwchar utf8str_to_wchar(const std::string& utf8str, size_t offset, size_t length)
+{
+    switch (length)
+    {
+    case 2:
+        return ((utf8str[offset] & 0x1F) << 6) +
+                (utf8str[offset + 1] & 0x3F);
+    case 3:
+        return ((utf8str[offset] & 0x0F) << 12) +
+                ((utf8str[offset + 1] & 0x3F) << 6) +
+                (utf8str[offset + 2] & 0x3F);
+    case 4:
+        return ((utf8str[offset] & 0x07) << 18) +
+                ((utf8str[offset + 1] & 0x3F) << 12) +
+                ((utf8str[offset + 2] & 0x3F) << 6) +
+                (utf8str[offset + 3] & 0x3F);
+    case 5:
+        return ((utf8str[offset] & 0x03) << 24) +
+                ((utf8str[offset + 1] & 0x3F) << 18) +
+                ((utf8str[offset + 2] & 0x3F) << 12) +
+                ((utf8str[offset + 3] & 0x3F) << 6) +
+                (utf8str[offset + 4] & 0x3F);
+    case 6:
+        return ((utf8str[offset] & 0x01) << 30) +
+                ((utf8str[offset + 1] & 0x3F) << 24) +
+                ((utf8str[offset + 2] & 0x3F) << 18) +
+                ((utf8str[offset + 3] & 0x3F) << 12) +
+                ((utf8str[offset + 4] & 0x3F) << 6) +
+                (utf8str[offset + 5] & 0x3F);
+    case 7:
+        return ((utf8str[offset + 1] & 0x03) << 30) +
+                ((utf8str[offset + 2] & 0x3F) << 24) +
+                ((utf8str[offset + 3] & 0x3F) << 18) +
+                ((utf8str[offset + 4] & 0x3F) << 12) +
+                ((utf8str[offset + 5] & 0x3F) << 6) +
+                (utf8str[offset + 6] & 0x3F);
+    }
+    return LL_UNKNOWN_CHAR;
+}
+
+std::string utf8str_showBytesUTF8(const std::string& utf8str)
+{
+    std::string result;
+
+    bool in_sequence = false;
+    size_t sequence_size = 0;
+    size_t byte_index = 0;
+    size_t source_length = utf8str.size();
+
+    auto open_sequence = [&]()
+        {
+            if (!result.empty() && result.back() != '\n')
+                result += '\n'; // Use LF as a separator before new UTF-8 sequence
+            result += '[';
+            in_sequence = true;
+        };
+
+    auto close_sequence = [&]()
+        {
+            llwchar unicode = utf8str_to_wchar(utf8str, byte_index - sequence_size, sequence_size);
+            if (unicode != LL_UNKNOWN_CHAR)
+            {
+                result += llformat("+%04X", unicode);
+            }
+            result += ']';
+            in_sequence = false;
+            sequence_size = 0;
+        };
+
+    while (byte_index < source_length)
+    {
+        U8 byte = utf8str[byte_index];
+        if (byte >= 0x80) // Part of an UTF-8 sequence
+        {
+            if (!in_sequence) // Start new UTF-8 sequence
+            {
+                open_sequence();
+            }
+            else if (byte >= 0xC0) // Start another UTF-8 sequence
+            {
+                close_sequence();
+                open_sequence();
+            }
+            else // Continue the same UTF-8 sequence
+            {
+                result += '.';
+            }
+            result += llformat("%02X", byte); // The byte is represented in hexadecimal form
+            ++sequence_size;
+        }
+        else // ASCII symbol is represented as a character
+        {
+            if (in_sequence) // End of UTF-8 sequence
+            {
+                close_sequence();
+                if (byte != '\n')
+                {
+                    result += '\n'; // Use LF as a separator between UTF-8 and ASCII
+                }
+            }
+            result += byte;
+        }
+        ++byte_index;
+    }
+
+    if (in_sequence) // End of UTF-8 sequence
+    {
+        close_sequence();
+    }
+
+    return result;
+}
+
 #if LL_WINDOWS
 unsigned int ll_wstring_default_code_page()
 {
@@ -833,6 +970,40 @@ std::string LLStringOps::sDayFormat;
 std::string LLStringOps::sAM;
 std::string LLStringOps::sPM;
 
+// static
+bool LLStringOps::isEmoji(llwchar wch)
+{
+	int ublock = ublock_getCode(wch);
+	switch (ublock)
+	{
+		case UBLOCK_GENERAL_PUNCTUATION:
+		case UBLOCK_LETTERLIKE_SYMBOLS:
+		case UBLOCK_ARROWS:
+		case UBLOCK_MISCELLANEOUS_TECHNICAL:
+		case UBLOCK_ENCLOSED_ALPHANUMERICS:
+		case UBLOCK_GEOMETRIC_SHAPES:
+		case UBLOCK_MISCELLANEOUS_SYMBOLS:
+		case UBLOCK_DINGBATS:
+		case UBLOCK_CJK_SYMBOLS_AND_PUNCTUATION:
+		case UBLOCK_ENCLOSED_CJK_LETTERS_AND_MONTHS:
+		case UBLOCK_MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS:
+		case UBLOCK_EMOTICONS:
+		case UBLOCK_TRANSPORT_AND_MAP_SYMBOLS:
+#if U_ICU_VERSION_MAJOR_NUM > 56
+		// Boost uses ICU so we can't update it independently
+		case UBLOCK_SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS:
+#endif // U_ICU_VERSION_MAJOR_NUM > 56
+			return true;
+		default:
+#if U_ICU_VERSION_MAJOR_NUM > 56
+			return false;
+#else
+			// See https://en.wikipedia.org/wiki/Supplemental_Symbols_and_Pictographs
+			return wch >= 0x1F900 && wch <= 0x1F9FF;
+#endif // U_ICU_VERSION_MAJOR_NUM > 56
+	}
+}
+
 
 S32	LLStringOps::collate(const llwchar* a, const llwchar* b)
 { 
diff --git a/indra/llcommon/llstring.h b/indra/llcommon/llstring.h
index dbd60bc9c7..0eb2770004 100644
--- a/indra/llcommon/llstring.h
+++ b/indra/llcommon/llstring.h
@@ -190,6 +190,8 @@ public:
 	static bool isAlnum(char a) { return isalnum((unsigned char)a) != 0; }
 	static bool isAlnum(llwchar a) { return iswalnum(a) != 0; }
 
+	static bool isEmoji(llwchar wch);
+
 	static S32	collate(const char* a, const char* b) { return strcoll(a, b); }
 	static S32	collate(const llwchar* a, const llwchar* b);
 
@@ -356,6 +358,8 @@ public:
 	static void	replaceNonstandardASCII( string_type& string, T replacement );
 	static void	replaceChar( string_type& string, T target, T replacement );
 	static void replaceString( string_type& string, string_type target, string_type replacement );
+	static string_type capitalize(const string_type& str);
+	static void capitalize(string_type& str);
 	
 	static BOOL	containsNonprintable(const string_type& string);
 	static void	stripNonprintable(string_type& string);
@@ -706,6 +710,8 @@ LL_COMMON_API S32 wstring_utf8_length(const LLWString& wstr);
 // Length in bytes of this wide char in a UTF8 string
 LL_COMMON_API S32 wchar_utf8_length(const llwchar wc); 
 
+LL_COMMON_API std::string wchar_utf8_preview(const llwchar wc);
+
 LL_COMMON_API std::string utf8str_tolower(const std::string& utf8str);
 
 // Length in llwchar (UTF-32) of the first len units (16 bits) of the given UTF-16 string.
@@ -765,6 +771,9 @@ LL_COMMON_API std::string mbcsstring_makeASCII(const std::string& str);
 
 LL_COMMON_API std::string utf8str_removeCRLF(const std::string& utf8str);
 
+LL_COMMON_API llwchar utf8str_to_wchar(const std::string& utf8str, size_t offset, size_t length);
+
+LL_COMMON_API std::string utf8str_showBytesUTF8(const std::string& utf8str);
 
 #if LL_WINDOWS
 /* @name Windows string helpers
@@ -1622,6 +1631,29 @@ void LLStringUtilBase<T>::replaceTabsWithSpaces( string_type& str, size_type spa
 }
 
 //static
+template<class T>
+std::basic_string<T> LLStringUtilBase<T>::capitalize(const string_type& str)
+{
+	string_type result(str);
+	capitalize(result);
+	return result;
+}
+
+//static
+template<class T>
+void LLStringUtilBase<T>::capitalize(string_type& str)
+{
+	if (str.size())
+	{
+		auto last = str[0] = toupper(str[0]);
+		for (U32 i = 1; i < str.size(); ++i)
+		{
+			last = (last == ' ' || last == '-' || last == '_') ? str[i] = toupper(str[i]) : str[i];
+		}
+	}
+}
+
+//static
 template<class T> 
 BOOL LLStringUtilBase<T>::containsNonprintable(const string_type& string)
 {
diff --git a/indra/llcommon/tests/llsingleton_test.cpp b/indra/llcommon/tests/llsingleton_test.cpp
index 15ffe68e67..6f8aaaa0cb 100644
--- a/indra/llcommon/tests/llsingleton_test.cpp
+++ b/indra/llcommon/tests/llsingleton_test.cpp
@@ -47,8 +47,8 @@ public:                                             \
         DEP_INIT  /* dependency in initSingleton */ \
     } sDepFlag;                                     \
                                                     \
-    void initSingleton();                           \
-    void cleanupSingleton();                        \
+    void initSingleton() override;                  \
+    void cleanupSingleton() override;               \
 };                                                  \
                                                     \
 CLS::dep_flag CLS::sDepFlag = DEP_NONE
@@ -300,7 +300,7 @@ namespace tut
     {
         LLSINGLETON_EMPTY_CTOR(CircularPInit);
     public:
-        virtual void initSingleton()
+        virtual void initSingleton() override
         {
             // never mind indirection, just go straight for the circularity
             CircularPInit *pt = getInstance();