Proposal #2 to restore how UI/dialogs used to render by prioritizing fallback fonts.

With the emojis support, a new font was added, which not only provides emojis
but also fancy colorful replacements for UTF-8 characters that used to be
supported by our fallback (monochrome) fonts: this causes discrepancies and
unwanted/undesired changes in scripted objects menus (e.g. an empty circle or
square may render as a black, full one, a heart may render red instead of white),
not to mention the larger font size used by the emoji characters...

This patch restores the aspect of such menus/dialogs/UI elements with UTF-8
characters that *are* supported by the usual fallback fonts (fonts which may
also vary from one viewer to another, and from one OS to another), so that
everything keeps working/rendering as it always did so far, while not impairing
the use of new colorful emojis.

This second proposal ensures that:
- "genuine" emojis (in the 0x1f000-0x1ffff range), will *always* be rendered
  using the new emojis font (this solves, for example, the monochrome "yellow
  faces" issue seen with some characters in my first proposal).
- Special UTF-8 characters (in the 0x2000-0x32FF range) which have been used by
  scripters so far, will render as they used to, using the monochrome fallback
  fonts (this repairs scripted dialogs menus).
- Remaining special characters, that do not have a corresponding glyph in the
  monochrome font, but do have one in the emojis font, will use the latter font
  to render.

It also got the nice side-effect of removing the dependency on the ICU4C library.

Note however that the recent commit:
https://github.com/secondlife/viewer/commit/326055ba82c22fedde186c6a56bafd4fe87e613a
will need to be reverted to allow this patch to actually fix scripted dialogs.

Also, some cleanup might be needed in skins/default/xui/*/emoji_characters.xml to
remove from it the special UTF-8 characters that will no longer be rendered with
fanciful colors, but instead with the monochrome font glyphs.

10 files changed, 108 insertions, 175 deletions

diff --git a/autobuild.xml b/autobuild.xml
index de503a7d34..e2f78a3358 100644
--- a/autobuild.xml
+++ b/autobuild.xml
@@ -959,54 +959,6 @@
         <key>description</key>
         <string>Havok source code for libs and demos</string>
       </map>
-      <key>icu4c</key>
-      <map>
-        <key>canonical_repo</key>
-        <string>https://bitbucket.org/lindenlab/3p-icu4c</string>
-        <key>copyright</key>
-        <string>Copyright (c) 1995-2011 International Business Machines Corporation and others &lt;http://source.icu-project.org&gt;</string>
-        <key>description</key>
-        <string>ICU is a mature, widely used set of C/C++ and Java libraries providing Unicode and Globalization support for software applications. ICU is widely portable and gives applications the same results on all platforms and between C/C++ and Java software.</string>
-        <key>license</key>
-        <string>ICU, permissive non-copyleft free software license</string>
-        <key>license_file</key>
-        <string>LICENSES/icu.txt</string>
-        <key>name</key>
-        <string>icu4c</string>
-        <key>platforms</key>
-        <map>
-          <key>darwin64</key>
-          <map>
-            <key>archive</key>
-            <map>
-              <key>hash</key>
-              <string>47bc32b991385f1a6530e4c6179b07f64ca6edc7</string>
-              <key>hash_algorithm</key>
-              <string>sha1</string>
-              <key>url</key>
-              <string>https://github.com/secondlife/3p-icu4c/releases/download/v4.8.1-7d08d82/icu4c-4.8.1-darwin64-7d08d82.tar.zst</string>
-            </map>
-            <key>name</key>
-            <string>darwin64</string>
-          </map>
-          <key>windows64</key>
-          <map>
-            <key>archive</key>
-            <map>
-              <key>hash</key>
-              <string>b7db881dac80302e4d9010af34c0bf6ca9897df9</string>
-              <key>hash_algorithm</key>
-              <string>sha1</string>
-              <key>url</key>
-              <string>https://github.com/secondlife/3p-icu4c/releases/download/v4.8.1-7d08d82/icu4c-4.8.1-windows64-7d08d82.tar.zst</string>
-            </map>
-            <key>name</key>
-            <string>windows64</string>
-          </map>
-        </map>
-        <key>version</key>
-        <string>4.8.1-7d08d82</string>
-      </map>
       <key>jpegencoderbasic</key>
       <map>
         <key>platforms</key>
diff --git a/indra/cmake/CMakeLists.txt b/indra/cmake/CMakeLists.txt
index 05c51c018d..cb3b77300a 100644
--- a/indra/cmake/CMakeLists.txt
+++ b/indra/cmake/CMakeLists.txt
@@ -30,7 +30,6 @@ set(cmake_SOURCE_FILES
         GoogleMock.cmake
         Havok.cmake
         Hunspell.cmake
-        ICU4C.cmake
         JsonCpp.cmake
         LLAddBuildTest.cmake
         LLAppearance.cmake
diff --git a/indra/cmake/Copy3rdPartyLibs.cmake b/indra/cmake/Copy3rdPartyLibs.cmake
index 9f79c13a97..7938d4f54b 100644
--- a/indra/cmake/Copy3rdPartyLibs.cmake
+++ b/indra/cmake/Copy3rdPartyLibs.cmake
@@ -62,15 +62,6 @@ if(WINDOWS)
         uriparser.dll
         )
 
-    # ICU4C (same filenames for 32 and 64 bit builds)
-    set(release_files ${release_files} icudt48.dll)
-    set(release_files ${release_files} icuin48.dll)
-    set(release_files ${release_files} icuio48.dll)
-    set(release_files ${release_files} icule48.dll)
-    set(release_files ${release_files} iculx48.dll)
-    set(release_files ${release_files} icutu48.dll)
-    set(release_files ${release_files} icuuc48.dll)
-
     # OpenSSL
     if(ADDRESS_SIZE EQUAL 64)
         set(release_files ${release_files} libcrypto-1_1-x64.dll)
diff --git a/indra/cmake/ICU4C.cmake b/indra/cmake/ICU4C.cmake
deleted file mode 100644
index 7b27665483..0000000000
--- a/indra/cmake/ICU4C.cmake
+++ /dev/null
@@ -1,23 +0,0 @@
-# -*- cmake -*-
-include(Prebuilt)
-
-include_guard()
-
-add_library( ll::icu4c INTERFACE IMPORTED )
-
-
-use_system_binary(icu4c)
-use_prebuilt_binary(icu4c)
-if (WINDOWS)
-  target_link_libraries( ll::icu4c INTERFACE  icuuc)
-elseif(DARWIN)
-  target_link_libraries( ll::icu4c INTERFACE  icuuc)
-#elseif(LINUX)
-##  target_link_libraries( ll::icu4c INTERFACE  )
-else()
-  message(FATAL_ERROR "Invalid platform")
-endif()
-
-target_include_directories( ll::icu4c SYSTEM INTERFACE  ${LIBS_PREBUILT_DIR}/include/unicode )
-
-use_prebuilt_binary(dictionaries)
diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt
index c947184dc8..5f4ed2fffa 100644
--- a/indra/llcommon/CMakeLists.txt
+++ b/indra/llcommon/CMakeLists.txt
@@ -3,7 +3,6 @@
 project(llcommon)
 
 include(00-Common)
-include(ICU4C)
 include(LLCommon)
 include(bugsplat)
 include(Linking)
@@ -283,7 +282,6 @@ target_link_libraries(
         ll::uriparser
         ll::oslibraries
         ll::tracy
-        ll::icu4c
     )
 
 target_include_directories(llcommon INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/indra/llcommon/llstring.cpp b/indra/llcommon/llstring.cpp
index 4aa54bb12d..28c2ec5b39 100644
--- a/indra/llcommon/llstring.cpp
+++ b/indra/llcommon/llstring.cpp
@@ -30,7 +30,6 @@
 #include "llerror.h"
 #include "llfasttimer.h"
 #include "llsd.h"
-#include <unicode/uchar.h>
 #include <vector>
 
 #if LL_WINDOWS
@@ -1008,40 +1007,18 @@ std::string LLStringOps::sAM;
 std::string LLStringOps::sPM;
 
 // static
-bool LLStringOps::isEmoji(llwchar wch)
-{
-	int ublock = ublock_getCode(wch);
-	switch (ublock)
-	{
-		case UBLOCK_GENERAL_PUNCTUATION:
-		case UBLOCK_LETTERLIKE_SYMBOLS:
-		case UBLOCK_ARROWS:
-		case UBLOCK_MISCELLANEOUS_TECHNICAL:
-		case UBLOCK_ENCLOSED_ALPHANUMERICS:
-		case UBLOCK_GEOMETRIC_SHAPES:
-		case UBLOCK_MISCELLANEOUS_SYMBOLS:
-		case UBLOCK_DINGBATS:
-		case UBLOCK_CJK_SYMBOLS_AND_PUNCTUATION:
-		case UBLOCK_ENCLOSED_CJK_LETTERS_AND_MONTHS:
-		case UBLOCK_MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS:
-		case UBLOCK_EMOTICONS:
-		case UBLOCK_TRANSPORT_AND_MAP_SYMBOLS:
-#if U_ICU_VERSION_MAJOR_NUM > 56
-		// Boost uses ICU so we can't update it independently
-		case UBLOCK_SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS:
-#endif // U_ICU_VERSION_MAJOR_NUM > 56
-			return true;
-		default:
-#if U_ICU_VERSION_MAJOR_NUM > 56
-			return false;
+bool LLStringOps::isEmoji(llwchar a)
+{
+#if 0   // Do not consider special characters that might have a corresponding
+        // glyph in the monochorme fallback fonts as a "genuine" emoji. HB
+    return a == 0xa9 || a == 0xae || (a >= 0x2000 && a < 0x3300) ||
+           (a >= 0x1f000 && a < 0x20000);
 #else
-			// See https://en.wikipedia.org/wiki/Supplemental_Symbols_and_Pictographs
-			return wch >= 0x1F900 && wch <= 0x1F9FF;
-#endif // U_ICU_VERSION_MAJOR_NUM > 56
-	}
+    // These are indeed "genuine" emojis, we *do want* rendered as such. HB
+    return a >= 0x1f000 && a < 0x20000;
+#endif
 }
 
-
 S32	LLStringOps::collate(const llwchar* a, const llwchar* b)
 { 
 	#if LL_WINDOWS
diff --git a/indra/llcommon/llstring.h b/indra/llcommon/llstring.h
index a40359115e..a8d910298c 100644
--- a/indra/llcommon/llstring.h
+++ b/indra/llcommon/llstring.h
@@ -189,7 +189,8 @@ public:
 	static bool isAlnum(char a) { return isalnum((unsigned char)a) != 0; }
 	static bool isAlnum(llwchar a) { return iswalnum(a) != 0; }
 
-	static bool isEmoji(llwchar wch);
+    // Returns true when 'a' corresponds to a "genuine" emoji. HB
+	static bool isEmoji(llwchar a);
 
 	static S32	collate(const char* a, const char* b) { return strcoll(a, b); }
 	static S32	collate(const llwchar* a, const llwchar* b);
diff --git a/indra/llrender/llfontfreetype.cpp b/indra/llrender/llfontfreetype.cpp
index d87fb5245c..f987c599ad 100644
--- a/indra/llrender/llfontfreetype.cpp
+++ b/indra/llrender/llfontfreetype.cpp
@@ -354,11 +354,10 @@ void LLFontFreetype::clearFontStreams()
 }
 #endif
 
-void LLFontFreetype::addFallbackFont(const LLPointer<LLFontFreetype>& fallback_font, const char_functor_t& functor)
+void LLFontFreetype::addFallbackFont(const LLPointer<LLFontFreetype>& fallback_font,
+                                     const char_functor_t& functor)
 {
-	// Insert functor fallbacks before generic fallbacks
-	mFallbackFonts.insert((functor) ? std::find_if(mFallbackFonts.begin(), mFallbackFonts.end(), [](const fallback_font_t& fe) { return !fe.second; }) : mFallbackFonts.end(),
-	                      std::make_pair(fallback_font, functor));
+    mFallbackFonts.emplace_back(fallback_font, functor);
 }
 
 F32 LLFontFreetype::getLineHeight() const
@@ -450,55 +449,100 @@ BOOL LLFontFreetype::hasGlyph(llwchar wch) const
 
 LLFontGlyphInfo* LLFontFreetype::addGlyph(llwchar wch, EFontGlyphType glyph_type) const
 {
-	if (mFTFace == NULL)
-		return FALSE;
-
-	llassert(!mIsFallback);
-	llassert(glyph_type < EFontGlyphType::Count);
-	//LL_DEBUGS() << "Adding new glyph for " << wch << " to font" << LL_ENDL;
-
-	FT_UInt glyph_index;
-
-	// Fallback fonts with a functor have precedence over everything else
-	fallback_font_vector_t::const_iterator it_fallback = mFallbackFonts.cbegin();
-	/* This leads to a bug SL-19831 "Check marks in the menu are less visible."
-	** Also, LLFontRegistry::createFont() says: "Fallback fonts don't render"
-	for (; it_fallback != mFallbackFonts.cend() && it_fallback->second; ++it_fallback)
-	{
-		if (it_fallback->second(wch))
-		{
-			glyph_index = FT_Get_Char_Index(it_fallback->first->mFTFace, wch);
-			if (glyph_index)
-			{
-				return addGlyphFromFont(it_fallback->first, wch, glyph_index, glyph_type);
-			}
-		}
-	}
-	*/
-
-	// Initialize char to glyph map
-	glyph_index = FT_Get_Char_Index(mFTFace, wch);
-	if (glyph_index == 0)
-	{
-		//LL_INFOS() << "Trying to add glyph from fallback font!" << LL_ENDL;
-		for (; it_fallback != mFallbackFonts.cend(); ++it_fallback)
-		{
-			glyph_index = FT_Get_Char_Index(it_fallback->first->mFTFace, wch);
-			if (glyph_index)
-			{
-				return addGlyphFromFont(it_fallback->first, wch, glyph_index, glyph_type);
-			}
-		}
-	}
-	
-	std::pair<char_glyph_info_map_t::iterator, char_glyph_info_map_t::iterator> range_it = mCharGlyphInfoMap.equal_range(wch);
-	char_glyph_info_map_t::iterator iter = 
-		std::find_if(range_it.first, range_it.second, [&glyph_type](const char_glyph_info_map_t::value_type& entry) { return entry.second->mGlyphType == glyph_type; });
-	if (iter == range_it.second)
-	{
-		return addGlyphFromFont(this, wch, glyph_index, glyph_type);
-	}
-	return NULL;
+    if (!mFTFace)
+    {
+        return NULL;
+    }
+
+    llassert(!mIsFallback);
+    llassert(glyph_type < EFontGlyphType::Count);
+    //LL_DEBUGS() << "Adding new glyph for " << wch << " to font" << LL_ENDL;
+
+    // Initialize char to glyph map
+    FT_UInt glyph_index = FT_Get_Char_Index(mFTFace, wch);
+    if (glyph_index == 0)
+    {
+        // No corresponding glyph in this font: look for a glyph in fallback
+        // fonts.
+        size_t count = mFallbackFonts.size();
+        if (LLStringOps::isEmoji(wch))
+        {
+            // This is a "genuine" emoji (in the range 0x1f000-0x20000): print
+            // it using the emoji font(s) if possible. HB
+            for (size_t i = 0; i < count; ++i)
+            {
+                const fallback_font_t& pair = mFallbackFonts[i];
+                if (!pair.second || !pair.second(wch))
+                {
+                    // If this font does not have a functor, or the character
+                    // does not pass the functor, reject it. Note: we keep the
+                    // functor test (despite the fact we already tested for
+                    // LLStringOps::isEmoji(wch) above), in case we would use
+                    // different, more restrictive or partionned functors in
+                    // the future with several different emoji fonts. HB
+                    continue;
+                }
+                glyph_index = FT_Get_Char_Index(pair.first->mFTFace, wch);
+                if (glyph_index)
+                {
+                    return addGlyphFromFont(pair.first, wch, glyph_index,
+                                            glyph_type);
+                }
+            }
+        }
+        // Then try and find a monochrome fallback font that could print this
+        // glyph: such fonts do *not* have a functor. We give priority to
+        // monochrome fonts for non-genuine emojis so that UI elements which
+        // used to render with them before the emojis font introduction (e.g.
+        // check marks in menus, or LSL dialogs text and buttons) do render the
+        // same way as they always did. HB
+        std::vector<size_t> emoji_fonts_idx;
+        for (size_t i = 0; i < count; ++i)
+        {
+            const fallback_font_t& pair = mFallbackFonts[i];
+            if (pair.second)
+            {
+                // If this font got a functor, remember the index for later and
+                // try the next fallback font. HB
+                emoji_fonts_idx.push_back(i);
+                continue;
+            }
+            glyph_index = FT_Get_Char_Index(pair.first->mFTFace, wch);
+            if (glyph_index)
+            {
+                return addGlyphFromFont(pair.first, wch, glyph_index,
+                                        glyph_type);
+            }
+        }
+        // Everything failed so far: this character is not a genuine emoji,
+        // neither a special character known from our monochrome fallback
+        // fonts: make a last try, using the emoji font(s), but ignoring the
+        // functor to render using whatever (colorful) glyph that might be
+        // available in such fonts for this character. HB
+        for (size_t j = 0, count2 = emoji_fonts_idx.size(); j < count2; ++j)
+        {
+            const fallback_font_t& pair = mFallbackFonts[emoji_fonts_idx[j]];
+            glyph_index = FT_Get_Char_Index(pair.first->mFTFace, wch);
+            if (glyph_index)
+            {
+                return addGlyphFromFont(pair.first, wch, glyph_index,
+                                        glyph_type);
+            }
+        }
+    }
+
+    auto range_it = mCharGlyphInfoMap.equal_range(wch);
+    char_glyph_info_map_t::iterator iter = 
+        std::find_if(range_it.first, range_it.second,
+                     [&glyph_type](const char_glyph_info_map_t::value_type& entry)
+                     {
+                        return entry.second->mGlyphType == glyph_type;
+                     });
+    if (iter == range_it.second)
+    {
+        return addGlyphFromFont(this, wch, glyph_index, glyph_type);
+    }
+    return NULL;
 }
 
 LLFontGlyphInfo* LLFontFreetype::addGlyphFromFont(const LLFontFreetype *fontp, llwchar wch, U32 glyph_index, EFontGlyphType requested_glyph_type) const
diff --git a/indra/newview/CMakeLists.txt b/indra/newview/CMakeLists.txt
index 5e6b8c1b87..c01cd2c34f 100644
--- a/indra/newview/CMakeLists.txt
+++ b/indra/newview/CMakeLists.txt
@@ -18,7 +18,6 @@ include(DragDrop)
 include(EXPAT)
 include(FMODSTUDIO)
 include(Hunspell)
-include(ICU4C)
 include(JPEGEncoderBasic)
 include(JsonCpp)
 include(LLAppearance)
@@ -1934,7 +1933,6 @@ target_link_libraries(${VIEWER_BINARY_NAME}
         ${LLPHYSICSEXTENSIONS_LIBRARIES}
         ll::bugsplat
         ll::tracy
-        ll::icu4c
         )
 
 if( TARGET ll::intel_memops )
diff --git a/indra/newview/viewer_manifest.py b/indra/newview/viewer_manifest.py
index c7f32d0da9..3efd88964d 100755
--- a/indra/newview/viewer_manifest.py
+++ b/indra/newview/viewer_manifest.py
@@ -559,10 +559,6 @@ class Windows_x86_64_Manifest(ViewerManifest):
                 self.path("OpenAL32.dll")
                 self.path("alut.dll")
 
-            # For ICU4C
-            self.path("icudt48.dll")
-            self.path("icuuc48.dll")
-
             # For textures
             self.path("openjp2.dll")