diff options
Diffstat (limited to 'indra/llcommon/llstring.cpp')
| -rw-r--r-- | indra/llcommon/llstring.cpp | 835 |
1 files changed, 835 insertions, 0 deletions
diff --git a/indra/llcommon/llstring.cpp b/indra/llcommon/llstring.cpp new file mode 100644 index 0000000000..50fd881ad7 --- /dev/null +++ b/indra/llcommon/llstring.cpp @@ -0,0 +1,835 @@ +/** + * @file llstring.cpp + * @brief String utility functions and the LLString class. + * + * Copyright (c) 2001-$CurrentYear$, Linden Research, Inc. + * $License$ + */ + +#include "linden_common.h" + +#include "llstring.h" +#include "llerror.h" + +std::string ll_safe_string(const char* in) +{ + if(in) return std::string(in); + return std::string(); +} + +U8 hex_as_nybble(char hex) +{ + if((hex >= '0') && (hex <= '9')) + { + return (U8)(hex - '0'); + } + else if((hex >= 'a') && (hex <='f')) + { + return (U8)(10 + hex - 'a'); + } + else if((hex >= 'A') && (hex <='F')) + { + return (U8)(10 + hex - 'A'); + } + return 0; // uh - oh, not hex any more... +} + + +// See http://www.unicode.org/Public/BETA/CVTUTF-1-2/ConvertUTF.c +// for the Unicode implementation - this doesn't match because it was written before finding +// it. + + +std::ostream& operator<<(std::ostream &s, const LLWString &wstr) +{ + std::string utf8_str = wstring_to_utf8str(wstr); + s << utf8_str; + return s; +} + +std::string rawstr_to_utf8(const std::string& raw) +{ + LLWString wstr(utf8str_to_wstring(raw)); + return wstring_to_utf8str(wstr); +} + +S32 wchar_to_utf8chars(llwchar in_char, char* outchars) +{ + U32 cur_char = (U32)in_char; + char* base = outchars; + if (cur_char < 0x80) + { + *outchars++ = (U8)cur_char; + } + else if (cur_char < 0x800) + { + *outchars++ = 0xC0 | (cur_char >> 6); + *outchars++ = 0x80 | (cur_char & 0x3F); + } + else if (cur_char < 0x10000) + { + *outchars++ = 0xE0 | (cur_char >> 12); + *outchars++ = 0x80 | ((cur_char >> 6) & 0x3F); + *outchars++ = 0x80 | (cur_char & 0x3F); + } + else if (cur_char < 0x200000) + { + *outchars++ = 0xF0 | (cur_char >> 18); + *outchars++ = 0x80 | ((cur_char >> 12) & 0x3F); + *outchars++ = 0x80 | ((cur_char >> 6) & 0x3F); + *outchars++ = 0x80 | cur_char & 0x3F; + } + else if (cur_char < 0x4000000) + { + *outchars++ = 0xF8 | (cur_char >> 24); + *outchars++ = 0x80 | ((cur_char >> 18) & 0x3F); + *outchars++ = 0x80 | ((cur_char >> 12) & 0x3F); + *outchars++ = 0x80 | ((cur_char >> 6) & 0x3F); + *outchars++ = 0x80 | cur_char & 0x3F; + } + else if (cur_char < 0x80000000) + { + *outchars++ = 0xFC | (cur_char >> 30); + *outchars++ = 0x80 | ((cur_char >> 24) & 0x3F); + *outchars++ = 0x80 | ((cur_char >> 18) & 0x3F); + *outchars++ = 0x80 | ((cur_char >> 12) & 0x3F); + *outchars++ = 0x80 | ((cur_char >> 6) & 0x3F); + *outchars++ = 0x80 | cur_char & 0x3F; + } + else + { + llwarns << "Invalid Unicode character " << cur_char << "!" << llendl; + *outchars++ = LL_UNKNOWN_CHAR; + } + return outchars - base; +} + +S32 utf16chars_to_wchar(const U16* inchars, llwchar* outchar) +{ + const U16* base = inchars; + U16 cur_char = *inchars++; + llwchar char32 = cur_char; + if ((cur_char >= 0xD800) && (cur_char <= 0xDFFF)) + { + // Surrogates + char32 = ((llwchar)(cur_char - 0xD800)) << 10; + cur_char = *inchars++; + char32 += (llwchar)(cur_char - 0xDC00) + 0x0010000UL; + } + else + { + char32 = (llwchar)cur_char; + } + *outchar = char32; + return inchars - base; +} + +S32 utf16chars_to_utf8chars(const U16* inchars, char* outchars, S32* nchars8p) +{ + // Get 32 bit char32 + llwchar char32; + S32 nchars16 = utf16chars_to_wchar(inchars, &char32); + // Convert to utf8 + S32 nchars8 = wchar_to_utf8chars(char32, outchars); + if (nchars8p) + { + *nchars8p = nchars8; + } + return nchars16; +} + +llutf16string wstring_to_utf16str(const LLWString &utf32str, S32 len) +{ + llutf16string out; + + S32 i = 0; + while (i < len) + { + U32 cur_char = utf32str[i]; + if (cur_char > 0xFFFF) + { + out += (0xD7C0 + (cur_char >> 10)); + out += (0xDC00 | (cur_char & 0x3FF)); + } + else + { + out += cur_char; + } + i++; + } + return out; +} + +llutf16string wstring_to_utf16str(const LLWString &utf32str) +{ + const S32 len = (S32)utf32str.length(); + return wstring_to_utf16str(utf32str, len); +} + +llutf16string utf8str_to_utf16str ( const LLString& utf8str ) +{ + LLWString wstr = utf8str_to_wstring ( utf8str ); + return wstring_to_utf16str ( wstr ); +} + + +LLWString utf16str_to_wstring(const llutf16string &utf16str, S32 len) +{ + LLWString wout; + + S32 i = 0; + // craziness to make gcc happy (llutf16string.c_str() is tweaked on linux): + const U16* chars16 = &(*(utf16str.begin())); + while (i < len) + { + llwchar cur_char; + i += utf16chars_to_wchar(chars16+i, &cur_char); + wout += cur_char; + } + return wout; +} + +LLWString utf16str_to_wstring(const llutf16string &utf16str) +{ + const S32 len = (S32)utf16str.length(); + return utf16str_to_wstring(utf16str, len); +} + +S32 wchar_utf8_length(const llwchar wc) +{ + if (wc < 0x80) + { + // This case will also catch negative values which are + // technically invalid. + return 1; + } + else if (wc < 0x800) + { + return 2; + } + else if (wc < 0x10000) + { + return 3; + } + else if (wc < 0x200000) + { + return 4; + } + else if (wc < 0x4000000) + { + return 5; + } + else + { + return 6; + } +} + + +S32 wstring_utf8_length(const LLWString& wstr) +{ + S32 len = 0; + for (S32 i = 0; i < (S32)wstr.length(); i++) + { + len += wchar_utf8_length(wstr[i]); + } + return len; +} + + +LLWString utf8str_to_wstring(const std::string& utf8str, S32 len) +{ + LLWString wout; + + S32 i = 0; + while (i < len) + { + llwchar unichar; + U8 cur_char = utf8str[i]; + + if (cur_char < 0x80) + { + // Ascii character, just add it + unichar = cur_char; + } + else + { + S32 cont_bytes = 0; + if ((cur_char >> 5) == 0x6) // Two byte UTF8 -> 1 UTF32 + { + unichar = (0x1F&cur_char); + cont_bytes = 1; + } + else if ((cur_char >> 4) == 0xe) // Three byte UTF8 -> 1 UTF32 + { + unichar = (0x0F&cur_char); + cont_bytes = 2; + } + else if ((cur_char >> 3) == 0x1e) // Four byte UTF8 -> 1 UTF32 + { + unichar = (0x07&cur_char); + cont_bytes = 3; + } + else if ((cur_char >> 2) == 0x3e) // Five byte UTF8 -> 1 UTF32 + { + unichar = (0x03&cur_char); + cont_bytes = 4; + } + else if ((cur_char >> 1) == 0x7e) // Six byte UTF8 -> 1 UTF32 + { + unichar = (0x01&cur_char); + cont_bytes = 5; + } + else + { + wout += LL_UNKNOWN_CHAR; + ++i; + continue; + } + + // Check that this character doesn't go past the end of the string + S32 end = (len < (i + cont_bytes)) ? len : (i + cont_bytes); + do + { + ++i; + + cur_char = utf8str[i]; + if ( (cur_char >> 6) == 0x2 ) + { + unichar <<= 6; + unichar += (0x3F&cur_char); + } + else + { + // Malformed sequence - roll back to look at this as a new char + unichar = LL_UNKNOWN_CHAR; + --i; + break; + } + } while(i < end); + + // Handle overlong characters and NULL characters + if ( ((cont_bytes == 1) && (unichar < 0x80)) + || ((cont_bytes == 2) && (unichar < 0x800)) + || ((cont_bytes == 3) && (unichar < 0x10000)) + || ((cont_bytes == 4) && (unichar < 0x200000)) + || ((cont_bytes == 5) && (unichar < 0x4000000)) ) + { + unichar = LL_UNKNOWN_CHAR; + } + } + + wout += unichar; + ++i; + } + return wout; +} + +LLWString utf8str_to_wstring(const std::string& utf8str) +{ + const S32 len = (S32)utf8str.length(); + return utf8str_to_wstring(utf8str, len); +} + +std::string wstring_to_utf8str(const LLWString& utf32str, S32 len) +{ + std::string out; + + S32 i = 0; + while (i < len) + { + char tchars[8]; /* Flawfinder: ignore */ + S32 n = wchar_to_utf8chars(utf32str[i], tchars); + tchars[n] = 0; + out += tchars; + i++; + } + return out; +} + +std::string wstring_to_utf8str(const LLWString& utf32str) +{ + const S32 len = (S32)utf32str.length(); + return wstring_to_utf8str(utf32str, len); +} + +std::string utf16str_to_utf8str(const llutf16string& utf16str) +{ + return wstring_to_utf8str(utf16str_to_wstring(utf16str)); +} + +std::string utf16str_to_utf8str(const llutf16string& utf16str, S32 len) +{ + return wstring_to_utf8str(utf16str_to_wstring(utf16str, len), len); +} + + +//LLWString wstring_truncate(const LLWString &wstr, const S32 max_len) +//{ +// return wstr.substr(0, llmin((S32)wstr.length(), max_len)); +//} +// +// +//LLWString wstring_trim(const LLWString &wstr) +//{ +// LLWString outstr; +// outstr = wstring_trimhead(wstr); +// outstr = wstring_trimtail(outstr); +// return outstr; +//} +// +// +//LLWString wstring_trimhead(const LLWString &wstr) +//{ +// if(wstr.empty()) +// { +// return wstr; +// } +// +// S32 i = 0; +// while((i < (S32)wstr.length()) && iswspace(wstr[i])) +// { +// i++; +// } +// return wstr.substr(i, wstr.length() - i); +//} +// +// +//LLWString wstring_trimtail(const LLWString &wstr) +//{ +// if(wstr.empty()) +// { +// return wstr; +// } +// +// S32 len = (S32)wstr.length(); +// +// S32 i = len - 1; +// while (i >= 0 && iswspace(wstr[i])) +// { +// i--; +// } +// +// if (i >= 0) +// { +// return wstr.substr(0, i + 1); +// } +// return wstr; +//} +// +// +//LLWString wstring_copyinto(const LLWString &dest, const LLWString &src, const S32 insert_offset) +//{ +// llassert( insert_offset <= (S32)dest.length() ); +// +// LLWString out_str = dest.substr(0, insert_offset); +// out_str += src; +// LLWString tail = dest.substr(insert_offset); +// out_str += tail; +// +// return out_str; +//} + + +//LLWString wstring_detabify(const LLWString &wstr, const S32 num_spaces) +//{ +// LLWString out_str; +// // Replace tabs with spaces +// for (S32 i = 0; i < (S32)wstr.length(); i++) +// { +// if (wstr[i] == '\t') +// { +// for (S32 j = 0; j < num_spaces; j++) +// out_str += ' '; +// } +// else +// { +// out_str += wstr[i]; +// } +// } +// return out_str; +//} + + +//LLWString wstring_makeASCII(const LLWString &wstr) +//{ +// // Replace non-ASCII chars with replace_char +// LLWString out_str = wstr; +// for (S32 i = 0; i < (S32)out_str.length(); i++) +// { +// if (out_str[i] > 0x7f) +// { +// out_str[i] = LL_UNKNOWN_CHAR; +// } +// } +// return out_str; +//} + + +//LLWString wstring_substChar(const LLWString &wstr, const llwchar target_char, const llwchar replace_char) +//{ +// // Replace all occurences of target_char with replace_char +// LLWString out_str = wstr; +// for (S32 i = 0; i < (S32)out_str.length(); i++) +// { +// if (out_str[i] == target_char) +// { +// out_str[i] = replace_char; +// } +// } +// return out_str; +//} +// +// +//LLWString wstring_tolower(const LLWString &wstr) +//{ +// LLWString out_str = wstr; +// for (S32 i = 0; i < (S32)out_str.length(); i++) +// { +// out_str[i] = towlower(out_str[i]); +// } +// return out_str; +//} +// +// +//LLWString wstring_convert_to_lf(const LLWString &wstr) +//{ +// const llwchar CR = 13; +// // Remove carriage returns from string with CRLF +// LLWString out_str; +// +// for (S32 i = 0; i < (S32)wstr.length(); i++) +// { +// if (wstr[i] != CR) +// { +// out_str += wstr[i]; +// } +// } +// return out_str; +//} +// +// +//LLWString wstring_convert_to_crlf(const LLWString &wstr) +//{ +// const llwchar LF = 10; +// const llwchar CR = 13; +// // Remove carriage returns from string with CRLF +// LLWString out_str; +// +// for (S32 i = 0; i < (S32)wstr.length(); i++) +// { +// if (wstr[i] == LF) +// { +// out_str += CR; +// } +// out_str += wstr[i]; +// } +// return out_str; +//} + + +//S32 wstring_compare_insensitive(const LLWString &lhs, const LLWString &rhs) +//{ +// +// if (lhs == rhs) +// { +// return 0; +// } +// +// if (lhs.empty()) +// { +// return rhs.empty() ? 0 : 1; +// } +// +// if (rhs.empty()) +// { +// return -1; +// } +// +//#ifdef LL_LINUX +// // doesn't work because gcc 2.95 doesn't correctly implement c_str(). Sigh... +// llerrs << "wstring_compare_insensitive doesn't work on Linux!" << llendl; +// return 0; +//#else +// LLWString lhs_lower = lhs; +// LLWString::toLower(lhs_lower); +// std::string lhs_lower = wstring_to_utf8str(lhs_lower); +// LLWString rhs_lower = lhs; +// LLWString::toLower(rhs_lower); +// std::string rhs_lower = wstring_to_utf8str(rhs_lower); +// +// return strcmp(lhs_lower.c_str(), rhs_lower.c_str()); +//#endif +//} + + +std::string utf8str_trim(const std::string& utf8str) +{ + LLWString wstr = utf8str_to_wstring(utf8str); + LLWString::trim(wstr); + return wstring_to_utf8str(wstr); +} + + +std::string utf8str_tolower(const std::string& utf8str) +{ + LLWString out_str = utf8str_to_wstring(utf8str); + LLWString::toLower(out_str); + return wstring_to_utf8str(out_str); +} + + +S32 utf8str_compare_insensitive(const std::string& lhs, const std::string& rhs) +{ + LLWString wlhs = utf8str_to_wstring(lhs); + LLWString wrhs = utf8str_to_wstring(rhs); + return LLWString::compareInsensitive(wlhs.c_str(), wrhs.c_str()); +} + +std::string utf8str_truncate(const std::string& utf8str, const S32 max_len) +{ + if (0 == max_len) + { + return std::string(); + } + if ((S32)utf8str.length() <= max_len) + { + return utf8str; + } + else + { + S32 cur_char = max_len; + + // If we're ASCII, we don't need to do anything + if ((U8)utf8str[cur_char] > 0x7f) + { + // If first two bits are (10), it's the tail end of a multibyte char. We need to shift back + // to the first character + while (0x80 == (0xc0 & utf8str[cur_char])) + { + cur_char--; + // Keep moving forward until we hit the first char; + if (cur_char == 0) + { + // Make sure we don't trash memory if we've got a bogus string. + break; + } + } + } + // The byte index we're on is one we want to get rid of, so we only want to copy up to (cur_char-1) chars + return utf8str.substr(0, cur_char); + } +} + +std::string utf8str_substChar( + const std::string& utf8str, + const llwchar target_char, + const llwchar replace_char) +{ + LLWString wstr = utf8str_to_wstring(utf8str); + LLWString::replaceChar(wstr, target_char, replace_char); + //wstr = wstring_substChar(wstr, target_char, replace_char); + return wstring_to_utf8str(wstr); +} + +std::string utf8str_makeASCII(const std::string& utf8str) +{ + LLWString wstr = utf8str_to_wstring(utf8str); + LLWString::_makeASCII(wstr); + return wstring_to_utf8str(wstr); +} + +std::string mbcsstring_makeASCII(const std::string& wstr) +{ + // Replace non-ASCII chars with replace_char + std::string out_str = wstr; + for (S32 i = 0; i < (S32)out_str.length(); i++) + { + if ((U8)out_str[i] > 0x7f) + { + out_str[i] = LL_UNKNOWN_CHAR; + } + } + return out_str; +} + +S32 LLStringOps::collate(const llwchar* a, const llwchar* b) +{ + #if LL_WINDOWS + // in Windows, wide string functions operator on 16-bit strings, + // not the proper 32 bit wide string + return strcmp(wstring_to_utf8str(LLWString(a)).c_str(), wstring_to_utf8str(LLWString(b)).c_str()); + #else + return wcscoll(a, b); + #endif +} + +namespace LLStringFn +{ + void replace_nonprintable(std::basic_string<char>& string, char replacement) + { + const char MIN = 0x20; + std::basic_string<char>::size_type len = string.size(); + for(std::basic_string<char>::size_type ii = 0; ii < len; ++ii) + { + if(string[ii] < MIN) + { + string[ii] = replacement; + } + } + } + + void replace_nonprintable( + std::basic_string<llwchar>& string, + llwchar replacement) + { + const llwchar MIN = 0x20; + const llwchar MAX = 0x7f; + std::basic_string<llwchar>::size_type len = string.size(); + for(std::basic_string<llwchar>::size_type ii = 0; ii < len; ++ii) + { + if((string[ii] < MIN) || (string[ii] > MAX)) + { + string[ii] = replacement; + } + } + } + + void replace_nonprintable_and_pipe(std::basic_string<char>& str, + char replacement) + { + const char MIN = 0x20; + const char PIPE = 0x7c; + std::basic_string<char>::size_type len = str.size(); + for(std::basic_string<char>::size_type ii = 0; ii < len; ++ii) + { + if( (str[ii] < MIN) || (str[ii] == PIPE) ) + { + str[ii] = replacement; + } + } + } + + void replace_nonprintable_and_pipe(std::basic_string<llwchar>& str, + llwchar replacement) + { + const llwchar MIN = 0x20; + const llwchar MAX = 0x7f; + const llwchar PIPE = 0x7c; + std::basic_string<llwchar>::size_type len = str.size(); + for(std::basic_string<llwchar>::size_type ii = 0; ii < len; ++ii) + { + if( (str[ii] < MIN) || (str[ii] > MAX) || (str[ii] == PIPE) ) + { + str[ii] = replacement; + } + } + } +} + + +//////////////////////////////////////////////////////////// +// Testing + +#ifdef _DEBUG + +template<class T> +void LLStringBase<T>::testHarness() +{ + LLString s1; + + llassert( s1.c_str() == NULL ); + llassert( s1.size() == 0 ); + llassert( s1.empty() ); + + LLString s2( "hello"); + llassert( !strcmp( s2.c_str(), "hello" ) ); + llassert( s2.size() == 5 ); + llassert( !s2.empty() ); + LLString s3( s2 ); + + llassert( "hello" == s2 ); + llassert( s2 == "hello" ); + llassert( s2 > "gello" ); + llassert( "gello" < s2 ); + llassert( "gello" != s2 ); + llassert( s2 != "gello" ); + + LLString s4 = s2; + llassert( !s4.empty() ); + s4.empty(); + llassert( s4.empty() ); + + LLString s5(""); + llassert( s5.empty() ); + + llassert( isValidIndex(s5, 0) ); + llassert( !isValidIndex(s5, 1) ); + + s3 = s2; + s4 = "hello again"; + + s4 += "!"; + s4 += s4; + llassert( s4 == "hello again!hello again!" ); + + + LLString s6 = s2 + " " + s2; + LLString s7 = s6; + llassert( s6 == s7 ); + llassert( !( s6 != s7) ); + llassert( !(s6 < s7) ); + llassert( !(s6 > s7) ); + + llassert( !(s6 == "hi")); + llassert( s6 == "hello hello"); + llassert( s6 < "hi"); + + llassert( s6[1] == 'e' ); + s6[1] = 'f'; + llassert( s6[1] == 'f' ); + + s2.erase( 4, 1 ); + llassert( s2 == "hell"); + s2.insert( 0, 'y' ); + llassert( s2 == "yhell"); + s2.erase( 1, 3 ); + llassert( s2 == "yl"); + s2.insert( 1, "awn, don't yel"); + llassert( s2 == "yawn, don't yell"); + + LLString s8 = s2.substr( 6, 5 ); + llassert( s8 == "don't" ); + + LLString s9 = " \t\ntest \t\t\n "; + trim(s9); + llassert( s9 == "test" ); + + s8 = "abc123&*(ABC"; + + s9 = s8; + toUpper(s9); + llassert( s9 == "ABC123&*(ABC" ); + + s9 = s8; + toLower(s9); + llassert( s9 == "abc123&*(abc" ); + + + LLString s10( 10, 'x' ); + llassert( s10 == "xxxxxxxxxx" ); + + LLString s11( "monkey in the middle", 7, 2 ); + llassert( s11 == "in" ); + + LLString s12; //empty + s12 += "foo"; + llassert( s12 == "foo" ); + + LLString s13; //empty + s13 += 'f'; + llassert( s13 == "f" ); +} + + +#endif // _DEBUG |