From af5c5a994b90a27e16ef6f2f5044e096269e4217 Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Wed, 27 Oct 2021 13:01:37 -0400 Subject: SL-16207: Update llstring.h handling of different string types. In llpreprocessor.h, consider the case of clang on Windows: #define LL_WCHAR_T_NATIVE there as well as for the Microsoft compiler with /Zc:wchar_t switch. In stdtypes.h, inject a LLWCHAR_IS_WCHAR_T symbol to allow the preprocessor to make decisions about when the types are identical. llstring.h's conversion logic deals with three types of wide strings (LLWString, std::wstring and utf16string) based on three types of wide char (llwchar, wchar_t and U16, respectively). Sometimes they're three distinct types, sometimes wchar_t is identical to llwchar and sometimes wchar_t is identical to U16. Rationalize the three cases using ll_convert_u16_alias() and new ll_convert_wstr_alias() macros. stringize.h was directly calling wstring_to_utf8str() and utf8str_to_wstring(), which was producing errors with VS 2019 clang since there isn't actually a wstring_to_utf8str(std::wstring) overload. Use ll_convert() instead, since that redirects to the relevant ll_convert_wide_to_string() function. (And now you see why we've been trying to migrate to the uniform ll_convert() wrapper!) Similarly, call ll_convert() instead of a two-step conversion from utf8str_to_wstring(), producing LLWString, then a character-by-character copy from LLWString to std::wstring. That isn't even correct: on Windows, we should be encoding from UTF32 to UTF16. --- indra/llcommon/stringize.h | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'indra/llcommon/stringize.h') diff --git a/indra/llcommon/stringize.h b/indra/llcommon/stringize.h index 38dd198ad3..31a114f167 100644 --- a/indra/llcommon/stringize.h +++ b/indra/llcommon/stringize.h @@ -52,7 +52,7 @@ std::basic_string gstringize(const T& item) */ inline std::string stringize(const std::wstring& item) { - return wstring_to_utf8str(item); + return ll_convert(item); } /** @@ -72,8 +72,7 @@ inline std::wstring wstringize(const std::string& item) { // utf8str_to_wstring() returns LLWString, which isn't necessarily the // same as std::wstring - LLWString s(utf8str_to_wstring(item)); - return std::wstring(s.begin(), s.end()); + return ll_convert(item); } /** @@ -146,11 +145,9 @@ void destringize_f(std::basic_string const & str, Functor const & f) * std::istringstream in(str); * in >> item1 >> item2 >> item3 ... ; * @endcode - * @NOTE - once we get generic lambdas, we shouldn't need DEWSTRINGIZE() any - * more since DESTRINGIZE() should do the right thing with a std::wstring. But - * until then, the lambda we pass must accept the right std::basic_istream. */ -#define DESTRINGIZE(STR, EXPRESSION) (destringize_f((STR), [&](std::istream& in){in >> EXPRESSION;})) -#define DEWSTRINGIZE(STR, EXPRESSION) (destringize_f((STR), [&](std::wistream& in){in >> EXPRESSION;})) +#define DESTRINGIZE(STR, EXPRESSION) (destringize_f((STR), [&](auto& in){in >> EXPRESSION;})) +// legacy name, just use DESTRINGIZE() going forward +#define DEWSTRINGIZE(STR, EXPRESSION) DESTRINGIZE(STR, EXPRESSION) #endif /* ! defined(LL_STRINGIZE_H) */ -- cgit v1.2.3 From a32a45163d18f9b5998e469a356f870dbdb034ad Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 23 Nov 2021 09:58:54 -0500 Subject: SL-16094: Extend stringize() to support variadic arguments. It's useful to be able to say STRINGIZE(item0 << item1 << item2), and we use that a lot in our code base. But weird syntax aside, there are a couple advantages to being able to write stringize(item0, item1, item2). First, it allows stringize() to be used from within some other variadic function, without having to make that function a macro that accepts an arbitrary insertion-operator expression. There's no such thing as a member macro. Second, particularly for variadic functions, it allows us to optimize the single-argument case stringize(item0). A macro can't do that. When item0 is already a string of the desired char type, instead of streaming it into a std::ostringstream and retrieving it again, we can simply return the input string. When it's a pointer to the desired char type, we can directly construct the result string without the help of std::ostringstream. When it's a string of some other char type, we can engage ll_convert() to perform needed conversions. We generalize and optimize the generic gstringize() function, retaining the role of stringize() and wstringize() as thin wrappers that merely provide the desired char type. Optimizing the single-argument case requires separately defining gstringize() with two or more arguments: the general case. Then gstringize(arg) is delegated to a gstringize_impl class template so we can partially specialize to recognize a std::basic_string argument, as well as desired_char_type*. Both these specializations engage ll_convert(), which already handles the trivial case when no conversion is required. Use of ll_convert() in this role supercedes and generalizes the previous wstring_to_utf8str() and utf8str_to_wstring() overloads. Also introduce stream_to(std::ostream&, ...) to support variadic streaming to other destinations, e.g. a file, std::cout, ... --- indra/llcommon/stringize.h | 109 +++++++++++++++++++++++++++++++++------------ 1 file changed, 80 insertions(+), 29 deletions(-) (limited to 'indra/llcommon/stringize.h') diff --git a/indra/llcommon/stringize.h b/indra/llcommon/stringize.h index 38dd198ad3..bc91f188e4 100644 --- a/indra/llcommon/stringize.h +++ b/indra/llcommon/stringize.h @@ -31,58 +31,109 @@ #include #include +#include /** - * gstringize(item) encapsulates an idiom we use constantly, using - * operator<<(std::ostringstream&, TYPE) followed by std::ostringstream::str() - * or their wstring equivalents - * to render a string expressing some item. + * stream_to(std::ostream&, items, ...) streams each item in the parameter list + * to the passed std::ostream using the insertion operator <<. This can be + * used, for instance, to make a simple print() function, e.g.: + * + * @code + * template + * void print(Items&&... items) + * { + * stream_to(std::cout, std::forward(items)...); + * } + * @endcode */ -template -std::basic_string gstringize(const T& item) +// recursion tail +template +void stream_to(std::basic_ostream& out) {} +// stream one or more items +template +void stream_to(std::basic_ostream& out, T&& item, Items&&... items) { - std::basic_ostringstream out; - out << item; - return out.str(); + out << std::move(item); + stream_to(out, std::forward(items)...); } +// why we use function overloads, not function template specializations: +// http://www.gotw.ca/publications/mill17.htm + /** - *partial specialization of stringize for handling wstring - *TODO: we should have similar specializations for wchar_t[] but not until it is needed. + * gstringize(item, ...) encapsulates an idiom we use constantly, using + * operator<<(std::ostringstream&, TYPE) followed by std::ostringstream::str() + * or their wstring equivalents to render a string expressing one or more items. */ -inline std::string stringize(const std::wstring& item) +// two or more args - the case of a single argument is handled separately +template +auto gstringize(T0&& item0, T1&& item1, Items&&... items) { - return wstring_to_utf8str(item); + std::basic_ostringstream out; + stream_to(out, std::forward(item0), std::forward(item1), + std::forward(items)...); + return out.str(); } -/** - * Specialization of gstringize for std::string return types - */ -template -std::string stringize(const T& item) +// generic single argument: stream to out, as above +template +struct gstringize_impl { - return gstringize(item); + auto operator()(typename boost::call_traits::param_type arg) + { + std::basic_ostringstream out; + out << arg; + return out.str(); + } +}; + +// partially specialize for a single STRING argument - +// note that ll_convert(T) already handles the trivial case +template +struct gstringize_impl> +{ + auto operator()(const std::basic_string& arg) + { + return ll_convert>(arg); + } +}; + +// partially specialize for a single CHARTYPE* argument - +// since it's not a basic_string and we do want to optimize this common case +template +struct gstringize_impl +{ + auto operator()(const INCHAR* arg) + { + return ll_convert>(arg); + } +}; + +// gstringize(single argument) +template +auto gstringize(T&& item) +{ + // use decay so we don't require separate specializations for T, const + // T, T&, const T& ... + return gstringize_impl>()(std::forward(item)); } /** - * Specialization for generating wstring from string. - * Both a convenience function and saves a miniscule amount of overhead. + * Specialization of gstringize for std::string return types */ -inline std::wstring wstringize(const std::string& item) +template +auto stringize(Items&&... items) { - // utf8str_to_wstring() returns LLWString, which isn't necessarily the - // same as std::wstring - LLWString s(utf8str_to_wstring(item)); - return std::wstring(s.begin(), s.end()); + return gstringize(std::forward(items)...); } /** * Specialization of gstringize for std::wstring return types */ -template -std::wstring wstringize(const T& item) +template +auto wstringize(Items&&... items) { - return gstringize(item); + return gstringize(std::forward(items)...); } /** -- cgit v1.2.3 From adc2666dbb2444194a5df84711207def7eba074c Mon Sep 17 00:00:00 2001 From: Nat Goodspeed Date: Tue, 23 Nov 2021 10:11:16 -0500 Subject: SL-16094: Tweak llstring merge --- indra/llcommon/stringize.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'indra/llcommon/stringize.h') diff --git a/indra/llcommon/stringize.h b/indra/llcommon/stringize.h index 8501beb16d..12df693910 100644 --- a/indra/llcommon/stringize.h +++ b/indra/llcommon/stringize.h @@ -53,7 +53,7 @@ void stream_to(std::basic_ostream& out) {} template void stream_to(std::basic_ostream& out, T&& item, Items&&... items) { - out << std::move(item); + out << std::forward(item); stream_to(out, std::forward(items)...); } @@ -113,8 +113,8 @@ struct gstringize_impl template auto gstringize(T&& item) { - // use decay so we don't require separate specializations for T, const - // T, T&, const T& ... + // use decay so we don't require separate specializations + // for T, const T, T&, const T& ... return gstringize_impl>()(std::forward(item)); } -- cgit v1.2.3