DRTVWR-476, SL-13528: Use freopen_s() instead of assigning stderr.

The llappviewerwin32.cpp create_console() function called by
LLAppViewerWin32::initConsole() used to assign *stderr = *(new FILE* value),
and so forth for stdout and stdin. That dubious tactic no longer works with
the new Windows CRT introduced with VS 2015. freopen_s() works much better.

1 files changed, 51 insertions, 49 deletions

diff --git a/indra/newview/llappviewerwin32.cpp b/indra/newview/llappviewerwin32.cpp
index f0aa355342..156a1c5893 100644
--- a/indra/newview/llappviewerwin32.cpp
+++ b/indra/newview/llappviewerwin32.cpp
@@ -507,63 +507,65 @@ const S32 MAX_CONSOLE_LINES = 500;
 
 namespace {
 
-FILE* set_stream(const char* which, DWORD handle_id, const char* mode);
+void set_stream(const char* desc, FILE* fp, DWORD handle_id, const char* name, const char* mode="w");
 
 bool create_console()
 {
-	// allocate a console for this app
-	const bool isConsoleAllocated = AllocConsole();
-
-	// set the screen buffer to be big enough to let us scroll text
-	CONSOLE_SCREEN_BUFFER_INFO coninfo;
-	GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &coninfo);
-	coninfo.dwSize.Y = MAX_CONSOLE_LINES;
-	SetConsoleScreenBufferSize(GetStdHandle(STD_OUTPUT_HANDLE), coninfo.dwSize);
-
-	// redirect unbuffered STDOUT to the console
-	FILE* fp = set_stream("stdout", STD_OUTPUT_HANDLE, "w");
-	if (fp)
-	{
-		*stdout = *fp;
-	}
-
-	// redirect unbuffered STDIN to the console
-	fp = set_stream("stdin", STD_INPUT_HANDLE, "r");
-	if (fp)
-	{
-		*stdin = *fp;
-	}
+    // allocate a console for this app
+    const bool isConsoleAllocated = AllocConsole();
 
-	// redirect unbuffered STDERR to the console
-	fp = set_stream("stderr", STD_ERROR_HANDLE, "w");
-	if (fp)
-	{
-		*stderr = *fp;
-	}
+    if (isConsoleAllocated)
+    {
+        // set the screen buffer to be big enough to let us scroll text
+        CONSOLE_SCREEN_BUFFER_INFO coninfo;
+        GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &coninfo);
+        coninfo.dwSize.Y = MAX_CONSOLE_LINES;
+        SetConsoleScreenBufferSize(GetStdHandle(STD_OUTPUT_HANDLE), coninfo.dwSize);
+
+        // redirect unbuffered STDOUT to the console
+        set_stream("stdout", stdout, STD_OUTPUT_HANDLE, "CONOUT$");
+        // redirect unbuffered STDERR to the console
+        set_stream("stderr", stderr, STD_ERROR_HANDLE, "CONOUT$");
+        // redirect unbuffered STDIN to the console
+        // Don't bother: our console is solely for log output. We never read stdin.
+//      set_stream("stdin", stdin, STD_INPUT_HANDLE, "CONIN$", "r");
+    }
 
-	return isConsoleAllocated;
+    return isConsoleAllocated;
 }
 
-FILE* set_stream(const char* desc, DWORD handle_id, const char* mode)
+void set_stream(const char* desc, FILE* fp, DWORD handle_id, const char* name, const char* mode)
 {
-	auto l_std_handle = GetStdHandle(handle_id);
-	int h_con_handle = _open_osfhandle(reinterpret_cast<intptr_t>(l_std_handle), _O_TEXT);
-	if (h_con_handle == -1)
-	{
-		LL_WARNS() << "create_console() failed to open " << desc << " handle" << LL_ENDL;
-		return nullptr;
-	}
-	else
-	{
-		FILE* fp = _fdopen( h_con_handle, mode );
-		setvbuf( fp, NULL, _IONBF, 0 );
-		// Enable color processing on Windows 10 console windows.
-		DWORD dwMode = 0;
-		GetConsoleMode(l_std_handle, &dwMode);
-		dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-		SetConsoleMode(l_std_handle, dwMode);
-		return fp;
-	}
+    // SL-13528: This code used to be based on
+    // http://dslweb.nwnexus.com/~ast/dload/guicon.htm
+    // (referenced in https://stackoverflow.com/a/191880).
+    // But one of the comments on that StackOverflow answer points out that
+    // assigning to *stdout or *stderr "probably doesn't even work with the
+    // Universal CRT that was introduced in 2015," suggesting freopen_s()
+    // instead. Code below is based on https://stackoverflow.com/a/55875595.
+    auto std_handle = GetStdHandle(handle_id);
+    if (std_handle == INVALID_HANDLE_VALUE)
+    {
+        LL_WARNS() << "create_console() failed to get " << desc << " handle" << LL_ENDL;
+    }
+    else
+    {
+        if (mode == std::string("w"))
+        {
+            // Enable color processing on Windows 10 console windows.
+            DWORD dwMode = 0;
+            GetConsoleMode(std_handle, &dwMode);
+            dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+            SetConsoleMode(std_handle, dwMode);
+        }
+        // Redirect the passed fp to the console.
+        FILE* ignore;
+        if (freopen_s(&ignore, name, mode, fp) == 0)
+        {
+            // use unbuffered I/O
+            setvbuf( fp, NULL, _IONBF, 0 );
+        }
+    }
 }
 
 } // anonymous namespace