clean commit of teamcity service message related changes of unit test reporting into a clean repo

1 files changed, 509 insertions, 129 deletions

diff --git a/indra/test/test.cpp b/indra/test/test.cpp
index 57797f9d9f..d6959a2662 100644
--- a/indra/test/test.cpp
+++ b/indra/test/test.cpp
@@ -56,62 +56,75 @@
 namespace tut
 {
 	std::string sSourceDir;
-
+	
     test_runner_singleton runner;
 }
 
 class LLTestCallback : public tut::callback
 {
 public:
-	LLTestCallback(bool verbose_mode, std::ostream *stream) :
-		mVerboseMode(verbose_mode),
-		mTotalTests(0),
-		mPassedTests(0),
-		mFailedTests(0),
-		mSkippedTests(0),
-		mStream(stream)
+	LLTestCallback(bool verbose_mode, std::ostream *stream, std::string suitename) :
+	mVerboseMode(verbose_mode),
+	mTotalTests(0),
+	mPassedTests(0),
+	mFailedTests(0),
+	mSkippedTests(0),
+	mStream(stream),
+	suite_name(suitename)
 	{
 	}
-
-	void run_started()
+	
+	LLTestCallback()
+	{
+	}	
+	
+	virtual void run_started()
 	{
 		//std::cout << "run_started" << std::endl;
 	}
-
-	void test_completed(const tut::test_result& tr)
+	
+	virtual void group_started(const std::string& name) {
+		std::cout << "group_started name=" << name << std::endl;
+	}
+	
+	virtual void group_completed(const std::string& name) {
+		std::cout << "group_completed name=" << name << std::endl;
+	}
+	
+	virtual void test_completed(const tut::test_result& tr)
 	{
 		++mTotalTests;
 		std::ostringstream out;
 		out << "[" << tr.group << ", " << tr.test << "] ";
 		switch(tr.result)
 		{
-		case tut::test_result::ok:
-			++mPassedTests;
-			out << "ok";
-			break;
-		case tut::test_result::fail:
-			++mFailedTests;
-			out << "fail";
-			break;
-		case tut::test_result::ex:
-			++mFailedTests;
-			out << "exception";
-			break;
-		case tut::test_result::warn:
-			++mFailedTests;
-			out << "test destructor throw";
-			break;
-		case tut::test_result::term:
-			++mFailedTests;
-			out << "abnormal termination";
-			break;
-		case tut::test_result::skip:
-			++mSkippedTests;
-			out << "skipped known failure";
-			break;
-		default:
-			++mFailedTests;
-			out << "unknown";
+			case tut::test_result::ok:
+				++mPassedTests;
+				out << "ok";
+				break;
+			case tut::test_result::fail:
+				++mFailedTests;
+				out << "fail";
+				break;
+			case tut::test_result::ex:
+				++mFailedTests;
+				out << "exception";
+				break;
+			case tut::test_result::warn:
+				++mFailedTests;
+				out << "test destructor throw";
+				break;
+			case tut::test_result::term:
+				++mFailedTests;
+				out << "abnormal termination";
+				break;
+			case tut::test_result::skip:
+				++mSkippedTests;			
+				out << "skipped known failure";
+				break;
+			default:
+				++mFailedTests;
+				out << "unknown";
 		}
 		if(mVerboseMode || (tr.result != tut::test_result::ok))
 		{
@@ -127,8 +140,8 @@ public:
 			std::cout << out.str() << std::endl;
 		}
 	}
-
-	void run_completed()
+	
+	virtual void run_completed()
 	{
 		if (mStream)
 		{
@@ -136,11 +149,11 @@ public:
 		}
 		run_completed_(std::cout);
 	}
-
-	int getFailedTests() const { return mFailedTests; }
 	
-private:
-	void run_completed_(std::ostream &stream)
+	virtual int getFailedTests() const { return mFailedTests; }
+	
+	//private:
+	virtual void run_completed_(std::ostream &stream)
 	{
 		stream << "\tTotal Tests:\t" << mTotalTests << std::endl;
 		stream << "\tPassed Tests:\t" << mPassedTests;
@@ -149,13 +162,13 @@ private:
 			stream << "\tYAY!! \\o/";
 		}
 		stream << std::endl;
-
+		
 		if (mSkippedTests > 0)
 		{
 			stream << "\tSkipped known failures:\t" << mSkippedTests
-				<< std::endl;
+			<< std::endl;
 		}
-
+		
 		if(mFailedTests > 0)
 		{
 			stream << "*********************************" << std::endl;
@@ -164,8 +177,161 @@ private:
 			stream << "*********************************" << std::endl;
 		}
 	}
+	
+protected:
+	std::string suite_name;
+	bool mVerboseMode;
+	int mTotalTests;
+	int mPassedTests;
+	int mFailedTests;
+	int mSkippedTests;
+	std::ostream *mStream;
+};
+
+// copy of LLTestCallback which should become a subclass (commented out below). Delete this LLTCTestCallback one fixed. 
 
+// TeamCity specific class which emits service messages
+// http://confluence.jetbrains.net/display/TCD3/Build+Script+Interaction+with+TeamCity;#BuildScriptInteractionwithTeamCity-testReporting
+
+class LLTCTestCallback : public tut::callback
+{
+public:
+	LLTCTestCallback(bool verbose_mode, std::ostream *stream, std::string suitename) :
+	mVerboseMode(verbose_mode),
+	mTotalTests(0),
+	mPassedTests(0),
+	mFailedTests(0),
+	mSkippedTests(0),
+	mStream(stream),
+	suite_name(suitename)
+	{
+	}
+	
+	LLTCTestCallback()
+	{
+	}	
+	
+	virtual void run_started()
+	{
+		//std::cout << "unit test run_started" << std::flush;
+	}
+	
+	virtual void group_started(const std::string& name) {
+		std::cout << "group_started name=" << name << std::endl;
+		std::cout << "##teamcity[testSuiteStarted name='" << name << "']\n"  << std::flush;
+	}
+	
+	virtual void group_completed(const std::string& name) {
+		std::cout << "group_completed name=" << name << std::endl;
+		std::cout << "##teamcity[testSuiteFinished name='" << name << "']\n"  << std::flush;
+	}
+	
+	virtual void test_completed(const tut::test_result& tr)
+	{
+		++mTotalTests;
+		std::ostringstream out;
+		out << "[" << tr.group << ", " << tr.test << "] \n";
+		switch(tr.result)
+		{
+			case tut::test_result::ok:
+				++mPassedTests;
+				out << "ok";
+				std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				break;
+			case tut::test_result::fail:
+				++mFailedTests;
+				out << "fail";
+				std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+				std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				break;
+			case tut::test_result::ex:
+				++mFailedTests;
+				out << "exception";
+				std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+				std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				break;
+			case tut::test_result::warn:
+				++mFailedTests;
+				out << "test destructor throw";
+				std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+				std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				break;
+			case tut::test_result::term:
+				++mFailedTests;
+				out << "abnormal termination";
+				std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+				std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				break;
+			case tut::test_result::skip:
+				++mSkippedTests;			
+				out << "skipped known failure";
+				std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testIgnored name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+				break;
+			default:
+				++mFailedTests;
+				out << "unknown";
+		}
+		if(mVerboseMode || (tr.result != tut::test_result::ok))
+		{
+			if(!tr.message.empty())
+			{
+				out << ": '" << tr.message << "'";
+			}
+			if (mStream)
+			{
+				*mStream << out.str() << std::endl;
+			}
+			
+			std::cout << out.str() << std::endl;
+		}
+	}
+	
+	virtual void run_completed()
+	{
+		if (mStream)
+		{
+			run_completed_(*mStream);
+		}
+		run_completed_(std::cout);
+	}
+	
+	virtual int getFailedTests() const { return mFailedTests; }
+	
+	//private:
+	virtual void run_completed_(std::ostream &stream)
+	{
+		stream << "\tTotal Tests:\t" << mTotalTests << std::endl;
+		stream << "\tPassed Tests:\t" << mPassedTests;
+		if (mPassedTests == mTotalTests)
+		{
+			stream << "\tYAY!! \\o/";
+		}
+		stream << std::endl;
+		
+		if (mSkippedTests > 0)
+		{
+			stream << "\tSkipped known failures:\t" << mSkippedTests
+			<< std::endl;
+		}
+		
+		if(mFailedTests > 0)
+		{
+			stream << "*********************************" << std::endl;
+			stream << "Failed Tests:\t" << mFailedTests << std::endl;
+			stream << "Please report or fix the problem." << std::endl;
+			stream << "*********************************" << std::endl;
+		}
+	}
+	
 protected:
+	std::string suite_name;
 	bool mVerboseMode;
 	int mTotalTests;
 	int mPassedTests;
@@ -174,6 +340,107 @@ protected:
 	std::ostream *mStream;
 };
 
+
+/*
+ // commented out subclass which should be fixed to eliminate the duplicated LLTestCallback and LLTCTestCallaback classes
+ // when this is fixed, the duplicated code in the if(getenv("TEAMCITY_PROJECT_NAME") statements below
+ // 
+ // currectly producing errors like thr following:
+ //		{path}viewer-tut-teamcity2/indra/build-darwin-i386/sharedlibs/RelWithDebInfo/RelWithDebInfo/PROJECT_llmessage_TEST_llmime 
+ //				--touch={path}viewer-tut-teamcity2/indra/build-darwin-i386/llmessage/PROJECT_llmessage_TEST_llmime_ok.txt 
+ //				--{path}viewer-tut-teamcity2/indra/llmessage
+ // 
+ //		run_started
+ //		group_started name=mime_index
+ //		##teamcity[testSuiteStarted name='mime_index']
+ //		Segmentation fault
+ 
+ 
+ // TeamCity specific class which emits service messages
+ // http://confluence.jetbrains.net/display/TCD3/Build+Script+Interaction+with+TeamCity;#BuildScriptInteractionwithTeamCity-testReporting
+ 
+ class LLTCTestCallback : public LLTestCallback
+ {
+ public:
+ LLTCTestCallback(bool verbose_mode, std::ostream *stream, std::string suitename) :
+ mVerboseMode(verbose_mode),
+ mTotalTests(0),
+ mPassedTests(0),
+ mFailedTests(0),
+ mSkippedTests(0),
+ mStream(stream),
+ suite_name(suitename)
+ {
+ }
+ 
+ LLTCTestCallback()
+ {
+ }	
+ 
+ virtual void group_started(const std::string& name) {
+ LLTestCallback::group_started(name);
+ std::cout << "##teamcity[testSuiteStarted name='" << name << "']\n"  << std::flush;
+ }
+ 
+ virtual void group_completed(const std::string& name) {
+ LLTestCallback::group_completed(name);
+ std::cout << "##teamcity[testSuiteFinished name='" << name << "']\n"  << std::flush;
+ }
+ 
+ virtual void test_completed(const tut::test_result& tr)
+ {
+ LLTestCallback::test_completed(tr);
+ 
+ switch(tr.result)
+ {
+ case tut::test_result::ok:
+ std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ break;
+ case tut::test_result::fail:
+ std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+ std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ break;
+ case tut::test_result::ex:
+ std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+ std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ break;
+ case tut::test_result::warn:
+ std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+ std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ break;
+ case tut::test_result::term:
+ std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testFailed name='" << tr.group << "." << tr.test << "' message='" << tr.message << "']\n" << std::flush;
+ std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ break;
+ case tut::test_result::skip:
+ std::cout << "##teamcity[testStarted name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testIgnored name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ std::cout << "##teamcity[testFinished name='" << tr.group << "." << tr.test << "']\n" << std::flush;
+ break;
+ default:
+ break;
+ }
+ 
+ }
+ 
+ protected:
+ std::string suite_name;
+ bool mVerboseMode;
+ int mTotalTests;
+ int mPassedTests;
+ int mFailedTests;
+ int mSkippedTests;
+ std::ostream *mStream;
+ };
+ 
+ }
+ */
+
 static const apr_getopt_option_t TEST_CL_OPTIONS[] =
 {
 	{"help", 'h', 0, "Print the help message."},
@@ -185,27 +452,28 @@ static const apr_getopt_option_t TEST_CL_OPTIONS[] =
 	{"touch", 't', 1, "Touch the given file if all tests succeed"},
 	{"wait", 'w', 0, "Wait for input before exit."},
 	{"debug", 'd', 0, "Emit full debug logs."},
+	{"suitename", 'x', 1, "Run tests using this suitename"},
 	{0, 0, 0, 0}
 };
 
 void stream_usage(std::ostream& s, const char* app)
 {
 	s << "Usage: " << app << " [OPTIONS]" << std::endl
-	  << std::endl;
-
+	<< std::endl;
+	
 	s << "This application runs the unit tests." << std::endl << std::endl;
-
+	
 	s << "Options: " << std::endl;
 	const apr_getopt_option_t* option = &TEST_CL_OPTIONS[0];
 	while(option->name)
 	{
 		s << "  ";
 		s << "  -" << (char)option->optch << ", --" << option->name
-		  << std::endl;
+		<< std::endl;
 		s << "\t" << option->description << std::endl << std::endl;
 		++option;
 	}
-
+	
 	s << "Examples:" << std::endl;
 	s << "  " << app << " --verbose" << std::endl;
 	s << "\tRun all the tests and report all results." << std::endl;
@@ -242,13 +510,13 @@ int main(int argc, char **argv)
 	LLError::initForApplication(".");
 	LLError::setFatalFunction(wouldHaveCrashed);
 	LLError::setDefaultLevel(LLError::LEVEL_ERROR);
-		//< *TODO: should come from error config file. Note that we
-		// have a command line option that sets this to debug.
+	//< *TODO: should come from error config file. Note that we
+	// have a command line option that sets this to debug.
 	
 #ifdef CTYPE_WORKAROUND
 	ctype_workaround();
 #endif
-
+	
 	apr_initialize();
 	apr_pool_t* pool = NULL;
 	if(APR_SUCCESS != apr_pool_create(&pool, NULL))
@@ -262,12 +530,13 @@ int main(int argc, char **argv)
 		std::cerr << "Unable to  pool" << std::endl;
 		return 1;
 	}
-
+	
 	// values used for controlling application
 	bool verbose_mode = false;
 	bool wait_at_exit = false;
 	std::string test_group;
-
+	std::string suite_name;
+	
 	// values use for options parsing
 	apr_status_t apr_err;
 	const char* opt_arg = NULL;
@@ -283,88 +552,199 @@ int main(int argc, char **argv)
 		{
 			char buf[255];		/* Flawfinder: ignore */
 			std::cerr << "Error parsing options: "
-					  << apr_strerror(apr_err, buf, 255) << std::endl;
+			<< apr_strerror(apr_err, buf, 255) << std::endl;
 			return 1;
 		}
 		switch (opt_id)
 		{
-		case 'g':
-			test_group.assign(opt_arg);
-			break;
-		case 'h':
-			stream_usage(std::cout, argv[0]);
-			return 0;
-			break;
-		case 'l':
-			stream_groups(std::cout, argv[0]);
-			return 0;
-		case 'v':
-			verbose_mode = true;
-			break;
-		case 'o':
-			output = new std::ofstream;
-			output->open(opt_arg);
-			break;
-		case 's':	// --sourcedir
-			tut::sSourceDir = opt_arg;
-			// For convenience, so you can use tut::sSourceDir + "myfile"
-			tut::sSourceDir += '/';
-			break;
-		case 't':
-			touch = opt_arg;
-			break;
-		case 'w':
-			wait_at_exit = true;
-			break;
-		case 'd':
-			// *TODO: should come from error config file. We set it to
-			// ERROR by default, so this allows full debug levels.
-			LLError::setDefaultLevel(LLError::LEVEL_DEBUG);
-			break;
-		default:
-			stream_usage(std::cerr, argv[0]);
-			return 1;
-			break;
+			case 'g':
+				test_group.assign(opt_arg);
+				break;
+			case 'h':
+				stream_usage(std::cout, argv[0]);
+				return 0;
+				break;
+			case 'l':
+				stream_groups(std::cout, argv[0]);
+				return 0;
+			case 'v':
+				verbose_mode = true;
+				break;
+			case 'o':
+				output = new std::ofstream;
+				output->open(opt_arg);
+				break;
+			case 's':	// --sourcedir
+				tut::sSourceDir = opt_arg;
+				// For convenience, so you can use tut::sSourceDir + "myfile"
+				tut::sSourceDir += '/';
+				break;
+			case 't':
+				touch = opt_arg;
+				break;
+			case 'w':
+				wait_at_exit = true;
+				break;
+			case 'd':
+				// *TODO: should come from error config file. We set it to
+				// ERROR by default, so this allows full debug levels.
+				LLError::setDefaultLevel(LLError::LEVEL_DEBUG);
+				break;
+			case 'x':
+				suite_name.assign(opt_arg);
+				break;
+			default:
+				stream_usage(std::cerr, argv[0]);
+				return 1;
+				break;
 		}
 	}
-
+	
+	/*
+	 // commented out test tunner logic which should be fixed when eliminate the duplicated LLTestCallback and LLTCTestCallaback classes
+	 // become proper class:subclass
+	 // if the Segmentation fault issue is resolved, all code in the block comments can be uncommented, and all code below can be removed.
+	 
+	 LLTestCallback* mycallback;
+	 if (getenv("TEAMCITY_PROJECT_NAME"))
+	 {
+	 mycallback = new LLTCTestCallback(verbose_mode, output, suite_name);
+	 
+	 }
+	 else
+	 {
+	 mycallback = new LLTestCallback(verbose_mode, output, suite_name);
+	 }
+	 
+	 tut::runner.get().set_callback(mycallback);
+	 
+	 if(test_group.empty())
+	 {
+	 tut::runner.get().run_tests();
+	 }
+	 else
+	 {
+	 tut::runner.get().run_tests(test_group);
+	 }
+	 
+	 bool success = (mycallback->getFailedTests() == 0);
+	 
+	 if (wait_at_exit)
+	 {
+	 std::cerr << "Press return to exit..." << std::endl;
+	 std::cin.get();
+	 }
+	 
+	 if (output)
+	 {
+	 output->close();
+	 delete output;
+	 }
+	 
+	 if (touch && success)
+	 {
+	 std::ofstream s;
+	 s.open(touch);
+	 s << "ok" << std::endl;
+	 s.close();
+	 }
+	 
+	 apr_terminate();
+	 
+	 int retval = (success ? 0 : 1);
+	 return retval;
+	 */
+	
 	// run the tests
-	LLTestCallback callback(verbose_mode, output);
-	tut::runner.get().set_callback(&callback);
 	
-	if(test_group.empty())
+	if (getenv("TEAMCITY_PROJECT_NAME"))
 	{
-		tut::runner.get().run_tests();
+		LLTCTestCallback* mycallback;
+		mycallback = new LLTCTestCallback(verbose_mode, output, suite_name);
+		
+		tut::runner.get().set_callback(mycallback);
+		
+		if(test_group.empty())
+		{
+			tut::runner.get().run_tests();
+		}
+		else
+		{
+			tut::runner.get().run_tests(test_group);
+		}
+		
+		bool success = (mycallback->getFailedTests() == 0);
+		
+		if (wait_at_exit)
+		{
+			std::cerr << "Press return to exit..." << std::endl;
+			std::cin.get();
+		}
+		
+		if (output)
+		{
+			output->close();
+			delete output;
+		}
+		
+		if (touch && success)
+		{
+			std::ofstream s;
+			s.open(touch);
+			s << "ok" << std::endl;
+			s.close();
+		}
+		
+		apr_terminate();
+		
+		int retval = (success ? 0 : 1);
+		return retval;
+		
+		
 	}
+	// NOT if (getenv("TEAMCITY_PROJECT_NAME"))
 	else
 	{
-		tut::runner.get().run_tests(test_group);
-	}
-
-	bool success = (callback.getFailedTests() == 0);
-
-	if (wait_at_exit)
-	{
-		std::cerr << "Press return to exit..." << std::endl;
-		std::cin.get();
-	}
-	
-	if (output)
-	{
-		output->close();
-		delete output;
-	}
-
-	if (touch && success)
-	{
-		std::ofstream s;
-		s.open(touch);
-		s << "ok" << std::endl;
-		s.close();
+		LLTestCallback* mycallback;
+		mycallback = new LLTestCallback(verbose_mode, output, suite_name);
+		
+		tut::runner.get().set_callback(mycallback);
+		
+		if(test_group.empty())
+		{
+			tut::runner.get().run_tests();
+		}
+		else
+		{
+			tut::runner.get().run_tests(test_group);
+		}
+		
+		bool success = (mycallback->getFailedTests() == 0);
+		
+		if (wait_at_exit)
+		{
+			std::cerr << "Press return to exit..." << std::endl;
+			std::cin.get();
+		}
+		
+		if (output)
+		{
+			output->close();
+			delete output;
+		}
+		
+		if (touch && success)
+		{
+			std::ofstream s;
+			s.open(touch);
+			s << "ok" << std::endl;
+			s.close();
+		}
+		
+		apr_terminate();
+		
+		int retval = (success ? 0 : 1);
+		return retval;
+		
 	}
-	
-	apr_terminate();
-	
-	int retval = (success ? 0 : 1);
-	return retval;
 }