diff options
| author | Brad Payne (Vir Linden) <vir@lindenlab.com> | 2014-02-25 13:25:40 -0500 | 
|---|---|---|
| committer | Brad Payne (Vir Linden) <vir@lindenlab.com> | 2014-02-25 13:25:40 -0500 | 
| commit | 895d52a399739962c38ddf571e57f85362823dff (patch) | |
| tree | a404be5fb01219c7f080c10d80017d1d44647dc3 | |
| parent | 948c0c559d14b73714652b581886cbcef391ed62 (diff) | |
| parent | de8fea13627cc5978b8a6135802a52864a11c39a (diff) | |
merge viewer-release to sunshine-external
53 files changed, 5871 insertions, 1523 deletions
| @@ -474,3 +474,4 @@ d40c66e410741de7e90b1ed6dac28dd8a2d7e1f6 3.6.8-release  0d9b9e50f1a8880e05f15688a9ec7d09e0e81013 3.6.13-release  5d746de933a98ca17887cde2fece80e9c7ab0b98 3.7.0-release  dcb4981ce255841b6083d8f65444b65d5a733a17 3.7.1-release +b842534cb4d76c9ef87676a62b1d2d19e79c015f 3.7.2-release diff --git a/autobuild.xml b/autobuild.xml index c213dc0d99..3d8f7c22b1 100755 --- a/autobuild.xml +++ b/autobuild.xml @@ -282,9 +282,9 @@              <key>archive</key>              <map>                <key>hash</key> -              <string>aaea644191807f51051cefa2fac11069</string> +              <string>f7d9b6a9c624364389b71209881f39de</string>                <key>url</key> -              <string>http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-darwin-20110316.tar.bz2</string> +              <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/Darwin/installer/curl-7.24.0-darwin-20130826.tar.bz2</string>              </map>              <key>name</key>              <string>darwin</string> @@ -294,9 +294,9 @@              <key>archive</key>              <map>                <key>hash</key> -              <string>2d9377951d99a1aa4735cea8d4b5aa71</string> +              <string>58b7bf45383c1b1bc24afb303b1519c8</string>                <key>url</key> -              <string>http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-linux-20110316.tar.bz2</string> +              <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/Linux/installer/curl-7.24.0-linux-20130826.tar.bz2</string>              </map>              <key>name</key>              <string>linux</string> @@ -306,9 +306,9 @@              <key>archive</key>              <map>                <key>hash</key> -              <string>fea96aa2a7d513397317194f3d6c979b</string> +              <string>8d9ccb0277a26bfe3f346c3c49ce4b58</string>                <key>url</key> -              <string>http://s3.amazonaws.com/viewer-source-downloads/install_pkgs/curl-7.21.1-windows-20110211.tar.bz2</string> +              <string>http://automated-builds-secondlife-com.s3.amazonaws.com/hg/repo/3p-curl/rev/280289/arch/CYGWIN/installer/curl-7.24.0-windows-20130826.tar.bz2</string>              </map>              <key>name</key>              <string>windows</string> diff --git a/indra/edit-me-to-trigger-new-build.txt b/indra/edit-me-to-trigger-new-build.txt index 48e8b566a6..c63ad74682 100755 --- a/indra/edit-me-to-trigger-new-build.txt +++ b/indra/edit-me-to-trigger-new-build.txt @@ -1 +1 @@ -Mon Apr 15 14:35:39 EDT 2013 +2014-02-25 10:34 diff --git a/indra/llcommon/CMakeLists.txt b/indra/llcommon/CMakeLists.txt index 66995d8083..b0ed763aa1 100755 --- a/indra/llcommon/CMakeLists.txt +++ b/indra/llcommon/CMakeLists.txt @@ -43,6 +43,7 @@ set(llcommon_SOURCE_FILES      llcriticaldamp.cpp      llcursortypes.cpp      lldate.cpp +    lldeadmantimer.cpp      lldependencies.cpp      lldictionary.cpp      llerror.cpp @@ -79,6 +80,7 @@ set(llcommon_SOURCE_FILES      llptrto.cpp       llprocess.cpp      llprocessor.cpp +    llprocinfo.cpp      llqueuedthread.cpp      llrand.cpp      llrefcount.cpp @@ -146,6 +148,7 @@ set(llcommon_HEADER_FILES      lldarray.h      lldarrayptr.h      lldate.h +    lldeadmantimer.h      lldefs.h      lldependencies.h      lldeleteutils.h @@ -206,6 +209,7 @@ set(llcommon_HEADER_FILES      llpriqueuemap.h      llprocess.h      llprocessor.h +    llprocinfo.h      llptrskiplist.h      llptrskipmap.h      llptrto.h @@ -323,12 +327,14 @@ if (LL_TESTS)    LL_ADD_INTEGRATION_TEST(bitpack "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llbase64 "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(lldate "" "${test_libs}") +  LL_ADD_INTEGRATION_TEST(lldeadmantimer "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(lldependencies "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llerror "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llframetimer "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llinstancetracker "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(lllazy "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llprocessor "" "${test_libs}") +  LL_ADD_INTEGRATION_TEST(llprocinfo "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llrand "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llsdserialize "" "${test_libs}")    LL_ADD_INTEGRATION_TEST(llsingleton "" "${test_libs}")                           diff --git a/indra/llcommon/lldeadmantimer.cpp b/indra/llcommon/lldeadmantimer.cpp new file mode 100644 index 0000000000..7d9097e344 --- /dev/null +++ b/indra/llcommon/lldeadmantimer.cpp @@ -0,0 +1,188 @@ +/**  +* @file lldeadmantimer.cpp +* @brief Simple deadman-switch timer. +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA +* $/LicenseInfo$ +*/ + + +#include "lldeadmantimer.h" + + +// *TODO:  Currently, this uses lltimer functions for its time +// aspects and this leaks into the apis in the U64s/F64s.  Would +// like to perhaps switch this over to TSC register-based timers +// sometime and drop the overhead some more. + + +//  Flag states and their meaning: +//  mActive  mDone   Meaning +//   false   false   Nothing running, no result available +//    true   false   Timer running, no result available +//   false    true   Timer finished, result can be read once +//    true    true   Not allowed +// +LLDeadmanTimer::LLDeadmanTimer(F64 horizon, bool inc_cpu) +	: mHorizon(time_type(llmax(horizon, F64(0.0)) * gClockFrequency)), +	  mActive(false),			// If true, a timer is running. +	  mDone(false),				// If true, timer has completed and can be read (once) +	  mStarted(U64L(0)), +	  mExpires(U64L(0)), +	  mStopped(U64L(0)), +	  mCount(U64L(0)), +	  mIncCPU(inc_cpu), +	  mUStartCPU(LLProcInfo::time_type(U64L(0))), +	  mUEndCPU(LLProcInfo::time_type(U64L(0))), +	  mSStartCPU(LLProcInfo::time_type(U64L(0))), +	  mSEndCPU(LLProcInfo::time_type(U64L(0))) +{} + + +// static +LLDeadmanTimer::time_type LLDeadmanTimer::getNow() +{ +	return LLTimer::getCurrentClockCount(); +} + + +void LLDeadmanTimer::start(time_type now) +{ +	// *TODO:  If active, let's complete an existing timer and save +	// the result to the side.  I think this will be useful later. +	// For now, wipe out anything in progress, start fresh. +	 +	if (! now) +	{ +		now = LLTimer::getCurrentClockCount(); +	} +	mActive = true; +	mDone = false; +	mStarted = now; +	mExpires = now + mHorizon; +	mStopped = now; +	mCount = U64L(0); +	if (mIncCPU) +	{ +		LLProcInfo::getCPUUsage(mUStartCPU, mSStartCPU); +	} +} + + +void LLDeadmanTimer::stop(time_type now) +{ +	if (! mActive) +	{ +		return; +	} + +	if (! now) +	{ +		now = getNow(); +	} +	mStopped = now; +	mActive = false; +	mDone = true; +	if (mIncCPU) +	{ +		LLProcInfo::getCPUUsage(mUEndCPU, mSEndCPU); +	} +} + + +bool LLDeadmanTimer::isExpired(time_type now, F64 & started, F64 & stopped, U64 & count, +							   U64 & user_cpu, U64 & sys_cpu) +{ +	const bool status(isExpired(now, started, stopped, count)); +	if (status) +	{ +		user_cpu = U64(mUEndCPU - mUStartCPU); +		sys_cpu = U64(mSEndCPU - mSStartCPU); +	} +	return status; +} + +		 +bool LLDeadmanTimer::isExpired(time_type now, F64 & started, F64 & stopped, U64 & count) +{ +	if (mActive && ! mDone) +	{ +		if (! now) +		{ +			now = getNow(); +		} + +		if (now >= mExpires) +		{ +			// mStopped from ringBell() is the value we want +			mActive = false; +			mDone = true; +		} +	} + +	if (! mDone) +	{ +		return false; +	} +	 +	started = mStarted * gClockFrequencyInv; +	stopped = mStopped * gClockFrequencyInv; +	count = mCount; +	mDone = false; + +	return true; +} + +	 +void LLDeadmanTimer::ringBell(time_type now, unsigned int count) +{ +	if (! mActive) +	{ +		return; +	} +	 +	if (! now) +	{ +		now = getNow(); +	} + +	if (now >= mExpires) +	{ +		// Timer has expired, this event will be dropped +		mActive = false; +		mDone = true; +	} +	else +	{ +		// Timer renewed, keep going +		mStopped = now; +		mExpires = now + mHorizon; +		mCount += count; +		if (mIncCPU) +		{ +			LLProcInfo::getCPUUsage(mUEndCPU, mSEndCPU); +		} +	} +	 +	return; +} + diff --git a/indra/llcommon/lldeadmantimer.h b/indra/llcommon/lldeadmantimer.h new file mode 100644 index 0000000000..980976e176 --- /dev/null +++ b/indra/llcommon/lldeadmantimer.h @@ -0,0 +1,214 @@ +/**  +* @file   lldeadmantimer.h +* @brief  Interface to a simple event timer with a deadman's switch +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA +* $/LicenseInfo$ +*/ + +#ifndef	LL_DEADMANTIMER_H +#define	LL_DEADMANTIMER_H + + +#include "linden_common.h" + +#include "lltimer.h" +#include "llprocinfo.h" + + +/// @file lldeadmantimer.h +/// +/// There are interesting user-experienced events in the viewer that +/// would seem to have well-defined start and stop points but which +/// actually lack such milestones in the code.  Such events (like +/// time to load meshes after logging in, initial inventory load, +/// display name fetch) can be defined somewhat after-the-fact by +/// noticing when we no longer perform operations towards their +/// completion.  This class is intended to help in such applications. +/// +/// What it implements is a deadman's switch (also known as a +/// keepalive switch and a doorbell switch).  The basic operation is +/// as follows: +/// +/// * LLDeadmanTimer is instantiated with a horizon value in seconds, +///   one for each event of interest. +/// * When an event starts, @see start() is invoked to begin a +///   timing operation. +/// * As operations are performed in service of the event (issuing +///   HTTP requests, receiving responses), @see ringBell() is invoked +///   to inform the timer that the operation is still active. +/// * If the operation is canceled or otherwise terminated, @see +///   stop() can be called to end the timing operation. +/// * Concurrent with the ringBell() calls, the program makes +///   periodic (shorter than the horizon but not too short) calls +///   to @see isExpired() to see if the event has expired due to +///   either a stop() call or lack of activity (defined as a ringBell() +///   call in the previous 'horizon' seconds).  If it has expired, +///   the caller also receives start, stop and count values for the +///   event which the application can then report in whatever manner +///   it sees fit. +/// * The timer becomes passive after an isExpired() call that returns +///   true.  It can then be restarted with a new start() call. +/// +/// Threading:  Instances are not thread-safe.  They also use +/// timing code from lltimer.h which is also unsafe. +/// +/// Allocation:  Not refcounted, may be stack or heap allocated. +/// + +class LL_COMMON_API LLDeadmanTimer +{ +public: +	/// Public types + +	/// Low-level time type chosen for compatibility with +	/// LLTimer::getCurrentClockCount() which is the basis +	/// of time operations in this class.  This is likely +	/// to change in a future version in a move to TSC-based +	/// timing. +	typedef U64 time_type; +	 +public: +	/// Construct and initialize an LLDeadmanTimer +	/// +	/// @param horizon	Time, in seconds, after the last @see ringBell() +	///                 call at which point the timer will consider itself +	///					expired. +	/// +	/// @param inc_cpu	If true, gather system and user cpu stats while +	///					running the timer.  This does require more syscalls +	///					during updates.  If false, cpu usage data isn't +	///					collected and will be zero if queried. +	LLDeadmanTimer(F64 horizon, bool inc_cpu); + +	~LLDeadmanTimer()  +		{} +	 +private: +	LLDeadmanTimer(const LLDeadmanTimer &);				// Not defined +	void operator=(const LLDeadmanTimer &);				// Not defined + +public: +	/// Get the current time.  Zero-basis for this time +	/// representation is not defined and is different on +	/// different platforms.  Do not attempt to compute +	/// negative times relative to the first value returned, +	/// there may not be enough 'front porch' on the range +	/// to prevent wraparound. +	/// +	/// Note:  Implementation is expected to change in a +	/// future release as well. +	/// +	static time_type getNow(); + +	/// Begin timing.  If the timer is already active, it is reset +	///	and timing begins now. +	/// +	/// @param now		Current time as returned by @see +	///					LLTimer::getCurrentClockCount().  If zero, +	///					method will lookup current time. +	/// +	void start(time_type now); + +	/// End timing.  Actively declare the end of the event independent +	/// of the deadman's switch operation.  @see isExpired() will return +	/// true and appropriate values will be returned. +	/// +	/// @param now		Current time as returned by @see +	///					LLTimer::getCurrentClockCount().  If zero, +	///					method will lookup current time. +	/// +	void stop(time_type now); + +	/// Declare that something interesting happened.  This has two +	/// effects on an unexpired-timer.  1)  The expiration time +	/// is extended for 'horizon' seconds after the 'now' value. +	/// 2)  An internal counter associated with the event is incremented +	/// by the @ref count parameter.  This count is returned via the +	/// @see isExpired() method. +	/// +	/// @param now		Current time as returned by @see +	///					LLTimer::getCurrentClockCount().  If zero, +	///					method will lookup current time. +	/// +	/// @param count	Count of events to be associated with +	///					this bell ringing. +	/// +	void ringBell(time_type now, unsigned int count); +	 +	/// Checks the status of the timer.  If the timer has expired, +	/// also returns various timer-related stats.  Unlike ringBell(), +	/// does not extend the horizon, it only checks for expiration. +	/// +	/// @param now		Current time as returned by @see +	///					LLTimer::getCurrentClockCount().  If zero, +	///					method will lookup current time. +	/// +	/// @param started	If expired, the starting time of the event is +	///					returned to the caller via this reference. +	/// +	/// @param stopped	If expired, the ending time of the event is +	///					returned to the caller via this reference. +	///					Ending time will be that provided in the +	///					stop() method or the last ringBell() call +	///					leading to expiration, whichever (stop() call +	///					or notice of expiration) happened first. +	/// +	/// @param count	If expired, the number of ringBell() calls +	///					made prior to expiration. +	/// +	/// @param user_cpu	Amount of CPU spent in user mode by the process +	///					during the event.  Value in microseconds and will +	///					read zero if not enabled by the constructor. +	/// +	/// @param sys_cpu	Amount of CPU spent in system mode by the process. +	/// +	/// @return			true if the timer has expired, false otherwise. +	///					If true, it also returns the started, +	///					stopped and count values otherwise these are +	///					left unchanged. +	/// +	bool isExpired(time_type now, F64 & started, F64 & stopped, U64 & count, +				   U64 & user_cpu, U64 & sys_cpu); + +	/// Identical to the six-arugment form except it does without the +	/// CPU time return if the caller isn't interested in it. +	bool isExpired(time_type now, F64 & started, F64 & stopped, U64 & count); + +protected: +	time_type					mHorizon; +	bool						mActive; +	bool						mDone; +	time_type					mStarted; +	time_type					mExpires; +	time_type					mStopped; +	time_type					mCount; + +	const bool					mIncCPU;		// Include CPU metrics in timer +	LLProcInfo::time_type		mUStartCPU; +	LLProcInfo::time_type		mUEndCPU; +	LLProcInfo::time_type		mSStartCPU; +	LLProcInfo::time_type		mSEndCPU; +}; +	 + +#endif	// LL_DEADMANTIMER_H diff --git a/indra/llcommon/llprocinfo.cpp b/indra/llcommon/llprocinfo.cpp new file mode 100644 index 0000000000..c00f979b0b --- /dev/null +++ b/indra/llcommon/llprocinfo.cpp @@ -0,0 +1,94 @@ +/**  +* @file llprocinfo.cpp +* @brief Process, cpu and resource usage information APIs. +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA +* $/LicenseInfo$ +*/ + + +#include "llprocinfo.h" + +#if LL_WINDOWS + +#define	PSAPI_VERSION	1 +#include "windows.h" +#include "psapi.h" + +#elif LL_DARWIN + +#include <sys/resource.h> +#include <mach/mach.h> +	 +#else + +#include <sys/time.h> +#include <sys/resource.h> + +#endif // LL_WINDOWS/LL_DARWIN + + +// static +void LLProcInfo::getCPUUsage(time_type & user_time, time_type & system_time) +{ +#if LL_WINDOWS + +	HANDLE self(GetCurrentProcess());			// Does not have to be closed +	FILETIME ft_dummy, ft_system, ft_user; + +	GetProcessTimes(self, &ft_dummy, &ft_dummy, &ft_system, &ft_user); +	ULARGE_INTEGER uli; +	uli.u.LowPart = ft_system.dwLowDateTime; +	uli.u.HighPart = ft_system.dwHighDateTime; +	system_time = uli.QuadPart / U64L(10);		// Convert to uS +	uli.u.LowPart = ft_user.dwLowDateTime; +	uli.u.HighPart = ft_user.dwHighDateTime; +	user_time = uli.QuadPart / U64L(10); +	 +#elif LL_DARWIN + +	struct rusage usage; + +	if (getrusage(RUSAGE_SELF, &usage)) +	{ +		user_time = system_time = time_type(0U); +		return; +	} +	user_time = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; +	system_time = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; + +#else // Linux + +	struct rusage usage; + +	if (getrusage(RUSAGE_SELF, &usage)) +	{ +		user_time = system_time = time_type(0U); +		return; +	} +	user_time = U64(usage.ru_utime.tv_sec) * U64L(1000000) + usage.ru_utime.tv_usec; +	system_time = U64(usage.ru_stime.tv_sec) * U64L(1000000) + usage.ru_stime.tv_usec; +	 +#endif // LL_WINDOWS/LL_DARWIN/Linux +} + + diff --git a/indra/llcommon/llprocinfo.h b/indra/llcommon/llprocinfo.h new file mode 100644 index 0000000000..e78bcf490a --- /dev/null +++ b/indra/llcommon/llprocinfo.h @@ -0,0 +1,68 @@ +/**  +* @file   llprocinfo.h +* @brief  Interface to process/cpu/resource information services. +* @author monty@lindenlab.com +* +* $LicenseInfo:firstyear=2013&license=viewerlgpl$ +* Second Life Viewer Source Code +* Copyright (C) 2013, Linden Research, Inc. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; +* version 2.1 of the License only. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA +* +* Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA +* $/LicenseInfo$ +*/ + +#ifndef	LL_PROCINFO_H +#define	LL_PROCINFO_H + + +#include "linden_common.h" + +/// @file llprocinfo.h +/// +/// Right now, this is really a namespace disguised as a class. +/// It wraps some types and functions to return information about +/// process resource consumption in a non-OS-specific manner. +/// +/// Threading:  No instances so that's thread-safe.  Implementations +/// of static functions should be thread-safe, they mostly involve +/// direct syscall invocations. +/// +/// Allocation:  Not instantiatable. + +class LL_COMMON_API LLProcInfo +{ +public: +	/// Public types + +	typedef U64 time_type;								/// Relative microseconds +	 +private: +	LLProcInfo();										// Not defined +	~LLProcInfo();										// Not defined +	LLProcInfo(const LLProcInfo &);						// Not defined +	void operator=(const LLProcInfo &);					// Not defined + +public: +	/// Get accumulated system and user CPU time in +	/// microseconds.  Syscalls involved in every invocation. +	/// +	/// Threading:  expected to be safe. +	static void getCPUUsage(time_type & user_time, time_type & system_time); +}; +	 + +#endif	// LL_PROCINFO_H diff --git a/indra/llcommon/llthread.cpp b/indra/llcommon/llthread.cpp index 60adeeaeb7..e67d1bc57b 100755 --- a/indra/llcommon/llthread.cpp +++ b/indra/llcommon/llthread.cpp @@ -3,7 +3,7 @@   *   * $LicenseInfo:firstyear=2004&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -373,6 +373,36 @@ void LLMutex::lock()  #endif  } +bool LLMutex::trylock() +{ +	if(isSelfLocked()) +	{ //redundant lock +		mCount++; +		return true; +	} +	 +	apr_status_t status(apr_thread_mutex_trylock(mAPRMutexp)); +	if (APR_STATUS_IS_EBUSY(status)) +	{ +		return false; +	} +	 +#if MUTEX_DEBUG +	// Have to have the lock before we can access the debug info +	U32 id = LLThread::currentID(); +	if (mIsLocked[id] != FALSE) +		llerrs << "Already locked in Thread: " << id << llendl; +	mIsLocked[id] = TRUE; +#endif + +#if LL_DARWIN +	mLockingThread = LLThread::currentID(); +#else +	mLockingThread = sThreadID; +#endif +	return true; +} +  void LLMutex::unlock()  {  	if (mCount > 0) diff --git a/indra/llcommon/llthread.h b/indra/llcommon/llthread.h index f51d985b5f..8c7143304f 100755 --- a/indra/llcommon/llthread.h +++ b/indra/llcommon/llthread.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2004&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -156,7 +156,8 @@ public:  	virtual ~LLMutex();  	void lock();		// blocks -	void unlock(); +	bool trylock();		// non-blocking, returns true if lock held. +	void unlock();		// undefined behavior when called on mutex not being held  	bool isLocked(); 	// non-blocking, but does do a lock/unlock so not free  	bool isSelfLocked(); //return true if locked in a same thread  	U32 lockingThread() const; //get ID of locking thread @@ -174,6 +175,8 @@ protected:  #endif  }; +//============================================================================ +  // Actually a condition/mutex pair (since each condition needs to be associated with a mutex).  class LL_COMMON_API LLCondition : public LLMutex  { @@ -189,6 +192,8 @@ protected:  	apr_thread_cond_t *mAPRCondp;  }; +//============================================================================ +  class LLMutexLock  {  public: @@ -210,6 +215,43 @@ private:  //============================================================================ +// Scoped locking class similar in function to LLMutexLock but uses +// the trylock() method to conditionally acquire lock without +// blocking.  Caller resolves the resulting condition by calling +// the isLocked() method and either punts or continues as indicated. +// +// Mostly of interest to callers needing to avoid stalls who can +// guarantee another attempt at a later time. + +class LLMutexTrylock +{ +public: +	LLMutexTrylock(LLMutex* mutex) +		: mMutex(mutex), +		  mLocked(false) +	{ +		if (mMutex) +			mLocked = mMutex->trylock(); +	} + +	~LLMutexTrylock() +	{ +		if (mMutex && mLocked) +			mMutex->unlock(); +	} + +	bool isLocked() const +	{ +		return mLocked; +	} +	 +private: +	LLMutex*	mMutex; +	bool		mLocked; +}; + +//============================================================================ +  void LLThread::lockData()  {  	mDataLock->lock(); diff --git a/indra/llcommon/lltimer.h b/indra/llcommon/lltimer.h index 513de0605d..e73741217c 100755 --- a/indra/llcommon/lltimer.h +++ b/indra/llcommon/lltimer.h @@ -146,6 +146,13 @@ static inline time_t time_max()  	}  } +// These are really statics but they've been global for awhile +// and they're material to other timing classes.  If you are +// not implementing a timer class, do not use these directly. +extern LL_COMMON_API F64 gClockFrequency; +extern LL_COMMON_API F64 gClockFrequencyInv; +extern LL_COMMON_API F64 gClocksToMicroseconds; +  // Correction factor used by time_corrected() above.  extern LL_COMMON_API S32 gUTCOffset; diff --git a/indra/llcommon/tests/lldeadmantimer_test.cpp b/indra/llcommon/tests/lldeadmantimer_test.cpp new file mode 100644 index 0000000000..7fd2dde6e0 --- /dev/null +++ b/indra/llcommon/tests/lldeadmantimer_test.cpp @@ -0,0 +1,628 @@ +/**  + * @file lldeadmantimer_test.cpp + * @brief Tests for the LLDeadmanTimer class. + * + * $LicenseInfo:firstyear=2013&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2013, Linden Research, Inc. + *  + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + *  + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU + * Lesser General Public License for more details. + *  + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA + *  + * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "../lldeadmantimer.h" +#include "../llsd.h" +#include "../lltimer.h" + +#include "../test/lltut.h" + +// Convert between floating point time deltas and U64 time deltas. +// Reflects an implementation detail inside lldeadmantimer.cpp + +static LLDeadmanTimer::time_type float_time_to_u64(F64 delta) +{ +	return LLDeadmanTimer::time_type(delta * gClockFrequency); +} + +static F64 u64_time_to_float(LLDeadmanTimer::time_type delta) +{ +	return delta * gClockFrequencyInv; +} + + +namespace tut +{ + +struct deadmantimer_test +{ +	deadmantimer_test() +		{ +			// LLTimer internals updating +			update_clock_frequencies(); +		} +}; + +typedef test_group<deadmantimer_test> deadmantimer_group_t; +typedef deadmantimer_group_t::object deadmantimer_object_t; +tut::deadmantimer_group_t deadmantimer_instance("LLDeadmanTimer"); + +// Basic construction test and isExpired() call +template<> template<> +void deadmantimer_object_t::test<1>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(10.0, false); + +		ensure_equals("WOCM isExpired() returns false after ctor()", timer.isExpired(0, started, stopped, count), false); +		ensure_approximately_equals("WOCM t1 - isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WOCM t1 - isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WOCM t1 - isExpired() does not modify count", count, U64L(8)); +	} + +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(10.0, true); + +		ensure_equals("WCM isExpired() returns false after ctor()", timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); +		ensure_approximately_equals("WCM t1 - isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WCM t1 - isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WCM t1 - isExpired() does not modify count", count, U64L(8)); +		ensure_equals("WCM t1 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); +		ensure_equals("WCM t1 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); +	} +} + + +// Construct with zero horizon - not useful generally but will be useful in testing +template<> template<> +void deadmantimer_object_t::test<2>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(0.0, false);			// Zero is pre-expired +		 +		ensure_equals("WOCM isExpired() still returns false with 0.0 time ctor()", +					  timer.isExpired(0, started, stopped, count), false); +	} + +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(0.0, true);			// Zero is pre-expired +		 +		ensure_equals("WCM isExpired() still returns false with 0.0 time ctor()", +					  timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); +	} +} + + +// "pre-expired" timer - starting a timer with a 0.0 horizon will result in +// expiration on first test. +template<> template<> +void deadmantimer_object_t::test<3>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(0.0, false); + +		timer.start(0); +		ensure_equals("WOCM isExpired() returns true with 0.0 horizon time", +					  timer.isExpired(0, started, stopped, count), true); + 		ensure_approximately_equals("WOCM expired timer with no bell ringing has stopped == started", started, stopped, 8); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(0.0, true); + +		timer.start(0); +		ensure_equals("WCM isExpired() returns true with 0.0 horizon time", +					  timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), true); +		ensure_approximately_equals("WCM expired timer with no bell ringing has stopped == started", started, stopped, 8); +	} +} + + +// "pre-expired" timer - bell rings are ignored as we're already expired. +template<> template<> +void deadmantimer_object_t::test<4>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(0.0, false); +	 +		timer.start(0); +		timer.ringBell(LLDeadmanTimer::getNow() + float_time_to_u64(1000.0), 1); +		ensure_equals("WOCM isExpired() returns true with 0.0 horizon time after bell ring", +					  timer.isExpired(0, started, stopped, count), true); +		ensure_approximately_equals("WOCM ringBell has no impact on expired timer leaving stopped == started", started, stopped, 8); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(0.0, true); +	 +		timer.start(0); +		timer.ringBell(LLDeadmanTimer::getNow() + float_time_to_u64(1000.0), 1); +		ensure_equals("WCM isExpired() returns true with 0.0 horizon time after bell ring", +					  timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), true); +		ensure_approximately_equals("WCM ringBell has no impact on expired timer leaving stopped == started", started, stopped, 8); +	} +} + + +// start(0) test - unexpired timer reports unexpired +template<> template<> +void deadmantimer_object_t::test<5>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(10.0, false); +	 +		timer.start(0); +		ensure_equals("WOCM isExpired() returns false after starting with 10.0 horizon time", +					  timer.isExpired(0, started, stopped, count), false); +		ensure_approximately_equals("WOCM t5 - isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WOCM t5 - isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WOCM t5 - isExpired() does not modify count", count, U64L(8)); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(10.0, true); +	 +		timer.start(0); +		ensure_equals("WCM isExpired() returns false after starting with 10.0 horizon time", +					  timer.isExpired(0, started, stopped, count, user_cpu, sys_cpu), false); +		ensure_approximately_equals("WCM t5 - isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WCM t5 - isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WCM t5 - isExpired() does not modify count", count, U64L(8)); +		ensure_equals("WCM t5 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); +		ensure_equals("WCM t5 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); +	} +} + + +// start() test - start in the past but not beyond 1 horizon +template<> template<> +void deadmantimer_object_t::test<6>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(10.0, false); + +		// Would like to do subtraction on current time but can't because +		// the implementation on Windows is zero-based.  We wrap around +		// the backside resulting in a large U64 number. +	 +		LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); +		LLDeadmanTimer::time_type now(the_past + float_time_to_u64(5.0)); +		timer.start(the_past); +		ensure_equals("WOCM t6 - isExpired() returns false with 10.0 horizon time starting 5.0 in past", +					  timer.isExpired(now, started, stopped, count), false); +		ensure_approximately_equals("WOCM t6 - isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WOCM t6 - isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WOCM t6 - isExpired() does not modify count", count, U64L(8)); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(10.0, true); + +		// Would like to do subtraction on current time but can't because +		// the implementation on Windows is zero-based.  We wrap around +		// the backside resulting in a large U64 number. +	 +		LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); +		LLDeadmanTimer::time_type now(the_past + float_time_to_u64(5.0)); +		timer.start(the_past); +		ensure_equals("WCM t6 - isExpired() returns false with 10.0 horizon time starting 5.0 in past", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +		ensure_approximately_equals("WCM t6 - isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WCM t6 - isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("t6 - isExpired() does not modify count", count, U64L(8)); +		ensure_equals("WCM t6 - isExpired() does not modify user_cpu", user_cpu, U64L(29000)); +		ensure_equals("WCM t6 - isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); +	} +} + + +// start() test - start in the past but well beyond 1 horizon +template<> template<> +void deadmantimer_object_t::test<7>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(10.0, false); +		 +		// Would like to do subtraction on current time but can't because +		// the implementation on Windows is zero-based.  We wrap around +		// the backside resulting in a large U64 number. +	 +		LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); +		LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); +		timer.start(the_past); +		ensure_equals("WOCM t7 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", +					  timer.isExpired(now,started, stopped, count), true); +		ensure_approximately_equals("WOCM t7 - starting before horizon still gives equal started / stopped", started, stopped, 8); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(10.0, true); +		 +		// Would like to do subtraction on current time but can't because +		// the implementation on Windows is zero-based.  We wrap around +		// the backside resulting in a large U64 number. +	 +		LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); +		LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); +		timer.start(the_past); +		ensure_equals("WCM t7 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", +					  timer.isExpired(now,started, stopped, count, user_cpu, sys_cpu), true); +		ensure_approximately_equals("WOCM t7 - starting before horizon still gives equal started / stopped", started, stopped, 8); +	} +} + + +// isExpired() test - results are read-once.  Probes after first true are false. +template<> template<> +void deadmantimer_object_t::test<8>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(10.0, false); + +		// Would like to do subtraction on current time but can't because +		// the implementation on Windows is zero-based.  We wrap around +		// the backside resulting in a large U64 number. +	 +		LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); +		LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); +		timer.start(the_past); +		ensure_equals("WOCM t8 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", +					  timer.isExpired(now, started, stopped, count), true); +		 +		started = 42.0; +		stopped = 97.0; +		count = U64L(8); +		ensure_equals("WOCM t8 - second isExpired() returns false after true", +					  timer.isExpired(now, started, stopped, count), false); +		ensure_approximately_equals("WOCM t8 - 2nd isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WOCM t8 - 2nd isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WOCM t8 - 2nd isExpired() does not modify count", count, U64L(8)); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(10.0, true); + +		// Would like to do subtraction on current time but can't because +		// the implementation on Windows is zero-based.  We wrap around +		// the backside resulting in a large U64 number. +	 +		LLDeadmanTimer::time_type the_past(LLDeadmanTimer::getNow()); +		LLDeadmanTimer::time_type now(the_past + float_time_to_u64(20.0)); +		timer.start(the_past); +		ensure_equals("WCM t8 - isExpired() returns true with 10.0 horizon time starting 20.0 in past", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); +		 +		started = 42.0; +		stopped = 97.0; +		count = U64L(8); +		user_cpu = 29000; +		sys_cpu = 57000; +		ensure_equals("WCM t8 - second isExpired() returns false after true", +					  timer.isExpired(now, started, stopped, count), false); +		ensure_approximately_equals("WCM t8 - 2nd isExpired() does not modify started", started, F64(42.0), 2); +		ensure_approximately_equals("WCM t8 - 2nd isExpired() does not modify stopped", stopped, F64(97.0), 2); +		ensure_equals("WCM t8 - 2nd isExpired() does not modify count", count, U64L(8)); +		ensure_equals("WCM t8 - 2nd isExpired() does not modify user_cpu", user_cpu, U64L(29000)); +		ensure_equals("WCM t8 - 2nd isExpired() does not modify sys_cpu", sys_cpu, U64L(57000)); +	} +} + + +// ringBell() test - see that we can keep a timer from expiring +template<> template<> +void deadmantimer_object_t::test<9>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(5.0, false); + +		LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); +		F64 real_start(u64_time_to_float(now)); +		timer.start(0); + +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		ensure_equals("WOCM t9 - 5.0 horizon timer has not timed out after 10 1-second bell rings", +					  timer.isExpired(now, started, stopped, count), false); +		F64 last_good_ring(u64_time_to_float(now)); + +		// Jump forward and expire +		now += float_time_to_u64(10.0); +		ensure_equals("WOCM t9 - 5.0 horizon timer expires on 10-second jump", +					  timer.isExpired(now, started, stopped, count), true); +		ensure_approximately_equals("WOCM t9 - started matches start() time", started, real_start, 4); +		ensure_approximately_equals("WOCM t9 - stopped matches last ringBell() time", stopped, last_good_ring, 4); +		ensure_equals("WOCM t9 - 10 good ringBell()s", count, U64L(10)); +		ensure_equals("WOCM t9 - single read only", timer.isExpired(now, started, stopped, count), false); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); +		LLDeadmanTimer timer(5.0, true); + +		LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); +		F64 real_start(u64_time_to_float(now)); +		timer.start(0); + +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		ensure_equals("WCM t9 - 5.0 horizon timer has not timed out after 10 1-second bell rings", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +		F64 last_good_ring(u64_time_to_float(now)); + +		// Jump forward and expire +		now += float_time_to_u64(10.0); +		ensure_equals("WCM t9 - 5.0 horizon timer expires on 10-second jump", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); +		ensure_approximately_equals("WCM t9 - started matches start() time", started, real_start, 4); +		ensure_approximately_equals("WCM t9 - stopped matches last ringBell() time", stopped, last_good_ring, 4); +		ensure_equals("WCM t9 - 10 good ringBell()s", count, U64L(10)); +		ensure_equals("WCM t9 - single read only", timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +	} +} + + +// restart after expiration test - verify that restarts behave well +template<> template<> +void deadmantimer_object_t::test<10>() +{ +	{ +		// Without cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)); +		LLDeadmanTimer timer(5.0, false); + +		LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); +		F64 real_start(u64_time_to_float(now)); +		timer.start(0); + +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		ensure_equals("WOCM t10 - 5.0 horizon timer has not timed out after 10 1-second bell rings", +					  timer.isExpired(now, started, stopped, count), false); +		F64 last_good_ring(u64_time_to_float(now)); + +		// Jump forward and expire +		now += float_time_to_u64(10.0); +		ensure_equals("WOCM t10 - 5.0 horizon timer expires on 10-second jump", +					  timer.isExpired(now, started, stopped, count), true); +		ensure_approximately_equals("WOCM t10 - started matches start() time", started, real_start, 4); +		ensure_approximately_equals("WOCM t10 - stopped matches last ringBell() time", stopped, last_good_ring, 4); +		ensure_equals("WOCM t10 - 10 good ringBell()s", count, U64L(10)); +		ensure_equals("WOCM t10 - single read only", timer.isExpired(now, started, stopped, count), false); +		 +		// Jump forward and restart +		now += float_time_to_u64(1.0); +		real_start = u64_time_to_float(now); +		timer.start(now); +		 +		// Run a modified bell ring sequence +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		ensure_equals("WOCM t10 - 5.0 horizon timer has not timed out after 8 1-second bell rings", +					  timer.isExpired(now, started, stopped, count), false); +		last_good_ring = u64_time_to_float(now); + +		// Jump forward and expire +		now += float_time_to_u64(10.0); +		ensure_equals("WOCM t10 - 5.0 horizon timer expires on 8-second jump", +					  timer.isExpired(now, started, stopped, count), true); +		ensure_approximately_equals("WOCM t10 - 2nd started matches start() time", started, real_start, 4); +		ensure_approximately_equals("WOCM t10 - 2nd stopped matches last ringBell() time", stopped, last_good_ring, 4); +		ensure_equals("WOCM t10 - 8 good ringBell()s", count, U64L(8)); +		ensure_equals("WOCM t10 - single read only - 2nd start", +					  timer.isExpired(now, started, stopped, count), false); +	} +	{ +		// With cpu metrics +		F64 started(42.0), stopped(97.0); +		U64 count(U64L(8)), user_cpu(29000), sys_cpu(57000); + +		LLDeadmanTimer timer(5.0, true); + +		LLDeadmanTimer::time_type now(LLDeadmanTimer::getNow()); +		F64 real_start(u64_time_to_float(now)); +		timer.start(0); + +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		ensure_equals("WCM t10 - 5.0 horizon timer has not timed out after 10 1-second bell rings", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +		F64 last_good_ring(u64_time_to_float(now)); + +		// Jump forward and expire +		now += float_time_to_u64(10.0); +		ensure_equals("WCM t10 - 5.0 horizon timer expires on 10-second jump", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); +		ensure_approximately_equals("WCM t10 - started matches start() time", started, real_start, 4); +		ensure_approximately_equals("WCM t10 - stopped matches last ringBell() time", stopped, last_good_ring, 4); +		ensure_equals("WCM t10 - 10 good ringBell()s", count, U64L(10)); +		ensure_equals("WCM t10 - single read only", timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +		 +		// Jump forward and restart +		now += float_time_to_u64(1.0); +		real_start = u64_time_to_float(now); +		timer.start(now); +		 +		// Run a modified bell ring sequence +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		now += float_time_to_u64(1.0); +		timer.ringBell(now, 1); +		ensure_equals("WCM t10 - 5.0 horizon timer has not timed out after 8 1-second bell rings", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +		last_good_ring = u64_time_to_float(now); + +		// Jump forward and expire +		now += float_time_to_u64(10.0); +		ensure_equals("WCM t10 - 5.0 horizon timer expires on 8-second jump", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), true); +		ensure_approximately_equals("WCM t10 - 2nd started matches start() time", started, real_start, 4); +		ensure_approximately_equals("WCM t10 - 2nd stopped matches last ringBell() time", stopped, last_good_ring, 4); +		ensure_equals("WCM t10 - 8 good ringBell()s", count, U64L(8)); +		ensure_equals("WCM t10 - single read only - 2nd start", +					  timer.isExpired(now, started, stopped, count, user_cpu, sys_cpu), false); +	} +} + + + +} // end namespace tut diff --git a/indra/llcommon/tests/llprocinfo_test.cpp b/indra/llcommon/tests/llprocinfo_test.cpp new file mode 100644 index 0000000000..12d5a695ee --- /dev/null +++ b/indra/llcommon/tests/llprocinfo_test.cpp @@ -0,0 +1,91 @@ +/**  + * @file llprocinfo_test.cpp + * @brief Tests for the LLProcInfo class. + * + * $LicenseInfo:firstyear=2013&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2013, Linden Research, Inc. + *  + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + *  + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU + * Lesser General Public License for more details. + *  + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA + *  + * Linden Research, Inc., 945 Battery Street, San Francisco, CA  94111  USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "../llprocinfo.h" + +#include "../test/lltut.h" +#include "../lltimer.h" + + +static const LLProcInfo::time_type bad_user(289375U), bad_system(275U); + + +namespace tut +{ + +struct procinfo_test +{ +	procinfo_test() +		{ +		} +}; + +typedef test_group<procinfo_test> procinfo_group_t; +typedef procinfo_group_t::object procinfo_object_t; +tut::procinfo_group_t procinfo_instance("LLProcInfo"); + + +// Basic invocation works +template<> template<> +void procinfo_object_t::test<1>() +{ +	LLProcInfo::time_type user(bad_user), system(bad_system); + +	set_test_name("getCPUUsage() basic function"); + +	LLProcInfo::getCPUUsage(user, system); +	 +	ensure_not_equals("getCPUUsage() writes to its user argument", user, bad_user); +	ensure_not_equals("getCPUUsage() writes to its system argument", system, bad_system); +} + + +// Time increases +template<> template<> +void procinfo_object_t::test<2>() +{ +	LLProcInfo::time_type user(bad_user), system(bad_system); +	LLProcInfo::time_type user2(bad_user), system2(bad_system); + +	set_test_name("getCPUUsage() increases over time"); + +	LLProcInfo::getCPUUsage(user, system); +	 +	for (int i(0); i < 100000; ++i) +	{ +		ms_sleep(0); +	} +	 +	LLProcInfo::getCPUUsage(user2, system2); + +	ensure_equals("getCPUUsage() user value doesn't decrease over time", user2 >= user, true); +	ensure_equals("getCPUUsage() system value doesn't decrease over time", system2 >= system, true); +} + + +} // end namespace tut diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden new file mode 100644 index 0000000000..eb6ccab3bc --- /dev/null +++ b/indra/llcorehttp/README.Linden @@ -0,0 +1,671 @@ + + + +1.  HTTP Fetching in 15 Minutes + +    Let's start with a trivial working example.  You'll need a throwaway +    build of the viewer.  And we'll use indra/newview/llappviewer.cpp as +    the host module for these hacks. + +    First, add some headers: + + +        #include "httpcommon.h" +        #include "httprequest.h" +        #include "httphandler.h" + + +    You'll need to derive a class from HttpHandler (not HttpHandle). +    This is used to deliver notifications of HTTP completion to your +    code.  Place it near the top, before LLDeferredTaskList, say: + + +        class MyHandler : public LLCore::HttpHandler +        { +        public: +            MyHandler() +            : LLCore::HttpHandler() +            {} + +            virtual void onCompleted(LLCore::HttpHandle /* handle */, +                                     LLCore::HttpResponse * /* response */) +            { +                LL_INFOS("Hack") << "It is happening again." << LL_ENDL; + +                delete this;    // Last statement +            } +        }; + + +    Add some statics up there as well: + + +        // Our request object.  Allocate during initialiation. +        static LLCore::HttpRequest * my_request(NULL); + +        // The policy class for HTTP traffic. +        // Use HttpRequest::DEFAULT_POLICY_ID, but DO NOT SHIP WITH THIS VALUE!! +        static LLCore::HttpRequest::policy_t my_policy(LLCore::HttpRequest::DEFAULT_POLICY_ID); + +        // Priority for HTTP requests.  Use 0U. +        static LLCore::HttpRequest::priority_t my_priority(0U); + + +    In LLAppViewer::init() after mAppCoreHttp.init(), create a request object: + + +        my_request = new LLCore::HttpRequest(); + + +    In LLAppViewer::mainLoop(), just before entering the while loop, +    we'll kick off one HTTP request: + + +        // Construct a handler object (we'll use the heap this time): +        MyHandler * my_handler = new MyHandler; + +        // Issue a GET request to 'http://www.example.com/' kicking off +        // all the I/O, retry logic, etc. +        LLCore::HttpHandle handle; +        handle = my_request->requestGet(my_policy, +                                        my_priority, +                                        "http://www.example.com/", +                                        NULL, +                                        NULL, +                                        my_handler); +        if (LLCORE_HTTP_HANDLE_INVALID == handle) +        { +            LL_WARNS("Hack") << "Failed to launch HTTP request.  Try again." +                             << LL_ENDL; +        } + + +    Finally, arrange to periodically call update() on the request object +    to find out when the request completes.  This will be done by +    calling the onCompleted() method with status information and +    response data from the HTTP operation.  Add this to the +    LLAppViewer::idle() method after the ping: + + +        my_request->update(0); + + +    That's it.  Build it, run it and watch the log file.  You should get +    the "It is happening again." message indicating that the HTTP +    operation completed in some manner. + + +2.  What Does All That Mean + +    MyHandler/HttpHandler.  This class replaces the Responder-style in +    legacy code.  One method is currently defined.  It is used for all +    request completions, successful or failed: + + +        void onCompleted(LLCore::HttpHandle /* handle */, +                         LLCore::HttpResponse * /* response */); + + +    The onCompleted() method is invoked as a callback during calls to +    HttpRequest::update().  All I/O is completed asynchronously in +    another thread.  But notifications are polled by calling update() +    and invoking a handler for completed requests. + +    In this example, the invocation also deletes the handler (which is +    never referenced by the llcorehttp code again).  But other +    allocation models are possible including handlers shared by many +    requests, stack-based handlers and handlers mixed in with other, +    unrelated classes. + +    LLCore::HttpRequest().  Instances of this class are used to request +    all major functions of the library.  Initialization, starting +    requests, delivering final notification of completion and various +    utility operations are all done via instances.  There is one very +    important rule for instances: + +        Request objects may NOT be shared between threads. + +    my_priority.  The APIs support the idea of priority ordering of +    requests but it hasn't been implemented and the hope is that this +    will become useless and removed from the interface.  Use 0U except +    as noted. + +    my_policy.  This is an important one.  This library attempts to +    manage TCP connection usage more rigorously than in the past.  This +    is done by issuing requests to a queue that has various settable +    properties.  These establish connection usage for the queue as well +    as how queues compete with one another.  (This is patterned after +    class-based queueing used in various networking stacks.)  Several +    classes are pre-defined.  Deciding when to use an existing class and +    when to create a new one will determine what kind of experience +    users have.  We'll pick up this question in detail below. + +    requestGet().  Issues an ordinary HTTP GET request to a given URL +    and associating the request with a policy class, a priority and an +    response handler.  Two additional arguments, not used here, allow +    for additional headers on the request and for per-request options. +    If successful, the call returns a handle whose value is other than +    LLCORE_HTTP_HANDLE_INVALID.  The HTTP operation is then performed +    asynchronously by another thread without any additional work by the +    caller.  If the handle returned is invalid, you can get the status +    code by calling my_request->getStatus(). + +    update().  To get notification that the request has completed, a +    call to update() will invoke onCompleted() methods. + + +3.  Refinements, Necessary and Otherwise + +    MyHandler::onCompleted().  You'll want to do something useful with +    your response.  Distinguish errors from successes and getting the +    response body back in some form. + +    Add a new header: + + +        #include "bufferarray.h" + + +    Replace the existing MyHandler::onCompleted() definition with: + + +        virtual void onCompleted(LLCore::HttpHandle /* handle */, +                                 LLCore::HttpResponse * response) +        { +            LLCore::HttpStatus status = response->getStatus(); +            if (status) +            { +                // Successful request.  Try to fetch the data +                LLCore::BufferArray * data = response->getBody(); + +                if (data && data->size()) +                { +                    // There's some data.  A BufferArray is a linked list +                    // of buckets.  We'll create a linear buffer and copy +                    // the data into it. +                    size_t data_len = data->size(); +                    char * data_blob = new char [data_len + 1]; +                    data->read(0, data_blob, data_len); +                    data_blob[data_len] = '\0'; + +                    // Process the data now in NUL-terminated string. +                    // Needs more scrubbing but this will do. +                    LL_INFOS("Hack") << "Received:  " << data_blob << LL_ENDL; + +                    // Free the temporary data +                    delete [] data_blob; +                } +            } +            else +            { +                // Something went wrong.  Translate the status to +                // a meaningful message. +                LL_WARNS("Hack") << "HTTP GET failed.  Status:  " +                                 << status.toTerseString() +                                 << ", Reason:  " << status.toString() +                                 << LL_ENDL; +            }            + +            delete this;    // Last statement +        } + + +    HttpHeaders.  The header file "httprequest.h" documents the expected +    important headers that will go out with the request.  You can add to +    these by including an HttpHeaders object with the requestGet() call. +    These are typically setup once as part of init rather than +    dynamically created. + +    Add another header: + + +        #include "httpheaders.h" + + +    In LLAppViewer::mainLoop(), add this alongside the allocation of +    my_handler: + + +        // Additional headers for all requests +        LLCore::HttpHeaders * my_headers = new LLCore::HttpHeaders(); +        my_headers->append("Accept", "text/html, application/llsd+xml"); + + +    HttpOptions.  Options are similar and include a mix of value types. +    One interesting per-request option is the trace setting.  This +    enables various debug-type messages in the log file that show the +    progress of the request through the library.  It takes values from +    zero to three with higher values giving more verbose logging.  We'll +    use '2' and this will also give us a chance to verify that +    HttpHeaders works as expected. + +    Same as above, a new header: + + +        #include "httpoptions.h" + + +    And in LLAppView::mainLoop(): + + +        // Special options for requests +        LLCore::HttpOptions * my_options = new LLCore::HttpOptions(); +        my_options->setTrace(2); + + +    Now let's put that all together into a more complete requesting +    sequence.  Replace the existing invocation of requestGet() with this +    slightly more elaborate block: + + +        LLCore::HttpHandle handle; +        handle = my_request->requestGet(my_policy, +                                        my_priority, +                                        "http://www.example.com/", +                                        my_options, +                                        my_headers, +                                        my_handler); +        if (LLCORE_HTTP_HANDLE_INVALID == handle) +        { +             LLCore::HttpStatus status = my_request->getStatus(); + +             LL_WARNS("Hack") << "Failed to request HTTP GET.  Status:  " +                              << status.toTerseString() +                              << ", Reason:  " << status.toString() +                              << LL_ENDL; + +             delete my_handler;    // No longer needed. +             my_handler = NULL; +        } + + +    Build, run and examine the log file.  You'll get some new data with +    this run.  First, you should get the www.example.com home page +    content: + + +---------------------------------------------------------------------------- +2013-09-17T20:26:51Z INFO: MyHandler::onCompleted: Received:  <!doctype html> +<html> +<head> +    <title>Example Domain</title> + +    <meta charset="utf-8" /> +    <meta http-equiv="Content-type" content="text/html; charset=utf-8" /> +    <meta name="viewport" content="width=device-width, initial-scale=1" /> +    <style type="text/css"> +    body { +        background-color: #f0f0f2; +        margin: 0; +        padding: 0; +        font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; +         +    } +    div { +        width: 600px; +        margin: 5em auto; +        padding: 50px; +        background-color: #fff; +        border-radius: 1em; +    } +    a:link, a:visited { +        color: #38488f; +        text-decoration: none; +    } +    @media (max-width: 700px) { +        body { +            background-color: #fff; +        } +        div { +            width: auto; +            margin: 0 auto; +            border-radius: 0; +            padding: 1em; +        } +    } +    </style>     +</head> + +<body> +<div> +    <h1>Example Domain</h1> +    <p>This domain is established to be used for illustrative examples in documents. You may use this +    domain in examples without prior coordination or asking for permission.</p> +    <p><a href="http://www.iana.org/domains/example">More information...</a></p> +</div> +</body> +</html> +---------------------------------------------------------------------------- + + +    You'll also get a detailed trace of the HTTP operation itself.  Note +    the HEADEROUT line which shows the additional header added to the +    request. + + +---------------------------------------------------------------------------- +HttpService::processRequestQueue: TRACE, FromRequestQueue, Handle:  086D3148 +HttpLibcurl::addOp: TRACE, ToActiveQueue, Handle:  086D3148, Actives:  0, Readies:  0 +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  TEXT, Data:  About to connect() to www.example.com port 80 (#0)  +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  TEXT, Data:    Trying 93.184.216.119...  +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  TEXT, Data:  Connected to www.example.com (93.184.216.119) port 80 (#0)  +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  TEXT, Data:  Connected to www.example.com (93.184.216.119) port 80 (#0)  +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADEROUT, Data:  GET / HTTP/1.1  Host: www.example.com  Accept-Encoding: deflate, gzip  Connection: keep-alive  Keep-alive: 300  Accept: text/html, application/llsd+xml +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  HTTP/1.1 200 OK   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Accept-Ranges: bytes   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Cache-Control: max-age=604800   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Content-Type: text/html   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Date: Tue, 17 Sep 2013 20:26:56 GMT   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Etag: "3012602696"   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Expires: Tue, 24 Sep 2013 20:26:56 GMT   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Server: ECS (ewr/1590)   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  X-Cache: HIT   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  x-ec-custom-error: 1   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:  Content-Length: 1270   +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  HEADERIN, Data:     +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  DATAIN, Data:  256 Bytes +HttpOpRequest::debugCallback: TRACE, LibcurlDebug, Handle:  086D3148, Type:  TEXT, Data:  Connection #0 to host www.example.com left intact  +HttpLibcurl::completeRequest: TRACE, RequestComplete, Handle:  086D3148, Status:  Http_200 +HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle:  086D3148 +---------------------------------------------------------------------------- + + +4.  What Does All That Mean, Part 2 + +    HttpStatus.  The HttpStatus object encodes errors from libcurl, the +    library itself and HTTP status values.  It does this to avoid +    collapsing all non-HTTP error into a single '499' HTTP status and to +    make errors distinct. + +    To aid programming, the usual bool conversions are available so that +    you can write 'if (status)' and the expected thing will happen +    whether it's an HTTP, libcurl or library error.  There's also +    provision to override the treatment of HTTP errors (making 404 a +    success, say). + +    Share data, don't copy it.  The library was started with the goal of +    avoiding data copies as much as possible.  Instead, read-only data +    sharing across threads with atomic reference counts is used for a +    number of data types.  These currently are: + +        * BufferArray.  Linked list of data blocks/HTTP bodies. +        * HttpHeaders.  Shared headers for both requests and responses. +        * HttpOptions.  Request-only data modifying HTTP behavior. +        * HttpResponse.  HTTP response description given to onCompleted. + +    Using objects of these types requires a few rules: + +        * Constructor always gives a reference to caller. +        * References are dropped with release() not delete. +        * Additional references may be taken out with addRef(). +        * Unless otherwise stated, once an object is shared with another +          thread it should be treated as read-only.  There's no +          synchronization on the objects themselves. + +    HttpResponse.  You'll encounter this mainly in onCompleted() methods. +    Commonly-used interfaces on this object: + +        * getStatus() to return the final status of the request. +        * getBody() to retrieve the response body which may be NULL or +          zero-length. +        * getContentType() to return the value of the 'Content-Type' +          header or an empty string if none was sent. + +    This is a reference-counted object so you can call addRef() on it +    and hold onto the response for an arbitrary time.  But you'll +    usually just call a few methods and return from onCompleted() whose +    caller will release the object. + +    BufferArray.  The core data representation for request and response +    bodies.  In HTTP responses, it's fetched with the getBody() method +    and may be NULL or non-NULL with zero length.  All successful data +    handling should check both conditions before attempting to fetch +    data from the object.  Data access model uses simple read/write +    semantics: + +        * append() +        * size() +        * read() +        * write() + +    (There is a more sophisticated stream adapter that extends these +    methods and will be covered below.)  So, one way to retrieve data +    from a request is as follows: + + +        LLCore::BufferArray * data = response->getBody(); +        if (data && data->size()) +        { +            size_t data_len = data->size(); +            char * data_blob = new char [data_len + 1]; +            data->read(0, data_blob, data_len); + + +    HttpOptions and HttpResponse.  Really just simple containers of POD +    and std::string pairs.  But reference counted and the rule about not +    modifying after sharing must be followed.  You'll have the urge to +    change options dynamically at some point.  And you'll try to do that +    by just writing new values to the shared object.  And in tests +    everything will appear to work.  Then you ship and people in the +    real world start hitting read/write races in strings and then crash. +    Don't be lazy. + +    HttpHandle.  Uniquely identifies a request and can be used to +    identify it in an onCompleted() method or cancel it if it's still +    queued.  But as soon as a request's onCompleted() invocation +    returns, the handle becomes invalid and may be reused immediately +    for new requests.  Don't hold on to handles after notification. + + +5.  And Still More Refinements + +    (Note: The following refinements are just code fragments.  They +    don't directly fit into the working example above.  But they +    demonstrate several idioms you'll want to copy.) + +    LLSD, std::streambuf, std::iostream.  The read(), write() and +    append() methods may be adequate for your purposes.  But we use a +    lot of LLSD.  Its interfaces aren't particularly compatible with +    BufferArray.  And so two adapters are available to give +    stream-like behaviors:  BufferArrayStreamBuf and BufferArrayStream, +    which implement the std::streambuf and std::iostream interfaces, +    respectively. + +    A std::streambuf interface isn't something you'll want to use +    directly.  Instead, you'll use the much friendlier std::iostream +    interface found in BufferArrayStream.  This adapter gives you all +    the '>>' and '<<' operators you'll want as well as working +    directly with the LLSD conversion operators. + +    Some new headers: + + +        #include "bufferstream.h" +        #include "llsdserialize.h" + + +    And an updated fragment based on onCompleted() above: + + +                // Successful request.  Try to fetch the data +                LLCore::BufferArray * data = response->getBody(); +                LLSD resp_llsd; + +                if (data && data->size()) +                { +                    // There's some data and we expect this to be +                    // LLSD.  Checking of content type and validation +                    // during parsing would be admirable additions. +                    // But we'll forgo that now. +                    LLCore::BufferArrayStream data_stream(data); +                    LLSDSerialize::fromXML(resp_llsd, data_stream); +                } +                LL_INFOS("Hack") << "LLSD Received:  " << resp_llsd << LL_ENDL; +            } +            else +            { + + +    Converting an LLSD object into an XML stream stored in a +    BufferArray is just the reverse of the above: + + +        BufferArray * data = new BufferArray(); +        LLCore::BufferArrayStream data_stream(data); + +        LLSD src_llsd; +        src_llsd["foo"] = "bar"; + +        LLSDSerialize::toXML(src_llsd, data_stream); + +        // 'data' now contains an XML payload and can be sent +        // to a web service using the requestPut() or requestPost() +        //  methods. +        ...  requestPost(...); + +        // And don't forget to release the BufferArray. +        data->release(); +        data = NULL; + + +    LLSD will often go hand-in-hand with BufferArray and data +    transport.  But you can also do all the streaming I/O you'd expect +    of a std::iostream object: + + +        BufferArray * data = new BufferArray(); +        LLCore::BufferArrayStream data_stream(data); + +        data_stream << "Hello, World!" << 29.4 << '\n'; +        std::string str; +        data_stream >> str; +        std::cout << str << std::endl; + +        data->release(); +        // Actual delete will occur when 'data_stream' +        // falls out of scope and is destructed. + + +    Scoping objects and cleaning up.  The examples haven't bothered +    with cleanup of objects that are no longer needed.  Instead, most +    objects have been allocated as if they were global and eternal. +    You'll put the objects in more appropriate feature objects and +    clean them up as a group.  Here's a checklist for actions you may +    need to take on cleanup: + +    * Call delete on: +      o HttpHandlers created on the heap +      o HttpRequest objects +    * Call release() on: +      o BufferArray objects +      o HttpHeaders objects +      o HttpOptions objects +      o HttpResponse objects + +    On program exit, as threads wind down, the library continues to +    operate safely.  Threads don't interact via the library and even +    dangling references to HttpHandler objects are safe.  If you don't +    call HttpRequest::update(), handler references are never +    dereferenced. + +    You can take a more thorough approach to wind-down.  Keep a list +    of HttpHandles (not HttpHandlers) of outstanding requests.  For +    each of these, call HttpRequest::requestCancel() to cancel the +    operation.  (Don't add the cancel requests' handled to the list.) +    This will cancel the outstanding requests that haven't completed. +    Canceled or completed, all requests will queue notifications.  You +    can now cycle calling update() discarding responses.  Continue +    until all requests notify or a few seconds have passed. + +    Global startup and shutdown is handled in the viewer.  But you can +    learn about it in the code or in the documentation in the headers. + + +6.  Choosing a Policy Class + +    Now it's time to get rid of the default policy class.  Take a look +    at the policy class definitions in newview/llappcorehttp.h. +    Ideally, you'll find one that's compatible with what you're doing. +    Some of the compatibility guidelines are: + +    * Destination: Pair of host and port.  Mixing requests with +      different destinations may cause more connection setup and tear +      down. + +    * Method: http or https.  Usually moot given destination.  But +      mixing these may also cause connection churn. + +    * Transfer size: If you're moving 100MB at a time and you make your +      requests to the same policy class as a lot of small, fast event +      information that fast traffic is going to get stuck behind you +      and someone's experience is going to be miserable. + +    * Long poll requests: These are long-lived, must- do operations. +      They have a special home called AP_LONG_POLL. + +    * Concurrency: High concurrency (5 or more) and large transfer +      sizes are incompatible.  Another head-of-the-line problem.  High +      concurrency is tolerated when it's desired to get maximal +      throughput.  Mesh and texture downloads, for example. + +    * Pipelined: If your requests are not idempotent, stay away from +      anything marked 'soon' or 'yes'.  Hidden retries may be a +      problem for you.  For now, would also recommend keeping PUT and +      POST requests out of classes that may be pipelined.  Support for +      that is still a bit new. + +    If you haven't found a compatible match, you can either create a +    new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall +    class when all else fails.  Inventory query operations might be a +    candidate for a new class that supported pipelining on https:. +    Same with display name lookups and other bursty-at-login +    operations.  For other things, AP_DEFAULT will do what it can and +    will, in some way or another, tolerate any usage.  Whether the +    users' experiences are good are for you to determine. + +     +7.  FAQ + +    Q1.  What do these policy classes achieve? + +    A1.  Previously, HTTP-using code in the viewer was written as if +    it were some isolated, local operation that didn't have to +    consider resources, contention or impact on services and the +    larger environment.  The result was an application with on the +    order of 100 HTTP launch points in its codebase that could create +    dozens or even 100's of TCP connections zeroing in on grid +    services and disrupting networking equipment, web services and +    innocent users.  The use of policy classes (modeled on +    http://en.wikipedia.org/wiki/Class-based_queueing) is a means to +    restrict connection concurrency, good and necessary in itself.  In +    turn, that reduces demands on an expensive resource (connection +    setup and concurrency) which relieves strain on network points. +    That enables connection keepalive and opportunites for true +    improvements in throughput and user experience. + +    Another aspect of the classes is that they give some control over +    how competing demands for the network will be apportioned.  If +    mesh fetches, texture fetches and inventory queries are all being +    made at once, the relative weights of their classes' concurrency +    limits established that apportioning.  We now have an opportunity +    to balance the entire viewer system. + +    Q2.  How's that data sharing with refcounts working for you? + +    A2.  Meh.  It does reduce memory churn and the frequency at which +    free blocks must be moved between threads.  But it's also a design +    for static configuration and dynamic reconfiguration (not +    requiring a restart) is favored.  Creating new options for every +    request isn't too bad, it a sequence of "new, fill, request, +    release" for each requested operation.  That in contrast to doing +    the "new, fill, release" at startup.  The bad comes in getting at +    the source data.  One rule in this work was "no new thread +    problems."  And one source for those is pulling setting values out +    of gSettings in threads.  None of that is thread safe though we +    tend to get away with it. + +    Q3.  What needs to be done? + +    A3.  There's a To-Do list in _httpinternal.h.  It has both large +    and small projects here if someone would like to try changes. diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 008e4fd95c..f80d7f60f5 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -36,7 +36,8 @@  // General library to-do list  //  // - Implement policy classes.  Structure is mostly there just didn't -//   need it for the first consumer. +//   need it for the first consumer.  [Classes are there.  More +//   advanced features, like borrowing, aren't there yet.]  // - Consider Removing 'priority' from the request interface.  Its use  //   in an always active class can lead to starvation of low-priority  //   requests.  Requires coodination of priority values across all @@ -46,6 +47,7 @@  //   may not really need it.  // - Set/get for global policy and policy classes is clumsy.  Rework  //   it heading in a direction that allows for more dynamic behavior. +//   [Mostly fixed]  // - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the  //   pedantic.  // - Update downloader and other long-duration services are going to @@ -64,6 +66,12 @@  //   This won't help in the face of the router problems we've looked  //   at, however.  Detect starvation due to UDP activity and provide  //   feedback to it. +// - Change the transfer timeout scheme.  We're less interested in +//   absolute time, in most cases, than in continuous progress. +// - Many of the policy class settings are currently applied to the +//   entire class.  Some, like connection limits, would be better +//   applied to each destination target making multiple targets +//   independent.  //  // Integration to-do list  // - LLTextureFetch still needs a major refactor.  The use of @@ -73,7 +81,6 @@  //   the main source file.  // - Expand areas of usage eventually leading to the removal of LLCurl.  //   Rough order of expansion: -//   .  Mesh fetch  //   .  Avatar names  //   .  Group membership lists  //   .  Caps access in general @@ -97,8 +104,8 @@ namespace LLCore  {  // Maxium number of policy classes that can be defined. -// *TODO:  Currently limited to the default class, extend. -const int HTTP_POLICY_CLASS_LIMIT = 1; +// *TODO:  Currently limited to the default class + 1, extend. +const int HTTP_POLICY_CLASS_LIMIT = 8;  // Debug/informational tracing.  Used both  // as a global option and in per-request traces. @@ -129,6 +136,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10;  // Retries and time-on-queue are not included and aren't  // accounted for.  const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L;  const long HTTP_REQUEST_TIMEOUT_MIN = 0L;  const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; @@ -137,6 +145,11 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8;  const int HTTP_CONNECTION_LIMIT_MIN = 1;  const int HTTP_CONNECTION_LIMIT_MAX = 256; +// Miscellaneous defaults +const long HTTP_PIPELINING_DEFAULT = 0L; +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; +  // Tuning parameters  // Time worker thread sleeps after a pass through the diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index d49f615ac4..e56bc84174 100755 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -41,7 +41,8 @@ namespace LLCore  HttpLibcurl::HttpLibcurl(HttpService * service)  	: mService(service),  	  mPolicyCount(0), -	  mMultiHandles(NULL) +	  mMultiHandles(NULL), +	  mActiveHandles(NULL)  {} @@ -77,6 +78,9 @@ void HttpLibcurl::shutdown()  		delete [] mMultiHandles;  		mMultiHandles = NULL; + +		delete [] mActiveHandles; +		mActiveHandles = NULL;  	}  	mPolicyCount = 0; @@ -90,9 +94,12 @@ void HttpLibcurl::start(int policy_count)  	mPolicyCount = policy_count;  	mMultiHandles = new CURLM * [mPolicyCount]; +	mActiveHandles = new int [mPolicyCount]; +	  	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)  	{  		mMultiHandles[policy_class] = curl_multi_init(); +		mActiveHandles[policy_class] = 0;  	}  } @@ -110,8 +117,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()  	// Give libcurl some cycles to do I/O & callbacks  	for (int policy_class(0); policy_class < mPolicyCount; ++policy_class)  	{ -		if (! mMultiHandles[policy_class]) +		if (! mActiveHandles[policy_class] || ! mMultiHandles[policy_class]) +		{  			continue; +		}  		int running(0);  		CURLMcode status(CURLM_CALL_MULTI_PERFORM); @@ -132,12 +141,10 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport()  				CURL * handle(msg->easy_handle);  				CURLcode result(msg->data.result); -				if (completeRequest(mMultiHandles[policy_class], handle, result)) -				{ -					// Request is still active, don't get too sleepy -					ret = HttpService::NORMAL; -				} -				handle = NULL;			// No longer valid on return +				completeRequest(mMultiHandles[policy_class], handle, result); +				handle = NULL;					// No longer valid on return +				ret = HttpService::NORMAL;		// If anything completes, we may have a free slot. +												// Turning around quickly reduces connection gap by 7-10mS.  			}  			else if (CURLMSG_NONE == msg->msg)  			{ @@ -193,6 +200,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op)  	// On success, make operation active  	mActiveOps.insert(op); +	++mActiveHandles[op->mReqPolicy];  } @@ -214,6 +222,7 @@ bool HttpLibcurl::cancel(HttpHandle handle)  	// Drop references  	mActiveOps.erase(it); +	--mActiveHandles[op->mReqPolicy];  	op->release();  	return true; @@ -240,7 +249,7 @@ void HttpLibcurl::cancelRequest(HttpOpRequest * op)  	{  		LL_INFOS("CoreHttp") << "TRACE, RequestCanceled, Handle:  "  							 << static_cast<HttpHandle>(op) -							 << ", Status:  " << op->mStatus.toHex() +							 << ", Status:  " << op->mStatus.toTerseString()  							 << LL_ENDL;  	} @@ -275,6 +284,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode  	// Deactivate request  	mActiveOps.erase(it); +	--mActiveHandles[op->mReqPolicy];  	op->mCurlActive = false;  	// Set final status of request if it hasn't failed by other mechanisms yet @@ -316,7 +326,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode  	{  		LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle:  "  							 << static_cast<HttpHandle>(op) -							 << ", Status:  " << op->mStatus.toHex() +							 << ", Status:  " << op->mStatus.toTerseString()  							 << LL_ENDL;  	} @@ -336,19 +346,9 @@ int HttpLibcurl::getActiveCount() const  int HttpLibcurl::getActiveCountInClass(int policy_class) const  { -	int count(0); -	 -	for (active_set_t::const_iterator iter(mActiveOps.begin()); -		 mActiveOps.end() != iter; -		 ++iter) -	{ -		if ((*iter)->mReqPolicy == policy_class) -		{ -			++count; -		} -	} -	 -	return count; +	llassert_always(policy_class < mPolicyCount); + +	return mActiveHandles ? mActiveHandles[policy_class] : 0;  } diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 611f029ef5..67f98dd4f0 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -71,16 +71,22 @@ public:  	///  	/// @return			Indication of how long this method is  	///					willing to wait for next service call. +	/// +	/// Threading:  called by worker thread.  	HttpService::ELoopSpeed processTransport();  	/// Add request to the active list.  Caller is expected to have  	/// provided us with a reference count on the op to hold the  	/// request.  (No additional references will be added.) +	/// +	/// Threading:  called by worker thread.  	void addOp(HttpOpRequest * op);  	/// One-time call to set the number of policy classes to be  	/// serviced and to create the resources for each.  Value  	/// must agree with HttpPolicy::setPolicies() call. +	/// +	/// Threading:  called by init thread.  	void start(int policy_count);  	/// Synchronously stop libcurl operations.  All active requests @@ -91,9 +97,13 @@ public:  	/// respective reply queues.  	///  	/// Can be restarted with a start() call. +	/// +	/// Threading:  called by worker thread.  	void shutdown();  	/// Return global and per-class counts of active requests. +	/// +	/// Threading:  called by worker thread.  	int getActiveCount() const;  	int getActiveCountInClass(int policy_class) const; @@ -103,6 +113,7 @@ public:  	///  	/// @return			True if handle was found and operation canceled.  	/// +	/// Threading:  called by worker thread.  	bool cancel(HttpHandle handle);  protected: @@ -121,7 +132,8 @@ protected:  	HttpService *		mService;				// Simple reference, not owner  	active_set_t		mActiveOps;  	int					mPolicyCount; -	CURLM **			mMultiHandles; +	CURLM **			mMultiHandles;			// One handle per policy class +	int *				mActiveHandles;			// Active count per policy class  }; // end class HttpLibcurl  }  // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5cf5bc5930..5bb0654652 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ HttpOperation::HttpOperation()  	  mUserHandler(NULL),  	  mReqPolicy(HttpRequest::DEFAULT_POLICY_ID),  	  mReqPriority(0U), -	  mTracing(0) +	  mTracing(HTTP_TRACE_OFF)  {  	mMetricCreated = totalTime();  } @@ -94,7 +94,7 @@ void HttpOperation::stageFromRequest(HttpService *)  	// Default implementation should never be called.  This  	// indicates an operation making a transition that isn't  	// defined. -	LL_ERRS("HttpCore") << "Default stageFromRequest method may not be called." +	LL_ERRS("CoreHttp") << "Default stageFromRequest method may not be called."  						<< LL_ENDL;  } @@ -104,7 +104,7 @@ void HttpOperation::stageFromReady(HttpService *)  	// Default implementation should never be called.  This  	// indicates an operation making a transition that isn't  	// defined. -	LL_ERRS("HttpCore") << "Default stageFromReady method may not be called." +	LL_ERRS("CoreHttp") << "Default stageFromReady method may not be called."  						<< LL_ENDL;  } @@ -114,7 +114,7 @@ void HttpOperation::stageFromActive(HttpService *)  	// Default implementation should never be called.  This  	// indicates an operation making a transition that isn't  	// defined. -	LL_ERRS("HttpCore") << "Default stageFromActive method may not be called." +	LL_ERRS("CoreHttp") << "Default stageFromActive method may not be called."  						<< LL_ENDL;  } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 914627fad0..937a61187d 100755 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -72,7 +72,7 @@ class HttpService;  class HttpOperation : public LLCoreInt::RefCounted  {  public: -	/// Threading:  called by a consumer/application thread. +	/// Threading:  called by consumer thread.  	HttpOperation();  protected: @@ -108,7 +108,7 @@ public:  	///							by the worker thread.  This is passible data  	///							until notification is performed.  	/// -	/// Threading:  called by application thread. +	/// Threading:  called by consumer thread.  	///  	void setReplyPath(HttpReplyQueue * reply_queue,  					  HttpHandler * handler); @@ -141,7 +141,7 @@ public:  	/// call to HttpRequest::update().  This method does the necessary  	/// dispatching.  	/// -	/// Threading:  called by application thread. +	/// Threading:  called by consumer thread.  	///  	virtual void visitNotifier(HttpRequest *); diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index d8057364c4..43dd069bc6 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer,  							   unsigned int * last,  							   unsigned int * length); +// Similar for Retry-After headers.  Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return		0 if successfully parsed and seconds time delta +//				returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); +  // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and  // escape and format it for a tracing line in logging.  Absolutely @@ -74,14 +83,16 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub,  							   std::string & safe_line); -// OS-neutral string comparisons of various types -int os_strncasecmp(const char *s1, const char *s2, size_t n); -int os_strcasecmp(const char *s1, const char *s2); -char * os_strtok_r(char *str, const char *delim, char **saveptr); - +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); -static const char * const hdr_whitespace(" \t"); -static const char * const hdr_separator(": \t"); +// Error testing and reporting for libcurl status codes +void check_curl_easy_code(CURLcode code); +void check_curl_easy_code(CURLcode code, int curl_setopt_option);  } // end anonymous namespace @@ -104,12 +115,15 @@ HttpOpRequest::HttpOpRequest()  	  mCurlService(NULL),  	  mCurlHeaders(NULL),  	  mCurlBodyPos(0), +	  mCurlTemp(NULL), +	  mCurlTempLen(0),  	  mReplyBody(NULL),  	  mReplyOffset(0),  	  mReplyLength(0),  	  mReplyFullLength(0),  	  mReplyHeaders(NULL),  	  mPolicyRetries(0), +	  mPolicy503Retries(0),  	  mPolicyRetryAt(HttpTime(0)),  	  mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT)  { @@ -153,6 +167,10 @@ HttpOpRequest::~HttpOpRequest()  		mCurlHeaders = NULL;  	} +	delete [] mCurlTemp; +	mCurlTemp = NULL; +	mCurlTempLen = 0; +	  	if (mReplyBody)  	{  		mReplyBody->release(); @@ -208,6 +226,11 @@ void HttpOpRequest::stageFromActive(HttpService * service)  		mCurlHeaders = NULL;  	} +	// Also not needed on the other side +	delete [] mCurlTemp; +	mCurlTemp = NULL; +	mCurlTempLen = 0; +	  	addAsReply();  } @@ -226,6 +249,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request)  			response->setRange(mReplyOffset, mReplyLength, mReplyFullLength);  		}  		response->setContentType(mReplyConType); +		response->setRetries(mPolicyRetries, mPolicy503Retries);  		mUserHandler->onCompleted(static_cast<HttpHandle>(this), response); @@ -335,6 +359,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,  		{  			mProcFlags |= PF_SAVE_HEADERS;  		} +		if (options->getUseRetryAfter()) +		{ +			mProcFlags |= PF_USE_RETRY_AFTER; +		}  		mPolicyRetryLimit = options->getRetries();  		mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX);  		mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); @@ -350,6 +378,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id,  //  HttpStatus HttpOpRequest::prepareRequest(HttpService * service)  { +	CURLcode code; +	  	// Scrub transport and result data for retried op case  	mCurlActive = false;  	mCurlHandle = NULL; @@ -379,64 +409,108 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)  	// *FIXME:  better error handling later  	HttpStatus status; -	// Get policy options +	// Get global policy options  	HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions());  	mCurlHandle = LLCurl::createStandardCurlHandle(); - -	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); -	curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION,  readCallback);	 -	curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); -	curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); -	curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); -	curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); -	curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT);	 - -	const std::string * opt_value(NULL); -	long opt_long(0L); -	policy.get(HttpRequest::GP_LLPROXY, &opt_long); -	if (opt_long) +	if (! mCurlHandle) +	{ +		// We're in trouble.  We'll continue but it won't go well. +		LL_WARNS("CoreHttp") << "Failed to allocate libcurl easy handle.  Continuing." +							 << LL_ENDL; +		return HttpStatus(HttpStatus::LLCORE, HE_BAD_ALLOC); +	} +	code = curl_easy_setopt(mCurlHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); +	check_curl_easy_code(code, CURLOPT_IPRESOLVE); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); +	check_curl_easy_code(code, CURLOPT_NOSIGNAL); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1); +	check_curl_easy_code(code, CURLOPT_NOPROGRESS); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); +	check_curl_easy_code(code, CURLOPT_URL); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); +	check_curl_easy_code(code, CURLOPT_PRIVATE); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); +	check_curl_easy_code(code, CURLOPT_ENCODING); + +	// The Linksys WRT54G V5 router has an issue with frequent +	// DNS lookups from LAN machines.  If they happen too often, +	// like for every HTTP request, the router gets annoyed after +	// about 700 or so requests and starts issuing TCP RSTs to +	// new connections.  Reuse the DNS lookups for even a few +	// seconds and no RSTs. +	code = curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15); +	check_curl_easy_code(code, CURLOPT_DNS_CACHE_TIMEOUT); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); +	check_curl_easy_code(code, CURLOPT_AUTOREFERER); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); +	check_curl_easy_code(code, CURLOPT_FOLLOWLOCATION); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, HTTP_REDIRECTS_DEFAULT); +	check_curl_easy_code(code, CURLOPT_MAXREDIRS); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); +	check_curl_easy_code(code, CURLOPT_WRITEFUNCTION); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, this); +	check_curl_easy_code(code, CURLOPT_WRITEDATA); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); +	check_curl_easy_code(code, CURLOPT_READFUNCTION); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, this); +	check_curl_easy_code(code, CURLOPT_READDATA); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); +	check_curl_easy_code(code, CURLOPT_SSL_VERIFYPEER); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); +	check_curl_easy_code(code, CURLOPT_SSL_VERIFYHOST); + +	if (policy.mUseLLProxy)  	{  		// Use the viewer-based thread-safe API which has a  		// fast/safe check for proxy enable.  Would like to  		// encapsulate this someway...  		LLProxy::getInstance()->applyProxySettings(mCurlHandle);  	} -	else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) +	else if (policy.mHttpProxy.size())  	{  		// *TODO:  This is fine for now but get fuller socks5/  		// authentication thing going later.... -		curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); -		curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); +		check_curl_easy_code(code, CURLOPT_PROXY); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); +		check_curl_easy_code(code, CURLOPT_PROXYTYPE);  	} -	if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) +	if (policy.mCAPath.size())  	{ -		curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); +		check_curl_easy_code(code, CURLOPT_CAPATH);  	} -	if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) +	if (policy.mCAFile.size())  	{ -		curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); +		check_curl_easy_code(code, CURLOPT_CAINFO);  	}  	switch (mReqMethod)  	{  	case HOR_GET: -		curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); +		check_curl_easy_code(code, CURLOPT_HTTPGET);  		mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");  		mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300");  		break;  	case HOR_POST:  		{ -			curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); -			curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); +			check_curl_easy_code(code, CURLOPT_POST); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); +			check_curl_easy_code(code, CURLOPT_ENCODING);  			long data_size(0);  			if (mReqBody)  			{  				data_size = mReqBody->size();  			} -			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); -			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast<void *>(NULL)); +			check_curl_easy_code(code, CURLOPT_POSTFIELDS); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); +			check_curl_easy_code(code, CURLOPT_POSTFIELDSIZE);  			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");  			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive");  			mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); @@ -445,14 +519,17 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)  	case HOR_PUT:  		{ -			curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); +			check_curl_easy_code(code, CURLOPT_UPLOAD);  			long data_size(0);  			if (mReqBody)  			{  				data_size = mReqBody->size();  			} -			curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); -			curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); +			check_curl_easy_code(code, CURLOPT_INFILESIZE); +			code = curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); +			check_curl_easy_code(code, CURLOPT_POSTFIELDS);  			mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:");  			// *TODO: Should this be 'Keep-Alive' ?  			mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); @@ -470,9 +547,12 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)  	// Tracing  	if (mTracing >= HTTP_TRACE_CURL_HEADERS)  	{ -		curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); -		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); -		curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); +		check_curl_easy_code(code, CURLOPT_VERBOSE); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, this); +		check_curl_easy_code(code, CURLOPT_DEBUGDATA); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGFUNCTION, debugCallback); +		check_curl_easy_code(code, CURLOPT_DEBUGFUNCTION);  	}  	// There's a CURLOPT for this now... @@ -500,13 +580,22 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)  	// Request options  	long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); +	long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT);  	if (mReqOptions) -	{ + 	{  		timeout = mReqOptions->getTimeout();  		timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); +		xfer_timeout = mReqOptions->getTransferTimeout(); +		xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); +	} +	if (xfer_timeout == 0L) +	{ +		xfer_timeout = timeout;  	} -	curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); -	curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); +	check_curl_easy_code(code, CURLOPT_TIMEOUT); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); +	check_curl_easy_code(code, CURLOPT_CONNECTTIMEOUT);  	// Request headers  	if (mReqHeaders) @@ -514,12 +603,15 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service)  		// Caller's headers last to override  		mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders);  	} -	curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); +	code = curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); +	check_curl_easy_code(code, CURLOPT_HTTPHEADER); -	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) +	if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER))  	{ -		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); -		curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); +		check_curl_easy_code(code, CURLOPT_HEADERFUNCTION); +		code = curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); +		check_curl_easy_code(code, CURLOPT_HEADERDATA);  	}  	if (status) @@ -560,7 +652,7 @@ size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void  		{  			// Warn but continue if the read position moves beyond end-of-body  			// for some reason. -			LL_WARNS("HttpCore") << "Request body position beyond body size.  Truncating request body." +			LL_WARNS("CoreHttp") << "Request body position beyond body size.  Truncating request body."  								 << LL_ENDL;  		}  		return 0; @@ -577,10 +669,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi  {  	static const char status_line[] = "HTTP/";  	static const size_t status_line_len = sizeof(status_line) - 1; - -	static const char con_ran_line[] = "content-range:"; -	static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - +	static const char con_ran_line[] = "content-range"; +	static const char con_retry_line[] = "retry-after"; +	  	HttpOpRequest * op(static_cast<HttpOpRequest *>(userdata));  	const size_t hdr_size(size * nmemb); @@ -594,6 +685,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi  		op->mReplyOffset = 0;  		op->mReplyLength = 0;  		op->mReplyFullLength = 0; +		op->mReplyRetryAfter = 0;  		op->mStatus = HttpStatus();  		if (op->mReplyHeaders)  		{ @@ -612,6 +704,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi  			--wanted_hdr_size;  		}  	} + +	// Copy and normalize header fragments for the following +	// stages.  Would like to modify the data in-place but that +	// may not be allowed and we need one byte extra for NUL. +	// At the end of this we will have: +	// +	// If ':' present in header: +	//   1.  name points to text to left of colon which +	//       will be ascii lower-cased and left and right +	//       trimmed of whitespace. +	//   2.  value points to text to right of colon which +	//       will be left trimmed of whitespace. +	// Otherwise: +	//   1.  name points to header which will be left +	//       trimmed of whitespace. +	//   2.  value is NULL +	// Any non-NULL pointer may point to a zero-length string. +	// +	if (wanted_hdr_size >= op->mCurlTempLen) +	{ +		delete [] op->mCurlTemp; +		op->mCurlTempLen = 2 * wanted_hdr_size + 1; +		op->mCurlTemp = new char [op->mCurlTempLen]; +	} +	memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); +	op->mCurlTemp[wanted_hdr_size] = '\0'; +	char * name(op->mCurlTemp); +	char * value(strchr(name, ':')); +	if (value) +	{ +		*value++ = '\0'; +		os_strlower(name); +		name = os_strtrim(name); +		value = os_strltrim(value); +	} +	else +	{ +		// Doesn't look well-formed, do minimal normalization on it +		name = os_strltrim(name); +	} + +	// Normalized, now reject headers with empty names. +	if (! *name) +	{ +		// No use continuing +		return hdr_size; +	}  	// Save header if caller wants them in the response  	if (is_header && op->mProcFlags & PF_SAVE_HEADERS) @@ -621,43 +760,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi  		{  			op->mReplyHeaders = new HttpHeaders;  		} -		op->mReplyHeaders->appendNormal(hdr_data, wanted_hdr_size); +		op->mReplyHeaders->append(name, value ? value : "");  	} +	// From this point, header-specific processors are free to +	// modify the header value. +	  	// Detect and parse 'Content-Range' headers -	if (is_header && op->mProcFlags & PF_SCAN_RANGE_HEADER) +	if (is_header +		&& op->mProcFlags & PF_SCAN_RANGE_HEADER +		&& value && *value +		&& ! strcmp(name, con_ran_line))  	{ -		char hdr_buffer[128];			// Enough for a reasonable header -		size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); -		 -		memcpy(hdr_buffer, hdr_data, frag_size); -		hdr_buffer[frag_size] = '\0'; -		if (frag_size > con_ran_line_len && -			! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) +		unsigned int first(0), last(0), length(0); +		int status; + +		if (! (status = parse_content_range_header(value, &first, &last, &length)))  		{ -			unsigned int first(0), last(0), length(0); -			int status; +			// Success, record the fragment position +			op->mReplyOffset = first; +			op->mReplyLength = last - first + 1; +			op->mReplyFullLength = length; +		} +		else if (-1 == status) +		{ +			// Response is badly formed and shouldn't be accepted +			op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); +		} +		else +		{ +			// Ignore the unparsable. +			LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header:  '" +									  << std::string(hdr_data, wanted_hdr_size) +									  << "'.  Ignoring." +									  << LL_ENDL; +		} +	} -			if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) -			{ -				// Success, record the fragment position -				op->mReplyOffset = first; -				op->mReplyLength = last - first + 1; -				op->mReplyFullLength = length; -			} -			else if (-1 == status) -			{ -				// Response is badly formed and shouldn't be accepted -				op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); -			} -			else -			{ -				// Ignore the unparsable. -				LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header:  '" -										  << std::string(hdr_data, frag_size) -										  << "'.  Ignoring." -										  << LL_ENDL; -			} +	// Detect and parse 'Retry-After' headers +	if (is_header +		&& op->mProcFlags & PF_USE_RETRY_AFTER +		&& value && *value +		&& ! strcmp(name, con_retry_line)) +	{ +		int time(0); +		if (! parse_retry_after_header(value, &time)) +		{ +			op->mReplyRetryAfter = time;  		}  	} @@ -772,14 +921,16 @@ int parse_content_range_header(char * buffer,  							   unsigned int * last,  							   unsigned int * length)  { +	static const char * const hdr_whitespace(" \t"); +  	char * tok_state(NULL), * tok(NULL);  	bool match(true); -	if (! os_strtok_r(buffer, hdr_separator, &tok_state)) +	if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state)))  		match = false; -	if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) -		match = 0 == os_strcasecmp("bytes", tok); -	if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) +	else +		match = (0 == os_strcasecmp("bytes", tok)); +	if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state)))  		match = false;  	if (match)  	{ @@ -818,6 +969,25 @@ int parse_content_range_header(char * buffer,  } +int parse_retry_after_header(char * buffer, int * time) +{ +	char * endptr(buffer); +	long lcl_time(strtol(buffer, &endptr, 10)); +	if (*endptr == '\0' && endptr != buffer && lcl_time > 0) +	{ +		*time = lcl_time; +		return 0; +	} + +	// Could attempt to parse HTTP time here but we're not really +	// interested in it.  Scheduling based on wallclock time on +	// user hardware will lead to tears. +	 +	// Header is there but badly/unexpectedly formed, try to ignore it. +	return 1; +} + +  void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line)  {  	std::string out; @@ -854,15 +1024,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin  } -int os_strncasecmp(const char *s1, const char *s2, size_t n) -{ -#if LL_WINDOWS -	return _strnicmp(s1, s2, n); -#else -	return strncasecmp(s1, s2, n); -#endif	// LL_WINDOWS -} -  int os_strcasecmp(const char *s1, const char *s2)  { @@ -884,6 +1045,73 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate)  } -}  // end anonymous namespace +void os_strlower(char * str) +{ +	for (char c(0); (c = *str); ++str) +	{ +		*str = tolower(c); +	} +} -		 + +char * os_strtrim(char * lstr) +{ +	while (' ' == *lstr || '\t' == *lstr) +	{ +		++lstr; +	} +	if (*lstr) +	{ +		char * rstr(lstr + strlen(lstr)); +		while (lstr < rstr && *--rstr) +		{ +			if (' ' == *rstr || '\t' == *rstr) +			{ +				*rstr = '\0'; +			} +		} +		llassert(lstr <= rstr); +	} +	return lstr; +} + + +char * os_strltrim(char * lstr) +{ +	while (' ' == *lstr || '\t' == *lstr) +	{ +		++lstr; +	} +	return lstr; +} + + +void check_curl_easy_code(CURLcode code, int curl_setopt_option) +{ +	if (CURLE_OK != code) +	{ +		// Comment from old llcurl code which may no longer apply: +		// +		// linux appears to throw a curl error once per session for a bad initialization +		// at a pretty random time (when enabling cookies). +		LL_WARNS("CoreHttp") << "libcurl error detected:  " << curl_easy_strerror(code) +							 << ", curl_easy_setopt option:  " << curl_setopt_option +							 << LL_ENDL; +	} +} + + +void check_curl_easy_code(CURLcode code) +{ +	if (CURLE_OK != code) +	{ +		// Comment from old llcurl code which may no longer apply: +		// +		// linux appears to throw a curl error once per session for a bad initialization +		// at a pretty random time (when enabling cookies). +		LL_WARNS("CoreHttp") << "libcurl error detected:  " << curl_easy_strerror(code) +							 << LL_ENDL; +	} +} + +}  // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 74a349b0bf..2f628b5aba 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -157,6 +157,7 @@ protected:  	unsigned int		mProcFlags;  	static const unsigned int	PF_SCAN_RANGE_HEADER = 0x00000001U;  	static const unsigned int	PF_SAVE_HEADERS = 0x00000002U; +	static const unsigned int	PF_USE_RETRY_AFTER = 0x00000004U;  public:  	// Request data @@ -174,6 +175,8 @@ public:  	HttpService *		mCurlService;  	curl_slist *		mCurlHeaders;  	size_t				mCurlBodyPos; +	char *				mCurlTemp;				// Scratch buffer for header processing +	size_t				mCurlTempLen;  	// Result data  	HttpStatus			mStatus; @@ -183,9 +186,11 @@ public:  	size_t				mReplyFullLength;  	HttpHeaders *		mReplyHeaders;  	std::string			mReplyConType; +	int					mReplyRetryAfter;  	// Policy data  	int					mPolicyRetries; +	int					mPolicy503Retries;  	HttpTime			mPolicyRetryAt;  	int					mPolicyRetryLimit;  };  // end class HttpOpRequest diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 8198528a9b..a5363f9170 100755 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@  #include "_httpopsetget.h"  #include "httpcommon.h" +#include "httprequest.h"  #include "_httpservice.h"  #include "_httppolicy.h" @@ -43,10 +44,11 @@ namespace LLCore  HttpOpSetGet::HttpOpSetGet()  	: HttpOperation(), -	  mIsGlobal(false), -	  mDoSet(false), -	  mSetting(-1),				// Nothing requested -	  mLongValue(0L) +	  mReqOption(HttpRequest::PO_CONNECTION_LIMIT), +	  mReqClass(HttpRequest::INVALID_POLICY_ID), +	  mReqDoSet(false), +	  mReqLongValue(0L), +	  mReplyLongValue(0L)  {} @@ -54,37 +56,84 @@ HttpOpSetGet::~HttpOpSetGet()  {} -void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass)  { -	mIsGlobal = true; -	mSetting = setting; +	HttpStatus status; +	 +	mReqOption = opt; +	mReqClass = pclass; +	return status;  } -void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value)  { -	mIsGlobal = true; -	mDoSet = true; -	mSetting = setting; -	mStrValue = value; +	HttpStatus status; + +	if (! HttpService::sOptionDesc[opt].mIsLong) +	{ +		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); +	} +	if (! HttpService::sOptionDesc[opt].mIsDynamic) +	{ +		return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); +	} +	 +	mReqOption = opt; +	mReqClass = pclass; +	mReqDoSet = true; +	mReqLongValue = value; +	 +	return status;  } -void HttpOpSetGet::stageFromRequest(HttpService * service) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value)  { -	HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); -	HttpRequest::EGlobalPolicy setting(static_cast<HttpRequest::EGlobalPolicy>(mSetting)); +	HttpStatus status; + +	if (HttpService::sOptionDesc[opt].mIsLong) +	{ +		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); +	} +	if (! HttpService::sOptionDesc[opt].mIsDynamic) +	{ +		return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); +	} + +	mReqOption = opt; +	mReqClass = pclass; +	mReqDoSet = true; +	mReqStrValue = value; -	if (mDoSet) +	return status; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ +	if (mReqDoSet)  	{ -		mStatus = pol_opt.set(setting, mStrValue); +		if (HttpService::sOptionDesc[mReqOption].mIsLong) +		{ +			mStatus = service->setPolicyOption(mReqOption, mReqClass, +											   mReqLongValue, &mReplyLongValue); +		} +		else +		{ +			mStatus = service->setPolicyOption(mReqOption, mReqClass, +											   mReqStrValue, &mReplyStrValue); +		}  	} -	if (mStatus) +	else  	{ -		const std::string * value(NULL); -		if ((mStatus = pol_opt.get(setting, &value))) +		if (HttpService::sOptionDesc[mReqOption].mIsLong) +		{ +			mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); +		} +		else  		{ -			mStrValue = *value; +			mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue);  		}  	} diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h index 6966b9d94e..a1e76dd429 100755 --- a/indra/llcorehttp/_httpopsetget.h +++ b/indra/llcorehttp/_httpopsetget.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -46,7 +46,10 @@ namespace LLCore  /// configuration settings.  ///  /// *NOTE:  Expect this to change.  Don't really like it yet. - +/// +/// *TODO:  Can't return values to caller yet.  Need to do +/// something better with HttpResponse and visitNotifier(). +///  class HttpOpSetGet : public HttpOperation  {  public: @@ -61,19 +64,23 @@ private:  public:  	/// Threading:  called by application thread -	void setupGet(HttpRequest::EGlobalPolicy setting); -	void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); +	HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); +	HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); +	HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value);  	virtual void stageFromRequest(HttpService *);  public:  	// Request data -	bool				mIsGlobal; -	bool				mDoSet; -	int					mSetting; -	long				mLongValue; -	std::string			mStrValue; - +	HttpRequest::EPolicyOption	mReqOption; +	HttpRequest::policy_t		mReqClass; +	bool						mReqDoSet; +	long						mReqLongValue; +	std::string					mReqStrValue; + +	// Reply Data +	long						mReplyLongValue; +	std::string					mReplyStrValue;  };  // end class HttpOpSetGet diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 014bd37e2e..fd5a93e192 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -41,57 +41,70 @@ namespace LLCore  // Per-policy-class data for a running system. -// Collection of queues, parameters, history, metrics, etc. +// Collection of queues, options and other data  // for a single policy class.  //  // Threading:  accessed only by worker thread -struct HttpPolicy::State +struct HttpPolicy::ClassState  {  public: -	State() -		: mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), -		  mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), -		  mConnMin(1), -		  mNextSample(0), -		  mErrorCount(0), -		  mErrorFactor(0) +	ClassState() +		: mThrottleEnd(0), +		  mThrottleLeft(0L), +		  mRequestCount(0L)  		{}  	HttpReadyQueue		mReadyQueue;  	HttpRetryQueue		mRetryQueue;  	HttpPolicyClass		mOptions; - -	long				mConnMax; -	long				mConnAt; -	long				mConnMin; - -	HttpTime			mNextSample; -	unsigned long		mErrorCount; -	unsigned long		mErrorFactor; +	HttpTime			mThrottleEnd; +	long				mThrottleLeft; +	long				mRequestCount;  };  HttpPolicy::HttpPolicy(HttpService * service) -	: mActiveClasses(0), -	  mState(NULL), -	  mService(service) -{} +	: mService(service) +{ +	// Create default class +	mClasses.push_back(new ClassState()); +}  HttpPolicy::~HttpPolicy()  {  	shutdown(); + +	for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) +	{ +		delete (*it); +	} +	mClasses.clear();  	mService = NULL;  } +HttpRequest::policy_t HttpPolicy::createPolicyClass() +{ +	const HttpRequest::policy_t policy_class(mClasses.size()); +	if (policy_class >= HTTP_POLICY_CLASS_LIMIT) +	{ +		return HttpRequest::INVALID_POLICY_ID; +	} +	mClasses.push_back(new ClassState()); +	return policy_class; +} + +  void HttpPolicy::shutdown()  { -	for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) +	for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)  	{ -		HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); +		ClassState & state(*mClasses[policy_class]); +		 +		HttpRetryQueue & retryq(state.mRetryQueue);  		while (! retryq.empty())  		{  			HttpOpRequest * op(retryq.top()); @@ -101,7 +114,7 @@ void HttpPolicy::shutdown()  			op->release();  		} -		HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); +		HttpReadyQueue & readyq(state.mReadyQueue);  		while (! readyq.empty())  		{  			HttpOpRequest * op(readyq.top()); @@ -111,28 +124,11 @@ void HttpPolicy::shutdown()  			op->release();  		}  	} -	delete [] mState; -	mState = NULL; -	mActiveClasses = 0;  } -void HttpPolicy::start(const HttpPolicyGlobal & global, -					   const std::vector<HttpPolicyClass> & classes) -{ -	llassert_always(! mState); - -	mGlobalOptions = global; -	mActiveClasses = classes.size(); -	mState = new State [mActiveClasses]; -	for (int i(0); i < mActiveClasses; ++i) -	{ -		mState[i].mOptions = classes[i]; -		mState[i].mConnMax = classes[i].mConnectionLimit; -		mState[i].mConnAt = mState[i].mConnMax; -		mState[i].mConnMin = 2; -	} -} +void HttpPolicy::start() +{}  void HttpPolicy::addOp(HttpOpRequest * op) @@ -140,7 +136,8 @@ void HttpPolicy::addOp(HttpOpRequest * op)  	const int policy_class(op->mReqPolicy);  	op->mPolicyRetries = 0; -	mState[policy_class].mReadyQueue.push(op); +	op->mPolicy503Retries = 0; +	mClasses[policy_class]->mReadyQueue.push(op);  } @@ -155,25 +152,39 @@ void HttpPolicy::retryOp(HttpOpRequest * op)  			5000000				// ... to every 5.0 S.  		};  	static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); -	 +	static const HttpStatus error_503(503); +  	const HttpTime now(totalTime());  	const int policy_class(op->mReqPolicy); -	 -	const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); +	HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); +	bool external_delta(false); + +	if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) +	{ +		delta = op->mReplyRetryAfter * U64L(1000000); +		external_delta = true; +	}  	op->mPolicyRetryAt = now + delta;  	++op->mPolicyRetries; -	LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) -						 << " retry " << op->mPolicyRetries -						 << " scheduled for +" << (delta / HttpTime(1000)) -						 << " mS.  Status:  " << op->mStatus.toHex() -						 << LL_ENDL; -	if (op->mTracing > 0) +	if (error_503 == op->mStatus) +	{ +		++op->mPolicy503Retries; +	} +	LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) +						  << " retry " << op->mPolicyRetries +						  << " scheduled in " << (delta / HttpTime(1000)) +						  << " mS (" << (external_delta ? "external" : "internal") +						  << ").  Status:  " << op->mStatus.toTerseString() +						  << LL_ENDL; +	if (op->mTracing > HTTP_TRACE_OFF)  	{  		LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle:  "  							 << static_cast<HttpHandle>(op) +							 << ", Delta:  " << (delta / HttpTime(1000)) +							 << ", Retries:  " << op->mPolicyRetries  							 << LL_ENDL;  	} -	mState[policy_class].mRetryQueue.push(op); +	mClasses[policy_class]->mRetryQueue.push(op);  } @@ -188,21 +199,43 @@ void HttpPolicy::retryOp(HttpOpRequest * op)  // the worker thread may sleep hard otherwise will ask for  // normal polling frequency.  // +// Implements a client-side request rate throttle as well. +// This is intended to mimic and predict throttling behavior +// of grid services but that is difficult to do with different +// time bases.  This also represents a rigid coupling between +// viewer and server that makes it hard to change parameters +// and I hope we can make this go away with pipelining. +//  HttpService::ELoopSpeed HttpPolicy::processReadyQueue()  {  	const HttpTime now(totalTime());  	HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP);  	HttpLibcurl & transport(mService->getTransport()); -	for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) +	for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)  	{ -		State & state(mState[policy_class]); -		int active(transport.getActiveCountInClass(policy_class)); -		int needed(state.mConnAt - active);		// Expect negatives here - +		ClassState & state(*mClasses[policy_class]);  		HttpRetryQueue & retryq(state.mRetryQueue);  		HttpReadyQueue & readyq(state.mReadyQueue); + +		if (retryq.empty() && readyq.empty()) +		{ +			continue; +		} +		const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); +		const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); + +		if (throttle_current && state.mThrottleLeft <= 0) +		{ +			// Throttled condition, don't serve this class but don't sleep hard. +			result = HttpService::NORMAL; +			continue; +		} + +		int active(transport.getActiveCountInClass(policy_class)); +		int needed(state.mOptions.mConnectionLimit - active);		// Expect negatives here +  		if (needed > 0)  		{  			// First see if we have any retries... @@ -216,10 +249,27 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()  				op->stageFromReady(mService);  				op->release(); -					 + +				++state.mRequestCount;  				--needed; +				if (throttle_enabled) +				{ +					if (now >= state.mThrottleEnd) +					{ +						// Throttle expired, move to next window +						LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft +											  << " requests to go and " << state.mRequestCount +											  << " requests issued." << LL_ENDL; +						state.mThrottleLeft = state.mOptions.mThrottleRate; +						state.mThrottleEnd = now + HttpTime(1000000); +					} +					if (--state.mThrottleLeft <= 0) +					{ +						goto throttle_on; +					} +				}  			} -		 +			  			// Now go on to the new requests...  			while (needed > 0 && ! readyq.empty())  			{ @@ -229,10 +279,29 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()  				op->stageFromReady(mService);  				op->release(); +				++state.mRequestCount;  				--needed; +				if (throttle_enabled) +				{ +					if (now >= state.mThrottleEnd) +					{ +						// Throttle expired, move to next window +						LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft +											  << " requests to go and " << state.mRequestCount +											  << " requests issued." << LL_ENDL; +						state.mThrottleLeft = state.mOptions.mThrottleRate; +						state.mThrottleEnd = now + HttpTime(1000000); +					} +					if (--state.mThrottleLeft <= 0) +					{ +						goto throttle_on; +					} +				}  			}  		} -				 + +	throttle_on: +		  		if (! readyq.empty() || ! retryq.empty())  		{  			// If anything is ready, continue looping... @@ -246,9 +315,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue()  bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority)  { -	for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) +	for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)  	{ -		State & state(mState[policy_class]); +		ClassState & state(*mClasses[policy_class]);  		// We don't scan retry queue because a priority change there  		// is meaningless.  The request will be issued based on retry  		// intervals not priority value, which is now moot. @@ -276,9 +345,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior  bool HttpPolicy::cancel(HttpHandle handle)  { -	for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) +	for (int policy_class(0); policy_class < mClasses.size(); ++policy_class)  	{ -		State & state(mState[policy_class]); +		ClassState & state(*mClasses[policy_class]);  		// Scan retry queue  		HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); @@ -337,14 +406,14 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)  		LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op)  							 << " failed after " << op->mPolicyRetries  							 << " retries.  Reason:  " << op->mStatus.toString() -							 << " (" << op->mStatus.toHex() << ")" +							 << " (" << op->mStatus.toTerseString() << ")"  							 << LL_ENDL;  	}  	else if (op->mPolicyRetries)  	{ -		LL_WARNS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) -							 << " succeeded on retry " << op->mPolicyRetries << "." -							 << LL_ENDL; +		LL_DEBUGS("CoreHttp") << "HTTP request " << static_cast<HttpHandle>(op) +							  << " succeeded on retry " << op->mPolicyRetries << "." +							  << LL_ENDL;  	}  	op->stageFromActive(mService); @@ -352,13 +421,21 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op)  	return false;						// not active  } +	 +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) +{ +	llassert_always(pclass >= 0 && pclass < mClasses.size()); +	 +	return mClasses[pclass]->mOptions; +} +  int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const  { -	if (policy_class < mActiveClasses) +	if (policy_class < mClasses.size())  	{ -		return (mState[policy_class].mReadyQueue.size() -				+ mState[policy_class].mRetryQueue.size()); +		return (mClasses[policy_class]->mReadyQueue.size() +				+ mClasses[policy_class]->mRetryQueue.size());  	}  	return 0;  } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 03d92c0b8e..bf1aa74267 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -60,6 +60,9 @@ private:  	void operator=(const HttpPolicy &);			// Not defined  public: +	/// Threading:  called by init thread. +	HttpRequest::policy_t createPolicyClass(); +	  	/// Cancel all ready and retry requests sending them to  	/// their notification queues.  Release state resources  	/// making further request handling impossible. @@ -71,9 +74,8 @@ public:  	/// requests.  One-time call invoked before starting  	/// the worker thread.  	/// -	/// Threading:  called by application thread -	void start(const HttpPolicyGlobal & global, -			   const std::vector<HttpPolicyClass> & classes); +	/// Threading:  called by init thread +	void start();  	/// Give the policy layer some cycles to scan the ready  	/// queue promoting higher-priority requests to active @@ -93,7 +95,7 @@ public:  	/// and should not be modified by anyone until retrieved  	/// from queue.  	/// -	/// Threading:  called by any thread +	/// Threading:  called by worker thread  	void addOp(HttpOpRequest *);  	/// Similar to addOp, used when a caller wants to retry a @@ -130,30 +132,39 @@ public:  	/// Threading:  called by worker thread  	bool stageAfterCompletion(HttpOpRequest * op); -	// Get pointer to global policy options.  Caller is expected -	// to do context checks like no setting once running. +	/// Get a reference to global policy options.  Caller is expected +	/// to do context checks like no setting once running.  These +	/// are done, for example, in @see HttpService interfaces.  	///  	/// Threading:  called by any thread *but* the object may  	/// only be modified by the worker thread once running. -	///  	HttpPolicyGlobal & getGlobalOptions()  		{  			return mGlobalOptions;  		} +	/// Get a reference to class policy options.  Caller is expected +	/// to do context checks like no setting once running.  These +	/// are done, for example, in @see HttpService interfaces. +	/// +	/// Threading:  called by any thread *but* the object may +	/// only be modified by the worker thread once running and +	/// read accesses by other threads are exposed to races at +	/// that point. +	HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); +	  	/// Get ready counts for a particular policy class  	///  	/// Threading:  called by worker thread  	int getReadyCount(HttpRequest::policy_t policy_class) const;  protected: -	struct State; - -	int									mActiveClasses; -	State *								mState; -	HttpService *						mService;				// Naked pointer, not refcounted, not owner -	HttpPolicyGlobal					mGlobalOptions; +	struct ClassState; +	typedef std::vector<ClassState *>	class_list_t; +	HttpPolicyGlobal					mGlobalOptions; +	class_list_t						mClasses; +	HttpService *						mService;				// Naked pointer, not refcounted, not owner  };  // end class HttpPolicy  }  // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index a23b81322c..f34a8e9f1e 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -34,10 +34,10 @@ namespace LLCore  HttpPolicyClass::HttpPolicyClass() -	: mSetMask(0UL), -	  mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), +	: mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),  	  mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), -	  mPipelining(0) +	  mPipelining(HTTP_PIPELINING_DEFAULT), +	  mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT)  {} @@ -49,75 +49,75 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other)  {  	if (this != &other)  	{ -		mSetMask = other.mSetMask;  		mConnectionLimit = other.mConnectionLimit;  		mPerHostConnectionLimit = other.mPerHostConnectionLimit;  		mPipelining = other.mPipelining; +		mThrottleRate = other.mThrottleRate;  	}  	return *this;  }  HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) -	: mSetMask(other.mSetMask), -	  mConnectionLimit(other.mConnectionLimit), +	: mConnectionLimit(other.mConnectionLimit),  	  mPerHostConnectionLimit(other.mPerHostConnectionLimit), -	  mPipelining(other.mPipelining) +	  mPipelining(other.mPipelining), +	  mThrottleRate(other.mThrottleRate)  {} -HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value)  {  	switch (opt)  	{ -	case HttpRequest::CP_CONNECTION_LIMIT: +	case HttpRequest::PO_CONNECTION_LIMIT:  		mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX));  		break; -	case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: +	case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT:  		mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit);  		break; -	case HttpRequest::CP_ENABLE_PIPELINING: +	case HttpRequest::PO_ENABLE_PIPELINING:  		mPipelining = llclamp(value, 0L, 1L);  		break; +	case HttpRequest::PO_THROTTLE_RATE: +		mThrottleRate = llclamp(value, 0L, 1000000L); +		break; +  	default:  		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);  	} -	mSetMask |= 1UL << int(opt);  	return HttpStatus();  } -HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const  { -	static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); -	long * src(NULL); -	  	switch (opt)  	{ -	case HttpRequest::CP_CONNECTION_LIMIT: -		src = &mConnectionLimit; +	case HttpRequest::PO_CONNECTION_LIMIT: +		*value = mConnectionLimit;  		break; -	case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: -		src = &mPerHostConnectionLimit; +	case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: +		*value = mPerHostConnectionLimit;  		break; -	case HttpRequest::CP_ENABLE_PIPELINING: -		src = &mPipelining; +	case HttpRequest::PO_ENABLE_PIPELINING: +		*value = mPipelining; +		break; + +	case HttpRequest::PO_THROTTLE_RATE: +		*value = mThrottleRate;  		break;  	default:  		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);  	} -	if (! (mSetMask & (1UL << int(opt)))) -		return not_set; - -	*value = *src;  	return HttpStatus();  } diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h index d175413cbd..38f1194ded 100755 --- a/indra/llcorehttp/_httppolicyclass.h +++ b/indra/llcorehttp/_httppolicyclass.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@  namespace LLCore  { +/// Options struct for per-class policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces.  For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed.  When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading:  Single-threaded.  In practice, init thread before +/// worker starts, worker thread after.  class HttpPolicyClass  {  public: @@ -44,14 +56,14 @@ public:  	HttpPolicyClass(const HttpPolicyClass &);			// Not defined  public: -	HttpStatus set(HttpRequest::EClassPolicy opt, long value); -	HttpStatus get(HttpRequest::EClassPolicy opt, long * value); +	HttpStatus set(HttpRequest::EPolicyOption opt, long value); +	HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const;  public: -	unsigned long				mSetMask;  	long						mConnectionLimit;  	long						mPerHostConnectionLimit;  	long						mPipelining; +	long						mThrottleRate;  };  // end class HttpPolicyClass  }  // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 72f409d3b1..1dc95f3dce 100755 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -34,8 +34,7 @@ namespace LLCore  HttpPolicyGlobal::HttpPolicyGlobal() -	: mSetMask(0UL), -	  mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), +	: mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT),  	  mTrace(HTTP_TRACE_OFF),  	  mUseLLProxy(0)  {} @@ -49,7 +48,6 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)  {  	if (this != &other)  	{ -		mSetMask = other.mSetMask;  		mConnectionLimit = other.mConnectionLimit;  		mCAPath = other.mCAPath;  		mCAFile = other.mCAFile; @@ -61,19 +59,19 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other)  } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value)  {  	switch (opt)  	{ -	case HttpRequest::GP_CONNECTION_LIMIT: +	case HttpRequest::PO_CONNECTION_LIMIT:  		mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX));  		break; -	case HttpRequest::GP_TRACE: +	case HttpRequest::PO_TRACE:  		mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX));  		break; -	case HttpRequest::GP_LLPROXY: +	case HttpRequest::PO_LLPROXY:  		mUseLLProxy = llclamp(value, 0L, 1L);  		break; @@ -81,24 +79,23 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value)  		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);  	} -	mSetMask |= 1UL << int(opt);  	return HttpStatus();  } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value)  {  	switch (opt)  	{ -	case HttpRequest::GP_CA_PATH: +	case HttpRequest::PO_CA_PATH:  		mCAPath = value;  		break; -	case HttpRequest::GP_CA_FILE: +	case HttpRequest::PO_CA_FILE:  		mCAFile = value;  		break; -	case HttpRequest::GP_HTTP_PROXY: +	case HttpRequest::PO_HTTP_PROXY:  		mCAFile = value;  		break; @@ -106,69 +103,54 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri  		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);  	} -	mSetMask |= 1UL << int(opt);  	return HttpStatus();  } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const  { -	static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); -	long * src(NULL); -	  	switch (opt)  	{ -	case HttpRequest::GP_CONNECTION_LIMIT: -		src = &mConnectionLimit; +	case HttpRequest::PO_CONNECTION_LIMIT: +		*value = mConnectionLimit;  		break; -	case HttpRequest::GP_TRACE: -		src = &mTrace; +	case HttpRequest::PO_TRACE: +		*value = mTrace;  		break; -	case HttpRequest::GP_LLPROXY: -		src = &mUseLLProxy; +	case HttpRequest::PO_LLPROXY: +		*value = mUseLLProxy;  		break;  	default:  		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);  	} -	if (! (mSetMask & (1UL << int(opt)))) -		return not_set; - -	*value = *src;  	return HttpStatus();  } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const  { -	static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); -	const std::string * src(NULL); -  	switch (opt)  	{ -	case HttpRequest::GP_CA_PATH: -		src = &mCAPath; +	case HttpRequest::PO_CA_PATH: +		*value = mCAPath;  		break; -	case HttpRequest::GP_CA_FILE: -		src = &mCAFile; +	case HttpRequest::PO_CA_FILE: +		*value = mCAFile;  		break; -	case HttpRequest::GP_HTTP_PROXY: -		src = &mHttpProxy; +	case HttpRequest::PO_HTTP_PROXY: +		*value = mHttpProxy;  		break;  	default:  		return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG);  	} -	if (! (mSetMask & (1UL << int(opt)))) -		return not_set; - -	*value = src;  	return HttpStatus();  } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index a50d0e4188..67c4ba9481 100755 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@  namespace LLCore  { +/// Options struct for global policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces.  For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed.  When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading:  Single-threaded.  In practice, init thread before +/// worker starts, worker thread after.  class HttpPolicyGlobal  {  public: @@ -46,13 +58,12 @@ private:  	HttpPolicyGlobal(const HttpPolicyGlobal &);			// Not defined  public: -	HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); -	HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); -	HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); -	HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); +	HttpStatus set(HttpRequest::EPolicyOption opt, long value); +	HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); +	HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; +	HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const;  public: -	unsigned long		mSetMask;  	long				mConnectionLimit;  	std::string			mCAPath;  	std::string			mCAFile; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0825888d0f..c94249dc2d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -43,6 +43,18 @@  namespace LLCore  { +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = +{ //    isLong     isDynamic  isGlobal    isClass +	{	true,		true,		true,		true	},		// PO_CONNECTION_LIMIT +	{	true,		true,		false,		true	},		// PO_PER_HOST_CONNECTION_LIMIT +	{	false,		false,		true,		false	},		// PO_CA_PATH +	{	false,		false,		true,		false	},		// PO_CA_FILE +	{	false,		true,		true,		false	},		// PO_HTTP_PROXY +	{	true,		true,		true,		false	},		// PO_LLPROXY +	{	true,		true,		true,		false	},		// PO_TRACE +	{	true,		true,		false,		true	},		// PO_ENABLE_PIPELINING +	{	true,		true,		false,		true	}		// PO_THROTTLE_RATE +};  HttpService * HttpService::sInstance(NULL);  volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); @@ -51,15 +63,9 @@ HttpService::HttpService()  	  mExitRequested(0U),  	  mThread(NULL),  	  mPolicy(NULL), -	  mTransport(NULL) -{ -	// Create the default policy class -	HttpPolicyClass pol_class; -	pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); -	pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); -	pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); -	mPolicyClasses.push_back(pol_class); -} +	  mTransport(NULL), +	  mLastPolicy(0) +{}  HttpService::~HttpService() @@ -149,13 +155,8 @@ void HttpService::term()  HttpRequest::policy_t HttpService::createPolicyClass()  { -	const HttpRequest::policy_t policy_class(mPolicyClasses.size()); -	if (policy_class >= HTTP_POLICY_CLASS_LIMIT) -	{ -		return 0; -	} -	mPolicyClasses.push_back(HttpPolicyClass()); -	return policy_class; +	mLastPolicy = mPolicy->createPolicyClass(); +	return mLastPolicy;  } @@ -188,8 +189,8 @@ void HttpService::startThread()  	}  	// Push current policy definitions, enable policy & transport components -	mPolicy->start(mPolicyGlobal, mPolicyClasses); -	mTransport->start(mPolicyClasses.size()); +	mPolicy->start(); +	mTransport->start(mLastPolicy + 1);  	mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1));  	sState = RUNNING; @@ -322,7 +323,7 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)  		{  			// Setup for subsequent tracing  			long tracing(HTTP_TRACE_OFF); -			mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); +			mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing);  			op->mTracing = (std::max)(op->mTracing, int(tracing));  			if (op->mTracing > HTTP_TRACE_OFF) @@ -345,4 +346,137 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop)  } +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, +										long * ret_value) +{ +	if (opt < HttpRequest::PO_CONNECTION_LIMIT											// option must be in range +		|| opt >= HttpRequest::PO_LAST													// ditto +		|| (! sOptionDesc[opt].mIsLong)													// datatype is long +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy)			// pclass in valid range +		|| (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal)	// global setting permitted +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass))	// class setting permitted +																						// can always get, no dynamic check +	{ +		return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); +	} + +	HttpStatus status; +	if (pclass == HttpRequest::GLOBAL_POLICY_ID) +	{ +		HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); +		 +		status = opts.get(opt, ret_value); +	} +	else +	{ +		HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + +		status = opts.get(opt, ret_value); +	} + +	return status; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, +										std::string * ret_value) +{ +	HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + +	if (opt < HttpRequest::PO_CONNECTION_LIMIT											// option must be in range +		|| opt >= HttpRequest::PO_LAST													// ditto +		|| (sOptionDesc[opt].mIsLong)													// datatype is string +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy)			// pclass in valid range +		|| (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal)	// global setting permitted +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass))	// class setting permitted +																						// can always get, no dynamic check +	{ +		return status; +	} + +	// Only global has string values +	if (pclass == HttpRequest::GLOBAL_POLICY_ID) +	{ +		HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + +		status = opts.get(opt, ret_value); +	} + +	return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, +										long value, long * ret_value) +{ +	HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); +	 +	if (opt < HttpRequest::PO_CONNECTION_LIMIT											// option must be in range +		|| opt >= HttpRequest::PO_LAST													// ditto +		|| (! sOptionDesc[opt].mIsLong)													// datatype is long +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy)			// pclass in valid range +		|| (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal)	// global setting permitted +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)		// class setting permitted +		|| (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic))						// dynamic setting permitted +	{ +		return status; +	} + +	if (pclass == HttpRequest::GLOBAL_POLICY_ID) +	{ +		HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); +		 +		status = opts.set(opt, value); +		if (status && ret_value) +		{ +			status = opts.get(opt, ret_value); +		} +	} +	else +	{ +		HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + +		status = opts.set(opt, value); +		if (status && ret_value) +		{ +			status = opts.get(opt, ret_value); +		} +	} + +	return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, +										const std::string & value, std::string * ret_value) +{ +	HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); +	 +	if (opt < HttpRequest::PO_CONNECTION_LIMIT											// option must be in range +		|| opt >= HttpRequest::PO_LAST													// ditto +		|| (sOptionDesc[opt].mIsLong)													// datatype is string +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy)			// pclass in valid range +		|| (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal)	// global setting permitted +		|| (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)		// class setting permitted +		|| (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic))						// dynamic setting permitted +	{ +		return status; +	} + +	// Only string values are global at this time +	if (pclass == HttpRequest::GLOBAL_POLICY_ID) +	{ +		HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); +		 +		status = opts.set(opt, value); +		if (status && ret_value) +		{ +			status = opts.get(opt, ret_value); +		} +	} + +	return status; +} +	 +  }  // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ffe0349d4d..cf23f3ab61 100755 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -53,6 +53,7 @@ namespace LLCore  class HttpRequestQueue;  class HttpPolicy;  class HttpLibcurl; +class HttpOpSetGet;  /// The HttpService class does the work behind the request queue.  It @@ -106,7 +107,7 @@ public:  		NORMAL,					///< continuous polling of request, ready, active queues  		REQUEST_SLEEP			///< can sleep indefinitely waiting for request queue write  	}; -		 +  	static void init(HttpRequestQueue *);  	static void term(); @@ -136,7 +137,7 @@ public:  	/// acquires its weaknesses.  	static bool isStopped(); -	/// Threading:  callable by consumer thread *once*. +	/// Threading:  callable by init thread *once*.  	void startThread();  	/// Threading:  callable by worker thread. @@ -181,27 +182,38 @@ public:  		}  	/// Threading:  callable by consumer thread. -	HttpPolicyGlobal & getGlobalOptions() -		{ -			return mPolicyGlobal; -		} - -	/// Threading:  callable by consumer thread.  	HttpRequest::policy_t createPolicyClass(); -	/// Threading:  callable by consumer thread. -	HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) -		{ -			llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); -			return mPolicyClasses[policy_class]; -		} -	  protected:  	void threadRun(LLCoreInt::HttpThread * thread);  	ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: +	friend class HttpOpSetGet; +	friend class HttpRequest; +	 +	// Used internally to describe what operations are allowed +	// on each policy option. +	struct OptionDescriptor +	{ +		bool		mIsLong; +		bool		mIsDynamic; +		bool		mIsGlobal; +		bool		mIsClass; +	}; +		 +	HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, +							   long * ret_value); +	HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, +							   std::string * ret_value); +	HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, +							   long value, long * ret_value); +	HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, +							   const std::string & value, std::string * ret_value);  protected: +	static const OptionDescriptor		sOptionDesc[HttpRequest::PO_LAST];  	static HttpService *				sInstance;  	// === shared data === @@ -210,13 +222,13 @@ protected:  	LLAtomicU32							mExitRequested;  	LLCoreInt::HttpThread *				mThread; -	// === consumer-thread-only data === -	HttpPolicyGlobal					mPolicyGlobal; -	std::vector<HttpPolicyClass>		mPolicyClasses; -	  	// === working-thread-only data ===  	HttpPolicy *						mPolicy;		// Simple pointer, has ownership  	HttpLibcurl *						mTransport;		// Simple pointer, has ownership +	 +	// === main-thread-only data === +	HttpRequest::policy_t				mLastPolicy; +	  };  // end class HttpService  }  // end namespace LLCore diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 909dc5b0cb..73c49687d7 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -39,6 +39,7 @@  #include "httprequest.h"  #include "httphandler.h"  #include "httpresponse.h" +#include "httpoptions.h"  #include "httpheaders.h"  #include "bufferarray.h"  #include "_mutex.h" @@ -57,6 +58,7 @@ void usage(std::ostream & out);  // Default command line settings  static int concurrency_limit(40); +static int highwater(100);  static char url_format[1024] = "http://example.com/some/path?texture_id=%s.texture";  #if defined(WIN32) @@ -79,11 +81,11 @@ public:  	WorkingSet();  	~WorkingSet(); -	bool reload(LLCore::HttpRequest *); +	bool reload(LLCore::HttpRequest *, LLCore::HttpOptions *);  	virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); -	void loadTextureUuids(FILE * in); +	void loadAssetUuids(FILE * in);  public:  	struct Spec @@ -93,24 +95,27 @@ public:  		int				mLength;  	};  	typedef std::set<LLCore::HttpHandle> handle_set_t; -	typedef std::vector<Spec> texture_list_t; +	typedef std::vector<Spec> asset_list_t;  public:  	bool						mVerbose;  	bool						mRandomRange; -	int							mMaxConcurrency; +	int							mRequestLowWater; +	int							mRequestHighWater;  	handle_set_t				mHandles;  	int							mRemaining;  	int							mLimit;  	int							mAt;  	std::string					mUrl; -	texture_list_t				mTextures; +	asset_list_t				mAssets;  	int							mErrorsApi;  	int							mErrorsHttp;  	int							mErrorsHttp404;  	int							mErrorsHttp416;  	int							mErrorsHttp500;  	int							mErrorsHttp503; +	int							mRetries; +	int							mRetriesHttp503;  	int							mSuccesses;  	long						mByteCount;  	LLCore::HttpHeaders *		mHeaders; @@ -158,7 +163,7 @@ int main(int argc, char** argv)  	bool do_verbose(false);  	int option(-1); -	while (-1 != (option = getopt(argc, argv, "u:c:h?Rv"))) +	while (-1 != (option = getopt(argc, argv, "u:c:h?RvH:")))  	{  		switch (option)  		{ @@ -182,6 +187,21 @@ int main(int argc, char** argv)  			}  			break; +		case 'H': +		    { +				unsigned long value; +				char * end; + +				value = strtoul(optarg, &end, 10); +				if (value < 1 || value > 100 || *end != '\0') +				{ +					usage(std::cerr); +					return 1; +				} +				highwater = value; +			} +			break; +  		case 'R':  			do_random = true;  			break; @@ -216,25 +236,32 @@ int main(int argc, char** argv)  	// Initialization  	init_curl();  	LLCore::HttpRequest::createService(); -	LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, -											  LLCore::HttpRequest::CP_CONNECTION_LIMIT, -											  concurrency_limit); +	LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, +											   LLCore::HttpRequest::DEFAULT_POLICY_ID, +											   concurrency_limit, +											   NULL);  	LLCore::HttpRequest::startThread();  	// Get service point  	LLCore::HttpRequest * hr = new LLCore::HttpRequest(); +	// Get request options +	LLCore::HttpOptions * opt = new LLCore::HttpOptions(); +	opt->setRetries(12); +	opt->setUseRetryAfter(true); +	  	// Get a handler/working set  	WorkingSet ws;  	// Fill the working set with work  	ws.mUrl = url_format; -	ws.loadTextureUuids(uuids); +	ws.loadAssetUuids(uuids);  	ws.mRandomRange = do_random;  	ws.mVerbose = do_verbose; -	ws.mMaxConcurrency = 100; +	ws.mRequestHighWater = highwater; +	ws.mRequestLowWater = ws.mRequestHighWater / 2; -	if (! ws.mTextures.size()) +	if (! ws.mAssets.size())  	{  		std::cerr << "No UUIDs found in file '" << argv[optind] << "'." << std::endl;  		return 1; @@ -246,9 +273,9 @@ int main(int argc, char** argv)  	// Run it  	int passes(0); -	while (! ws.reload(hr)) +	while (! ws.reload(hr, opt))  	{ -		hr->update(5000000); +		hr->update(0);  		ms_sleep(2);  		if (0 == (++passes % 200))  		{ @@ -265,6 +292,8 @@ int main(int argc, char** argv)  	std::cout << "HTTP 404 errors: " << ws.mErrorsHttp404 << "  HTTP 416 errors: " << ws.mErrorsHttp416  			  << "  HTTP 500 errors:  " << ws.mErrorsHttp500 << "  HTTP 503 errors: " << ws.mErrorsHttp503   			  << std::endl; +	std::cout << "Retries: " << ws.mRetries << "  Retries on 503: " << ws.mRetriesHttp503 +			  << std::endl;  	std::cout << "User CPU: " << (metrics.mEndUTime - metrics.mStartUTime)  			  << " uS  System CPU: " << (metrics.mEndSTime - metrics.mStartSTime)  			  << " uS  Wall Time: "  << (metrics.mEndWallTime - metrics.mStartWallTime) @@ -275,6 +304,8 @@ int main(int argc, char** argv)  	// Clean up  	hr->requestStopThread(NULL);  	ms_sleep(1000); +	opt->release(); +	opt = NULL;  	delete hr;  	LLCore::HttpRequest::destroyService();  	term_curl(); @@ -300,8 +331,10 @@ void usage(std::ostream & out)  		" -u <url_format>       printf-style format string for URL generation\n"  		"                       Default:  " << url_format << "\n"  		" -R                    Issue GETs with random Range: headers\n" -		" -c <limit>            Maximum request concurrency.  Range:  [1..100]\n" +		" -c <limit>            Maximum connection concurrency.  Range:  [1..100]\n"  		"                       Default:  " << concurrency_limit << "\n" +		" -H <limit>            HTTP request highwater (requests fed to llcorehttp).\n" +		"                       Range:  [1..100]  Default:  " << highwater << "\n"  		" -v                    Verbose mode.  Issue some chatter while running\n"  		" -h                    print this help\n"  		"\n" @@ -322,10 +355,12 @@ WorkingSet::WorkingSet()  	  mErrorsHttp416(0),  	  mErrorsHttp500(0),  	  mErrorsHttp503(0), +	  mRetries(0), +	  mRetriesHttp503(0),  	  mSuccesses(0),  	  mByteCount(0L)  { -	mTextures.reserve(30000); +	mAssets.reserve(30000);  	mHeaders = new LLCore::HttpHeaders;  	mHeaders->append("Accept", "image/x-j2c"); @@ -342,29 +377,35 @@ WorkingSet::~WorkingSet()  } -bool WorkingSet::reload(LLCore::HttpRequest * hr) +bool WorkingSet::reload(LLCore::HttpRequest * hr, LLCore::HttpOptions * opt)  { -	int to_do((std::min)(mRemaining, mMaxConcurrency - int(mHandles.size()))); +	if (mRequestLowWater <= mHandles.size()) +	{ +		// Haven't fallen below low-water level yet. +		return false; +	} +	 +	int to_do((std::min)(mRemaining, mRequestHighWater - int(mHandles.size())));  	for (int i(0); i < to_do; ++i)  	{  		char buffer[1024];  #if	defined(WIN32) -		_snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mTextures[mAt].mUuid.c_str()); +		_snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, mUrl.c_str(), mAssets[mAt].mUuid.c_str());  #else -		snprintf(buffer, sizeof(buffer), mUrl.c_str(), mTextures[mAt].mUuid.c_str()); +		snprintf(buffer, sizeof(buffer), mUrl.c_str(), mAssets[mAt].mUuid.c_str());  #endif -		int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mOffset); -		int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mTextures[mAt].mLength); +		int offset(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mOffset); +		int length(mRandomRange ? ((unsigned long) rand()) % 1000000UL : mAssets[mAt].mLength);  		LLCore::HttpHandle handle;  		if (offset || length)  		{ -			handle = hr->requestGetByteRange(0, 0, buffer, offset, length, NULL, mHeaders, this); +			handle = hr->requestGetByteRange(0, 0, buffer, offset, length, opt, mHeaders, this);  		}  		else  		{ -			handle = hr->requestGet(0, 0, buffer, NULL, mHeaders, this); +			handle = hr->requestGet(0, 0, buffer, opt, mHeaders, this);  		}  		if (! handle)  		{ @@ -410,7 +451,7 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r  		{  			// More success  			LLCore::BufferArray * data(response->getBody()); -			mByteCount += data->size(); +			mByteCount += data ? data->size() : 0;  			++mSuccesses;  		}  		else @@ -446,6 +487,10 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r  				++mErrorsApi;  			}  		} +		unsigned int retry(0U), retry_503(0U); +		response->getRetries(&retry, &retry_503); +		mRetries += int(retry); +		mRetriesHttp503 += int(retry_503);  		mHandles.erase(it);  	} @@ -459,21 +504,21 @@ void WorkingSet::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * r  } -void WorkingSet::loadTextureUuids(FILE * in) +void WorkingSet::loadAssetUuids(FILE * in)  {  	char buffer[1024];  	while (fgets(buffer, sizeof(buffer), in))  	{ -		WorkingSet::Spec texture; +		WorkingSet::Spec asset;  		char * state(NULL);  		char * token = strtok_r(buffer, " \t\n,", &state);  		if (token && 36 == strlen(token))  		{  			// Close enough for this function -			texture.mUuid = token; -			texture.mOffset = 0; -			texture.mLength = 0; +			asset.mUuid = token; +			asset.mOffset = 0; +			asset.mLength = 0;  			token = strtok_r(buffer, " \t\n,", &state);  			if (token)  			{ @@ -482,14 +527,14 @@ void WorkingSet::loadTextureUuids(FILE * in)  				if (token)  				{  					int length(atoi(token)); -					texture.mOffset = offset; -					texture.mLength = length; +					asset.mOffset = offset; +					asset.mLength = length;  				}  			} -			mTextures.push_back(texture); +			mAssets.push_back(asset);  		}  	} -	mRemaining = mLimit = mTextures.size(); +	mRemaining = mLimit = mAssets.size();  } diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 0738760763..c2f15155ac 100755 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -70,7 +70,8 @@ std::string HttpStatus::toString() const  			"Invalid datatype for argument or option",  			"Option has not been explicitly set",  			"Option is not dynamic and must be set early", -			"Invalid HTTP status code received from server" +			"Invalid HTTP status code received from server", +			"Could not allocate required resource"  		};  	static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); @@ -177,6 +178,44 @@ std::string HttpStatus::toString() const  } +std::string HttpStatus::toTerseString() const +{ +	std::ostringstream result; + +	unsigned int error_value((unsigned short) mStatus); +	 +	switch (mType) +	{ +	case EXT_CURL_EASY: +		result << "Easy_"; +		break; +		 +	case EXT_CURL_MULTI: +		result << "Multi_"; +		break; +		 +	case LLCORE: +		result << "Core_"; +		break; + +	default: +		if (isHttpStatus()) +		{ +			result << "Http_"; +			error_value = mType; +		} +		else +		{ +			result << "Unknown_"; +		} +		break; +	} +	 +	result << error_value; +	return result.str(); +} + +  // Pass true on statuses that might actually be cleared by a  // retry.  Library failures, calling problems, etc. aren't  // going to be fixed by squirting bits all over the Net. @@ -206,6 +245,5 @@ bool HttpStatus::isRetryable() const  			*this == inv_cont_range);	// Short data read disagrees with content-range  } -		  } // end namespace LLCore diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 41fb5164cf..9601f94125 100755 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -29,9 +29,9 @@  /// @package LLCore::HTTP  /// -/// This library implements a high-level, Indra-code-free client interface to -/// HTTP services based on actual patterns found in the viewer and simulator. -/// Interfaces are similar to those supplied by the legacy classes +/// This library implements a high-level, Indra-code-free (somewhat) client +/// interface to HTTP services based on actual patterns found in the viewer +/// and simulator.  Interfaces are similar to those supplied by the legacy classes  /// LLCurlRequest and LLHTTPClient.  To that is added a policy scheme that  /// allows an application to specify connection behaviors:  limits on  /// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. @@ -52,7 +52,7 @@  /// - "llcorehttp/httprequest.h"  /// - "llcorehttp/httpresponse.h"  /// -/// The library is still under early development and particular users +/// The library is still under development and particular users  /// may need access to internal implementation details that are found  /// in the _*.h header files.  But this is a crutch to be avoided if at  /// all possible and probably indicates some interface work is neeeded. @@ -66,6 +66,8 @@  ///   .  CRYPTO_set_id_callback(...)  /// - HttpRequest::createService() called to instantiate singletons  ///   and support objects. +/// - HttpRequest::startThread() to kick off the worker thread and +///   begin servicing requests.  ///  /// An HTTP consumer in an application, and an application may have many  /// consumers, does a few things: @@ -91,10 +93,12 @@  ///   objects.  /// - Do completion processing in your onCompletion() method.  /// -/// Code fragments: -/// Rather than a poorly-maintained example in comments, look in the -/// example subdirectory which is a minimal yet functional tool to do -/// GET request performance testing.  With four calls: +/// Code fragments. +/// +/// Initialization.  Rather than a poorly-maintained example in +/// comments, look in the example subdirectory which is a minimal +/// yet functional tool to do GET request performance testing. +/// With four calls:  ///  ///   	init_curl();  ///     LLCore::HttpRequest::createService(); @@ -103,7 +107,85 @@  ///  /// the program is basically ready to issue requests.  /// - +/// HttpHandler.  Having started life as a non-indra library, +/// this code broke away from the classic Responder model and +/// introduced a handler class to represent an interface for +/// request responses.  This is a non-reference-counted entity +/// which can be used as a base class or a mixin.  An instance +/// of a handler can be used for each request or can be shared +/// among any number of requests.  Your choice but expect to +/// code something like the following: +/// +///     class AppHandler : public LLCore::HttpHandler +///     { +///     public: +///         virtual void onCompleted(HttpHandle handle, +///                                  HttpResponse * response) +///         { +///             ... +///         } +///         ... +///     }; +///     ... +///     handler = new handler(...); +/// +/// +/// Issuing requests.  Using 'hr' above, +/// +///     hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, +///                    0,				// Priority, not used yet +///                    url, +///                    NULL,			// options +///                    NULL,            // additional headers +///                    handler); +/// +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, +/// the request was successfully issued and there will eventally +/// be a status delivered to the handler.  If invalid is returnedd, +/// the actual status can be retrieved by calling hr->getStatus(). +/// +/// Completing requests and delivering notifications.  Operations +/// are all performed by the worker thread and will be driven to +/// completion regardless of caller actions.  Notification of +/// completion (success or failure) is done by calls to +/// HttpRequest::update() which will invoke handlers for completed +/// requests: +/// +///     hr->update(0); +///       // Callbacks into handler->onCompleted() +/// +/// +/// Threads. +/// +/// Threads are supported and used by this library.  The various +/// classes, methods and members are documented with thread +/// constraints which programmers must follow and which are +/// defined as follows: +/// +/// consumer	Any thread that has instanced HttpRequest and is +///             issuing requests.  A particular instance can only +///             be used by one consumer thread but a consumer may +///             have many instances available to it. +/// init		Special consumer thread, usually the main thread, +///             involved in setting up the library at startup. +/// worker      Thread used internally by the library to perform +///             HTTP operations.  Consumers will not have to deal +///             with this thread directly but some APIs are reserved +///             to it. +/// any         Consumer or worker thread. +/// +/// For the most part, API users will not have to do much in the +/// way of ensuring thread safely.  However, there is a tremendous +/// amount of sharing between threads of read-only data.  So when +/// documentation declares that an option or header instance +/// becomes shared between consumer and worker, the consumer must +/// not modify the shared object. +/// +/// Internally, there is almost no thread synchronization.  During +/// normal operations (non-init, non-term), only the request queue +/// and the multiple reply queues are shared between threads and +/// only here are mutexes used. +///  #include "linden_common.h"		// Modifies curl/curl.h interfaces @@ -164,7 +246,10 @@ enum HttpError  	HE_OPT_NOT_DYNAMIC = 8,  	// Invalid HTTP status code returned by server -	HE_INVALID_HTTP_STATUS = 9 +	HE_INVALID_HTTP_STATUS = 9, +	 +	// Couldn't allocate resource, typically libcurl handle +	HE_BAD_ALLOC = 10  }; // end enum HttpError @@ -239,9 +324,10 @@ struct HttpStatus  			return *this;  		} -	static const type_enum_t EXT_CURL_EASY = 0; -	static const type_enum_t EXT_CURL_MULTI = 1; -	static const type_enum_t LLCORE = 2; +	static const type_enum_t EXT_CURL_EASY = 0;			///< mStatus is an error from a curl_easy_*() call +	static const type_enum_t EXT_CURL_MULTI = 1;		///< mStatus is an error from a curl_multi_*() call +	static const type_enum_t LLCORE = 2;				///< mStatus is an HE_* error code +														///< 100-999 directly represent HTTP status codes  	type_enum_t			mType;  	short				mStatus; @@ -297,6 +383,14 @@ struct HttpStatus  	/// LLCore itself).  	std::string toString() const; +	/// Convert status to a compact string representation +	/// of the form:  "<type>_<value>".  The <type> will be +	/// one of:  Core, Http, Easy, Multi, Unknown.  And +	/// <value> will be an unsigned integer.  More easily +	/// interpreted than the hex representation, it's still +	/// compact and easily searched. +	std::string toTerseString() const; +  	/// Returns true if the status value represents an  	/// HTTP response status (100 - 999).  	bool isHttpStatus() const diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 1699d19f8d..5bf1ecb4a5 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -38,7 +38,9 @@ HttpOptions::HttpOptions()  	  mWantHeaders(false),  	  mTracing(HTTP_TRACE_OFF),  	  mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), -	  mRetries(HTTP_RETRY_COUNT_DEFAULT) +	  mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), +	  mRetries(HTTP_RETRY_COUNT_DEFAULT), +	  mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT)  {} @@ -64,10 +66,21 @@ void HttpOptions::setTimeout(unsigned int timeout)  } +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ +	mTransferTimeout = timeout; +} + +  void HttpOptions::setRetries(unsigned int retries)  {  	mRetries = retries;  } +void HttpOptions::setUseRetryAfter(bool use_retry) +{ +	mUseRetryAfter = use_retry; +} +  }   // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 97e46a8cd3..4ab5ff18c4 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -68,36 +68,55 @@ protected:  	void operator=(const HttpOptions &);		// Not defined  public: +	// Default:   false  	void				setWantHeaders(bool wanted);  	bool				getWantHeaders() const  		{  			return mWantHeaders;  		} -	 + +	// Default:  0  	void				setTrace(int long);  	int					getTrace() const  		{  			return mTracing;  		} +	// Default:  30  	void				setTimeout(unsigned int timeout);  	unsigned int		getTimeout() const  		{  			return mTimeout;  		} +	// Default:  0 +	void				setTransferTimeout(unsigned int timeout); +	unsigned int		getTransferTimeout() const +		{ +			return mTransferTimeout; +		} + +	// Default:  8  	void				setRetries(unsigned int retries);  	unsigned int		getRetries() const  		{  			return mRetries;  		} + +	// Default:  true +	void				setUseRetryAfter(bool use_retry); +	bool				getUseRetryAfter() const +		{ +			return mUseRetryAfter; +		}  protected:  	bool				mWantHeaders;  	int					mTracing;  	unsigned int		mTimeout; +	unsigned int		mTransferTimeout;  	unsigned int		mRetries; -	 +	bool				mUseRetryAfter;  }; // end class HttpOptions diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 9b739a8825..7b1888e3eb 100755 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -54,12 +54,8 @@ namespace LLCore  // ==================================== -HttpRequest::policy_t HttpRequest::sNextPolicyID(1); - -  HttpRequest::HttpRequest() -	: //HttpHandler(), -	  mReplyQueue(NULL), +	: mReplyQueue(NULL),  	  mRequestQueue(NULL)  {  	mRequestQueue = HttpRequestQueue::instanceOf(); @@ -90,45 +86,91 @@ HttpRequest::~HttpRequest()  // ==================================== -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) +HttpRequest::policy_t HttpRequest::createPolicyClass()  {  	if (HttpService::RUNNING == HttpService::instanceOf()->getState())  	{ -		return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); +		return 0;  	} -	return HttpService::instanceOf()->getGlobalOptions().set(opt, value); +	return HttpService::instanceOf()->createPolicyClass();  } -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, +											  long value, long * ret_value)  {  	if (HttpService::RUNNING == HttpService::instanceOf()->getState())  	{  		return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);  	} -	return HttpService::instanceOf()->getGlobalOptions().set(opt, value); +	return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value);  } -HttpRequest::policy_t HttpRequest::createPolicyClass() +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, +											  const std::string & value, std::string * ret_value)  {  	if (HttpService::RUNNING == HttpService::instanceOf()->getState())  	{ -		return 0; +		return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC);  	} -	return HttpService::instanceOf()->createPolicyClass(); +	return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value);  } -HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, -											 EClassPolicy opt, -											 long value) +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, +										long value, HttpHandler * handler)  { -	if (HttpService::RUNNING == HttpService::instanceOf()->getState()) +	HttpStatus status; +	HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + +	HttpOpSetGet * op = new HttpOpSetGet(); +	if (! (status = op->setupSet(opt, pclass, value)))  	{ -		return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); +		op->release(); +		mLastReqStatus = status; +		return handle; +	} +	op->setReplyPath(mReplyQueue, handler); +	if (! (status = mRequestQueue->addOp(op)))			// transfers refcount +	{ +		op->release(); +		mLastReqStatus = status; +		return handle; +	} +	 +	mLastReqStatus = status; +	handle = static_cast<HttpHandle>(op); +	 +	return handle; +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, +										const std::string & value, HttpHandler * handler) +{ +	HttpStatus status; +	HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + +	HttpOpSetGet * op = new HttpOpSetGet(); +	if (! (status = op->setupSet(opt, pclass, value))) +	{ +		op->release(); +		mLastReqStatus = status; +		return handle; +	} +	op->setReplyPath(mReplyQueue, handler); +	if (! (status = mRequestQueue->addOp(op)))			// transfers refcount +	{ +		op->release(); +		mLastReqStatus = status; +		return handle;  	} -	return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); +	 +	mLastReqStatus = status; +	handle = static_cast<HttpHandle>(op); +	 +	return handle;  } @@ -474,31 +516,6 @@ HttpHandle HttpRequest::requestSpin(int mode)  	return handle;  } -// ==================================== -// Dynamic Policy Methods -// ==================================== - -HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) -{ -	HttpStatus status; -	HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - -	HttpOpSetGet * op = new HttpOpSetGet(); -	op->setupSet(GP_HTTP_PROXY, proxy); -	op->setReplyPath(mReplyQueue, handler); -	if (! (status = mRequestQueue->addOp(op)))			// transfers refcount -	{ -		op->release(); -		mLastReqStatus = status; -		return handle; -	} - -	mLastReqStatus = status; -	handle = static_cast<HttpHandle>(op); - -	return handle; -} -  }   // end namespace LLCore diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index ab2f302d34..651654844a 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -56,6 +56,9 @@ class BufferArray;  /// The class supports the current HTTP request operations:  ///  /// - requestGetByteRange:  GET with Range header for a single range of bytes +/// - requestGet: +/// - requestPost: +/// - requestPut:  ///  /// Policy Classes  /// @@ -100,9 +103,26 @@ public:  	/// Represents a default, catch-all policy class that guarantees  	/// eventual service for any HTTP request. -	static const int DEFAULT_POLICY_ID = 0; +	static const policy_t DEFAULT_POLICY_ID = 0; +	static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; +	static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; -	enum EGlobalPolicy +	/// Create a new policy class into which requests can be made. +	/// +	/// All class creation must occur before threads are started and +	/// transport begins.  Policy classes are limited to a small value. +	/// Currently that limit is the default class + 1. +	/// +	/// @return			If positive, the policy_id used to reference +	///					the class in other methods.  If 0, requests +	///					for classes have exceeded internal limits +	///					or caller has tried to create a class after +	///					threads have been started.  Caller must fallback +	///					and recover. +	/// +	static policy_t createPolicyClass(); + +	enum EPolicyOption  	{  		/// Maximum number of connections the library will use to  		/// perform operations.  This is somewhat soft as the underlying @@ -113,24 +133,40 @@ public:  		/// a somewhat soft value.  There may be an additional five  		/// connections per policy class depending upon runtime  		/// behavior. -		GP_CONNECTION_LIMIT, +		/// +		/// Both global and per-class +		PO_CONNECTION_LIMIT, + +		/// Limits the number of connections used for a single +		/// literal address/port pair within the class. +		/// +		/// Per-class only +		PO_PER_HOST_CONNECTION_LIMIT,  		/// String containing a system-appropriate directory name  		/// where SSL certs are stored. -		GP_CA_PATH, +		/// +		/// Global only +		PO_CA_PATH,  		/// String giving a full path to a file containing SSL certs. -		GP_CA_FILE, +		/// +		/// Global only +		PO_CA_FILE,  		/// String of host/port to use as simple HTTP proxy.  This is  		/// going to change in the future into something more elaborate  		/// that may support richer schemes. -		GP_HTTP_PROXY, +		/// +		/// Global only +		PO_HTTP_PROXY,  		/// Long value that if non-zero enables the use of the  		/// traditional LLProxy code for http/socks5 support.  If -		/// enabled, has priority over GP_HTTP_PROXY. -		GP_LLPROXY, +		// enabled, has priority over GP_HTTP_PROXY. +		/// +		/// Global only +		PO_LLPROXY,  		/// Long value setting the logging trace level for the  		/// library.  Possible values are: @@ -143,50 +179,59 @@ public:  		/// These values are also used in the trace modes for  		/// individual requests in HttpOptions.  Also be aware that  		/// tracing tends to impact performance of the viewer. -		GP_TRACE -	}; - -	/// Set a parameter on a global policy option.  Calls -	/// made after the start of the servicing thread are -	/// not honored and return an error status. -	/// -	/// @param opt		Enum of option to be set. -	/// @param value	Desired value of option. -	/// @return			Standard status code. -	static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); -	static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); - -	/// Create a new policy class into which requests can be made. -	/// -	/// @return			If positive, the policy_id used to reference -	///					the class in other methods.  If 0, an error -	///					occurred and @see getStatus() may provide more -	///					detail on the reason. -	static policy_t createPolicyClass(); - -	enum EClassPolicy -	{ -		/// Limits the number of connections used for the class. -		CP_CONNECTION_LIMIT, - -		/// Limits the number of connections used for a single -		/// literal address/port pair within the class. -		CP_PER_HOST_CONNECTION_LIMIT, +		/// +		/// Global only +		PO_TRACE,  		/// Suitable requests are allowed to pipeline on their  		/// connections when they ask for it. -		CP_ENABLE_PIPELINING +		/// +		/// Per-class only +		PO_ENABLE_PIPELINING, + +		/// Controls whether client-side throttling should be +		/// performed on this policy class.  Positive values +		/// enable throttling and specify the request rate +		/// (requests per second) that should be targetted. +		/// A value of zero, the default, specifies no throttling. +		/// +		/// Per-class only +		PO_THROTTLE_RATE, +		 +		PO_LAST  // Always at end  	}; -	 + +	/// Set a policy option for a global or class parameter at +	/// startup time (prior to thread start). +	/// +	/// @param opt			Enum of option to be set. +	/// @param pclass		For class-based options, the policy class ID to +	///					    be changed.  For globals, specify GLOBAL_POLICY_ID. +	/// @param value		Desired value of option. +	/// @param ret_value	Pointer to receive effective set value +	///						if successful.  May be NULL if effective +	///						value not wanted. +	/// @return				Standard status code. +	static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, +											long value, long * ret_value); +	static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, +											const std::string & value, std::string * ret_value); +  	/// Set a parameter on a class-based policy option.  Calls  	/// made after the start of the servicing thread are  	/// not honored and return an error status.  	/// -	/// @param policy_id		ID of class as returned by @see createPolicyClass(). -	/// @param opt				Enum of option to be set. -	/// @param value			Desired value of option. -	/// @return					Standard status code. -	static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); +	/// @param opt			Enum of option to be set. +	/// @param pclass		For class-based options, the policy class ID to +	///					    be changed.  Ignored for globals but recommend +	///					    using INVALID_POLICY_ID in this case. +	/// @param value		Desired value of option. +	/// @return				Handle of dynamic request.  Use @see getStatus() if +	///						the returned handle is invalid. +	HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, +							   HttpHandler * handler); +	HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, +							   HttpHandler * handler);  	/// @} @@ -488,16 +533,6 @@ public:  	/// @} -	/// @name DynamicPolicyMethods -	/// -	/// @{ - -	/// Request that a running transport pick up a new proxy setting. -	/// An empty string will indicate no proxy is to be used. -	HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); - -    /// @} -  protected:  	void generateNotification(HttpOperation * op); @@ -519,7 +554,6 @@ private:  	/// Must be established before any threading is allowed to  	/// start.  	/// -	static policy_t		sNextPolicyID;  	/// @}  	// End Global State diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index a552e48a1b..c974395b0a 100755 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -39,7 +39,9 @@ HttpResponse::HttpResponse()  	  mReplyLength(0U),  	  mReplyFullLength(0U),  	  mBufferArray(NULL), -	  mHeaders(NULL) +	  mHeaders(NULL), +	  mRetries(0U), +	  m503Retries(0U)  {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index f19b521fbf..aee64e2878 100755 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -149,6 +149,25 @@ public:  			mContentType = con_type;  		} +	/// Get and set retry attempt information on the request. +	void getRetries(unsigned int * retries, unsigned int * retries_503) const +		{ +			if (retries) +			{ +				*retries = mRetries; +			} +			if (retries_503) +			{ +				*retries_503 = m503Retries; +			} +		} + +	void setRetries(unsigned int retries, unsigned int retries_503) +		{ +			mRetries = retries; +			m503Retries = retries_503; +		} +  protected:  	// Response data here  	HttpStatus			mStatus; @@ -158,6 +177,8 @@ protected:  	BufferArray *		mBufferArray;  	HttpHeaders *		mHeaders;  	std::string			mContentType; +	unsigned int		mRetries; +	unsigned int		m503Retries;  }; diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 0f0876b467..43f7e36da5 100755 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -1222,7 +1222,7 @@ void HttpRequestTestObjectType::test<12>()  		HttpRequest::createService();  		// Enable tracing -		HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); +		HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL);  		// Start threading early so that thread memory is invariant  		// over the test. @@ -1340,7 +1340,7 @@ void HttpRequestTestObjectType::test<13>()  		HttpRequest::createService();  		// Enable tracing -		HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); +		HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL);  		// Start threading early so that thread memory is invariant  		// over the test. @@ -3175,6 +3175,142 @@ void HttpRequestTestObjectType::test<22>()  	}  } +template <> template <> +void HttpRequestTestObjectType::test<23>() +{ +	ScopedCurlInit ready; + +	set_test_name("HttpRequest GET 503s with 'Retry-After'"); + +	// This tests mainly that the code doesn't fall over if +	// various well- and mis-formed Retry-After headers are +	// sent along with the response.  Direct inspection of +	// the parsing result isn't supported. +	 +	// Handler can be stack-allocated *if* there are no dangling +	// references to it after completion of this method. +	// Create before memory record as the string copy will bump numbers. +	TestHandler2 handler(this, "handler"); +	std::string url_base(get_base_url() + "/503/");	// path to 503 generators +		 +	// record the total amount of dynamically allocated memory +	mMemTotal = GetMemTotal(); +	mHandlerCalls = 0; + +	HttpRequest * req = NULL; +	HttpOptions * opts = NULL; +	 +	try +	{ +		// Get singletons created +		HttpRequest::createService(); +		 +		// Start threading early so that thread memory is invariant +		// over the test. +		HttpRequest::startThread(); + +		// create a new ref counted object with an implicit reference +		req = new HttpRequest(); +		ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + +		opts = new HttpOptions(); +		opts->setRetries(1);			// Retry once only +		opts->setUseRetryAfter(true);	// Try to parse the retry-after header +		 +		// Issue a GET that 503s with valid retry-after +		mStatus = HttpStatus(503); +		int url_limit(6); +		for (int i(0); i < url_limit; ++i) +		{ +			std::ostringstream url; +			url << url_base << i << "/"; +			HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, +														 0U, +														 url.str(), +														 0, +														 0, +														 opts, +														 NULL, +														 &handler); + +			std::ostringstream testtag; +			testtag << "Valid handle returned for 503 request #" << i; +			ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); +		} +		 + +		// Run the notification pump. +		int count(0); +		int limit(LOOP_COUNT_LONG); +		while (count++ < limit && mHandlerCalls < url_limit) +		{ +			req->update(0); +			usleep(LOOP_SLEEP_INTERVAL); +		} +		ensure("Request executed in reasonable time", count < limit); +		ensure("One handler invocation for request", mHandlerCalls == url_limit); + +		// Okay, request a shutdown of the servicing thread +		mStatus = HttpStatus(); +		mHandlerCalls = 0; +		HttpHandle handle = req->requestStopThread(&handler); +		ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); +	 +		// Run the notification pump again +		count = 0; +		limit = LOOP_COUNT_LONG; +		while (count++ < limit && mHandlerCalls < 1) +		{ +			req->update(1000000); +			usleep(LOOP_SLEEP_INTERVAL); +		} +		ensure("Second request executed in reasonable time", count < limit); +		ensure("Second handler invocation", mHandlerCalls == 1); + +		// See that we actually shutdown the thread +		count = 0; +		limit = LOOP_COUNT_SHORT; +		while (count++ < limit && ! HttpService::isStopped()) +		{ +			usleep(LOOP_SLEEP_INTERVAL); +		} +		ensure("Thread actually stopped running", HttpService::isStopped()); + +		// release options +		opts->release(); +		opts = NULL; +		 +		// release the request object +		delete req; +		req = NULL; + +		// Shut down service +		HttpRequest::destroyService(); +	 +#if defined(WIN32) +		// Can only do this memory test on Windows.  On other platforms, +		// the LL logging system holds on to memory and produces what looks +		// like memory leaks... +	 +		// printf("Old mem:  %d, New mem:  %d\n", mMemTotal, GetMemTotal()); +		ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif +	} +	catch (...) +	{ +		stop_thread(req); +		if (opts) +		{ +			opts->release(); +			opts = NULL; +		} +		delete req; +		HttpRequest::destroyService(); +		throw; +	} +} + +  }  // end namespace tut  namespace diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index b5538528c5..0b379836c9 100755 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -259,6 +259,65 @@ void HttpStatusTestObjectType::test<7>()  	ensure(msg == "Unknown error");  } + +template <> template <> +void HttpStatusTestObjectType::test<8>() +{ +	set_test_name("HttpStatus toHex() nominal function"); +	 +	HttpStatus status(404); +	std::string msg = status.toHex(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure(msg == "01940001"); +} + + +template <> template <> +void HttpStatusTestObjectType::test<9>() +{ +	set_test_name("HttpStatus toTerseString() nominal function"); +	 +	HttpStatus status(404); +	std::string msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Normal HTTP 404", msg == "Http_404"); + +	status = HttpStatus(200); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Normal HTTP 200", msg == "Http_200"); + +	status = HttpStatus(200, HE_REPLY_ERROR); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Unsuccessful HTTP 200", msg == "Http_200");			// No distinction for error + +	status = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Easy couldn't connect error", msg == "Easy_7"); + +	status = HttpStatus(HttpStatus::EXT_CURL_MULTI, CURLM_OUT_OF_MEMORY); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Multi out-of-memory error", msg == "Multi_3"); + +	status = HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_SET); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Core option not set error", msg == "Core_7"); + +	status = HttpStatus(22000, 1); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Undecodable error", msg == "Unknown_1"); + +	status = HttpStatus(22000, -1); +	msg = status.toTerseString(); +	// std::cout << "Result:  " << msg << std::endl; +	ensure("Undecodable error 65535", msg == "Unknown_65535"); +} +  } // end namespace tut  #endif	// TEST_HTTP_STATUS_H diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 3c3af8dc75..04cde651c4 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -69,6 +69,15 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):                             "Content-Range: bytes 0-75/2983",                             "Content-Length: 76"      -- '/bug2295/inv_cont_range/0/'  Generates HE_INVALID_CONTENT_RANGE error in llcorehttp. +    - '/503/'           Generate 503 responses with various kinds +                        of 'retry-after' headers +    -- '/503/0/'            "Retry-After: 2"    +    -- '/503/1/'            "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" +    -- '/503/2/'            "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" +    -- '/503/3/'            "Retry-After: " +    -- '/503/4/'            "Retry-After: (*#*(@*(@(")" +    -- '/503/5/'            "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" +    -- '/503/6/'            "Retry-After: 1 2 3 4 5 6 7 8 9 10"      Some combinations make no sense, there's no effort to protect      you from that. @@ -143,22 +152,40 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):          if "/sleep/" in self.path:              time.sleep(30) -        if "fail" in self.path: -            status = data.get("status", 500) -            # self.responses maps an int status to a (short, long) pair of -            # strings. We want the longer string. That's why we pass a string -            # pair to get(): the [1] will select the second string, whether it -            # came from self.responses or from our default pair. -            reason = data.get("reason", -                               self.responses.get(status, -                                                  ("fail requested", -                                                   "Your request specified failure status %s " -                                                   "without providing a reason" % status))[1]) -            debug("fail requested: %s: %r", status, reason) -            self.send_error(status, reason) +        if "/503/" in self.path: +            # Tests for various kinds of 'Retry-After' header parsing +            body = None +            if "/503/0/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "2") +            elif "/503/1/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") +            elif "/503/2/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") +            elif "/503/3/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "") +            elif "/503/4/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "(*#*(@*(@(") +            elif "/503/5/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") +            elif "/503/6/" in self.path: +                self.send_response(503) +                self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") +            else: +                # Unknown request +                self.send_response(400) +                body = "Unknown /503/ path in server"              if "/reflect/" in self.path:                  self.reflect_headers() +            self.send_header("Content-type", "text/plain")              self.end_headers() +            if body: +                self.wfile.write(body)          elif "/bug2295/" in self.path:              # Test for https://jira.secondlife.com/browse/BUG-2295              # @@ -194,8 +221,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):              self.end_headers()              if body:                  self.wfile.write(body) -        else: -            # Normal response path +        elif "fail" not in self.path:              data = data.copy()          # we're going to modify              # Ensure there's a "reply" key in data, even if there wasn't before              data["reply"] = data.get("reply", llsd.LLSD("success")) @@ -210,6 +236,22 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):              self.end_headers()              if withdata:                  self.wfile.write(response) +        else:                           # fail requested +            status = data.get("status", 500) +            # self.responses maps an int status to a (short, long) pair of +            # strings. We want the longer string. That's why we pass a string +            # pair to get(): the [1] will select the second string, whether it +            # came from self.responses or from our default pair. +            reason = data.get("reason", +                               self.responses.get(status, +                                                  ("fail requested", +                                                   "Your request specified failure status %s " +                                                   "without providing a reason" % status))[1]) +            debug("fail requested: %s: %r", status, reason) +            self.send_error(status, reason) +            if "/reflect/" in self.path: +                self.reflect_headers() +            self.end_headers()      def reflect_headers(self):          for name in self.headers.keys(): diff --git a/indra/llmessage/llcurl.cpp b/indra/llmessage/llcurl.cpp index e8c508dcbf..c28494e2c8 100755 --- a/indra/llmessage/llcurl.cpp +++ b/indra/llmessage/llcurl.cpp @@ -6,7 +6,7 @@   *   * $LicenseInfo:firstyear=2006&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -370,9 +370,12 @@ LLCurl::Easy* LLCurl::Easy::getEasy()  		return NULL;  	} -	// set no DNS caching as default for all easy handles. This prevents them adopting a -	// multi handles cache if they are added to one. -	CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); +	// Enable a brief cache period for now.  This was zero for the longest time +	// which caused some routers grief and generated unneeded traffic.  For the +	// threaded resolver, we're using system resolution libraries and non-zero values +	// are preferred.  The c-ares resolver is another matter and it might not +	// track server changes as well. +	CURLcode result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_DNS_CACHE_TIMEOUT, 15);  	check_curl_code(result);  	result = curl_easy_setopt(easy->mCurlEasyHandle, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);  	check_curl_code(result); diff --git a/indra/newview/VIEWER_VERSION.txt b/indra/newview/VIEWER_VERSION.txt index 0b2eb36f50..c1e43e6d45 100644 --- a/indra/newview/VIEWER_VERSION.txt +++ b/indra/newview/VIEWER_VERSION.txt @@ -1 +1 @@ -3.7.2 +3.7.3 diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 1d62bba39f..75e6c4abb4 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -10068,11 +10068,21 @@      <key>Value</key>      <real>16</real>    </map> - +  <key>Mesh2MaxConcurrentRequests</key> +  <map> +    <key>Comment</key> +    <string>Number of connections to use for loading meshes.</string> +    <key>Persist</key> +    <integer>1</integer> +    <key>Type</key> +    <string>U32</string> +    <key>Value</key> +    <integer>8</integer> +  </map>    <key>MeshMaxConcurrentRequests</key>    <map>      <key>Comment</key> -    <string>Number of threads to use for loading meshes.</string> +    <string>Number of connections to use for loading meshes (legacy system).</string>      <key>Persist</key>      <integer>1</integer>      <key>Type</key> @@ -10080,6 +10090,28 @@      <key>Value</key>      <integer>32</integer>    </map> +  <key>MeshUseHttpRetryAfter</key> +  <map> +    <key>Comment</key> +    <string>If TRUE, use Retry-After response headers when rescheduling a mesh request that fails with an HTTP 503 status.  Static.</string> +    <key>Persist</key> +    <integer>1</integer> +    <key>Type</key> +    <string>Boolean</string> +    <key>Value</key> +    <boolean>1</boolean> +  </map> +  <key>MeshUseGetMesh1</key> +  <map> +    <key>Comment</key> +    <string>If TRUE, use the legacy GetMesh capability for mesh download requests.  Semi-dynamic (read at region crossings).</string> +    <key>Persist</key> +    <integer>1</integer> +    <key>Type</key> +    <string>Boolean</string> +    <key>Value</key> +    <boolean>0</boolean> +  </map>     <key>RunMultipleThreads</key>      <map>        <key>Comment</key> diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 0d7d41304d..70dcffefb2 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -28,18 +28,81 @@  #include "llappcorehttp.h" +#include "llappviewer.h"  #include "llviewercontrol.h" +// Here is where we begin to get our connection usage under control. +// This establishes llcorehttp policy classes that, among other +// things, limit the maximum number of connections to outside +// services.  Each of the entries below maps to a policy class and +// has a limit, sometimes configurable, of how many connections can +// be open at a time. +  const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); +static const struct +{ +	LLAppCoreHttp::EAppPolicy	mPolicy; +	U32							mDefault; +	U32							mMin; +	U32							mMax; +	U32							mRate; +	std::string					mKey; +	const char *				mUsage; +} init_data[] =					//  Default and dynamic values for classes +{ +	{ +		LLAppCoreHttp::AP_DEFAULT,			8,		8,		8,		0, +		"", +		"other" +	}, +	{ +		LLAppCoreHttp::AP_TEXTURE,			8,		1,		12,		0, +		"TextureFetchConcurrency", +		"texture fetch" +	}, +	{ +		LLAppCoreHttp::AP_MESH1,			32,		1,		128,	100, +		"MeshMaxConcurrentRequests", +		"mesh fetch" +	}, +	{ +		LLAppCoreHttp::AP_MESH2,			8,		1,		32,		100, +		"Mesh2MaxConcurrentRequests", +		"mesh2 fetch" +	}, +	{ +		LLAppCoreHttp::AP_LARGE_MESH,		2,		1,		8,		0, +		"", +		"large mesh fetch" +	}, +	{ +		LLAppCoreHttp::AP_UPLOADS,			2,		1,		8,		0, +		"", +		"asset upload" +	}, +	{ +		LLAppCoreHttp::AP_LONG_POLL,		32,		32,		32,		0, +		"", +		"long poll" +	} +}; + +static void setting_changed(); +  LLAppCoreHttp::LLAppCoreHttp()  	: mRequest(NULL),  	  mStopHandle(LLCORE_HTTP_HANDLE_INVALID),  	  mStopRequested(0.0), -	  mStopped(false), -	  mPolicyDefault(-1) -{} +	  mStopped(false) +{ +	for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) +	{ +		mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; +		mSettings[i] = 0U; +	} +}  LLAppCoreHttp::~LLAppCoreHttp() @@ -54,30 +117,28 @@ void LLAppCoreHttp::init()  	LLCore::HttpStatus status = LLCore::HttpRequest::createService();  	if (! status)  	{ -		LL_ERRS("Init") << "Failed to initialize HTTP services.  Reason:  " -						<< status.toString() +		LL_ERRS("Init") << "Failed to initialize HTTP services.  Reason:  " << status.toString()  						<< LL_ENDL;  	}  	// Point to our certs or SSH/https: will fail on connect -	status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, -														gDirUtilp->getCAFile()); +	status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, +														LLCore::HttpRequest::GLOBAL_POLICY_ID, +														gDirUtilp->getCAFile(), NULL);  	if (! status)  	{ -		LL_ERRS("Init") << "Failed to set CA File for HTTP services.  Reason:  " -						<< status.toString() +		LL_ERRS("Init") << "Failed to set CA File for HTTP services.  Reason:  " << status.toString()  						<< LL_ENDL;  	} -	// Establish HTTP Proxy.  "LLProxy" is a special string which directs -	// the code to use LLProxy::applyProxySettings() to establish any -	// HTTP or SOCKS proxy for http operations. -	status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); +	// Establish HTTP Proxy, if desired. +	status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_LLPROXY, +														LLCore::HttpRequest::GLOBAL_POLICY_ID, +														1, NULL);  	if (! status)  	{ -		LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services.  Reason:  " -						<< status.toString() -						<< LL_ENDL; +		LL_WARNS("Init") << "Failed to set HTTP proxy for HTTP services.  Reason:  " << status.toString() +						 << LL_ENDL;  	}  	// Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): @@ -90,47 +151,74 @@ void LLAppCoreHttp::init()  	{  		long trace_level(0L);  		trace_level = long(gSavedSettings.getU32(http_trace)); -		status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, trace_level); +		status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, +															LLCore::HttpRequest::GLOBAL_POLICY_ID, +															trace_level, NULL);  	}  	// Setup default policy and constrain if directed to -	mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; -	static const std::string texture_concur("TextureFetchConcurrency"); -	if (gSavedSettings.controlExists(texture_concur)) +	mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + +	// Setup additional policies based on table and some special rules +	for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i)  	{ -		U32 concur(llmin(gSavedSettings.getU32(texture_concur), U32(12))); +		const EAppPolicy policy(init_data[i].mPolicy); -		if (concur > 0) +		if (AP_DEFAULT == policy)  		{ -			LLCore::HttpStatus status; -			status = LLCore::HttpRequest::setPolicyClassOption(mPolicyDefault, -															   LLCore::HttpRequest::CP_CONNECTION_LIMIT, -															   concur); -			if (! status) -			{ -				LL_WARNS("Init") << "Unable to set texture fetch concurrency.  Reason:  " -								 << status.toString() -								 << LL_ENDL; -			} -			else -			{ -				LL_INFOS("Init") << "Application settings overriding default texture fetch concurrency.  New value:  " -								 << concur -								 << LL_ENDL; -			} +			// Pre-created +			continue; +		} + +		mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); +		if (! mPolicies[policy]) +		{ +			// Use default policy (but don't accidentally modify default) +			LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage +							 << ".  Using default policy." +							 << LL_ENDL; +			mPolicies[policy] = mPolicies[AP_DEFAULT]; +			continue;  		}  	} + +	// Need a request object to handle dynamic options before setting them +	mRequest = new LLCore::HttpRequest; + +	// Apply initial settings +	refreshSettings(true);  	// Kick the thread  	status = LLCore::HttpRequest::startThread();  	if (! status)  	{ -		LL_ERRS("Init") << "Failed to start HTTP servicing thread.  Reason:  " -						<< status.toString() +		LL_ERRS("Init") << "Failed to start HTTP servicing thread.  Reason:  " << status.toString()  						<< LL_ENDL;  	} -	mRequest = new LLCore::HttpRequest; +	// Register signals for settings and state changes +	for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) +	{ +		if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) +		{ +			LLPointer<LLControlVariable> cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); +			if (cntrl_ptr.isNull()) +			{ +				LL_WARNS("Init") << "Unable to set signal on global setting '" << init_data[i].mKey +								 << "'" << LL_ENDL; +			} +			else +			{ +				mSettingsSignal[i] = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); +			} +		} +	} +} + + +void setting_changed() +{ +	LLAppViewer::instance()->getAppCoreHttp().refreshSettings(false);  } @@ -173,6 +261,11 @@ void LLAppCoreHttp::cleanup()  		}  	} +	for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) +	{ +		mSettingsSignal[i].disconnect(); +	} +	  	delete mRequest;  	mRequest = NULL; @@ -185,6 +278,78 @@ void LLAppCoreHttp::cleanup()  	}  } +void LLAppCoreHttp::refreshSettings(bool initial) +{ +	LLCore::HttpStatus status; +	 +	for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) +	{ +		const EAppPolicy policy(init_data[i].mPolicy); + +		// Set any desired throttle +		if (initial && init_data[i].mRate) +		{ +			// Init-time only, can use the static setters here +			status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, +																mPolicies[policy], +																init_data[i].mRate, +																NULL); +			if (! status) +			{ +				LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage +								 << " throttle rate.  Reason:  " << status.toString() +								 << LL_ENDL; +			} +		} + +		// Get target connection concurrency value +		U32 setting(init_data[i].mDefault); +		if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) +		{ +			U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); +			if (new_setting) +			{ +				// Treat zero settings as an ask for default +				setting = llclamp(new_setting, init_data[i].mMin, init_data[i].mMax); +			} +		} + +		if (! initial && setting == mSettings[policy]) +		{ +			// Unchanged, try next setting +			continue; +		} +		 +		// Set it and report +		// *TODO:  These are intended to be per-host limits when we can +		// support that in llcorehttp/libcurl. +		LLCore::HttpHandle handle; +		handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, +										   mPolicies[policy], +										   setting, NULL); +		if (LLCORE_HTTP_HANDLE_INVALID == handle) +		{ +			status = mRequest->getStatus(); +			LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage +							 << " concurrency.  Reason:  " << status.toString() +							 << LL_ENDL; +		} +		else +		{ +			LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage +							  << " concurrency.  New value:  " << setting +							  << LL_ENDL; +			mSettings[policy] = setting; +			if (initial && setting != init_data[i].mDefault) +			{ +				LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage +								 << " concurrency.  New value:  " << setting +								 << LL_ENDL; +			} +		} +	} +} +  void LLAppCoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *)  { diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 241d73ad52..40e3042b84 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2012&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *   * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -41,6 +41,117 @@  class LLAppCoreHttp : public LLCore::HttpHandler  {  public: +	typedef LLCore::HttpRequest::policy_t policy_t; + +	enum EAppPolicy +	{ +		/// Catchall policy class.  Not used yet +		/// but will have a generous concurrency +		/// limit.  Deep queueing possible by having +		/// a chatty HTTP user. +		/// +		/// Destination:     anywhere +		/// Protocol:        http: or https: +		/// Transfer size:   KB-MB +		/// Long poll:       no +		/// Concurrency:     high  +		/// Request rate:    unknown +		/// Pipelined:       no +		AP_DEFAULT, + +		/// Texture fetching policy class.  Used to +		/// download textures via capability or SSA +		/// baking service.  Deep queueing of requests. +		/// Do not share. +		/// +		/// Destination:     simhost:12046 & bake-texture:80 +		/// Protocol:        http: +		/// Transfer size:   KB-MB +		/// Long poll:       no +		/// Concurrency:     high +		/// Request rate:    high +		/// Pipelined:       soon +		AP_TEXTURE, + +		/// Legacy mesh fetching policy class.  Used to +		/// download textures via 'GetMesh' capability. +		/// To be deprecated.  Do not share. +		/// +		/// Destination:     simhost:12046 +		/// Protocol:        http: +		/// Transfer size:   KB-MB +		/// Long poll:       no +		/// Concurrency:     dangerously high +		/// Request rate:    high +		/// Pipelined:       no +		AP_MESH1, + +		/// New mesh fetching policy class.  Used to +		/// download textures via 'GetMesh2' capability. +		/// Used when fetch request (typically one LOD) +		/// is 'small', currently defined as 2MB. +		/// Very deeply queued.  Do not share. +		/// +		/// Destination:     simhost:12046 +		/// Protocol:        http: +		/// Transfer size:   KB-MB +		/// Long poll:       no +		/// Concurrency:     high +		/// Request rate:    high +		/// Pipelined:       soon +		AP_MESH2, + +		/// Large mesh fetching policy class.  Used to +		/// download textures via 'GetMesh' or 'GetMesh2' +		/// capability.  Used when fetch request +		/// is not small to avoid head-of-line problem +		/// when large requests block a sequence of small, +		/// fast requests.  Can be shared with similar +		/// traffic that can wait for longish stalls +		/// (default timeout 600S). +		/// +		/// Destination:     simhost:12046 +		/// Protocol:        http: +		/// Transfer size:   MB +		/// Long poll:       no +		/// Concurrency:     low +		/// Request rate:    low +		/// Pipelined:       soon +		AP_LARGE_MESH, + +		/// Asset upload policy class.  Used to store +		/// assets (mesh only at the moment) via +		/// changeable URL.  Responses may take some +		/// time (default timeout 240S). +		/// +		/// Destination:     simhost:12043 +		/// Protocol:        https: +		/// Transfer size:   KB-MB +		/// Long poll:       no +		/// Concurrency:     low +		/// Request rate:    low +		/// Pipelined:       no +		AP_UPLOADS, + +		/// Long-poll-type HTTP requests.  Not +		/// bound by a connection limit.  Requests +		/// will typically hang around for a long +		/// time (~30S).  Only shareable with other +		/// long-poll requests. +		/// +		/// Destination:     simhost:12043 +		/// Protocol:        https: +		/// Transfer size:   KB +		/// Long poll:       yes +		/// Concurrency:     unlimited but low in practice +		/// Request rate:    low +		/// Pipelined:       no +		AP_LONG_POLL, + +		AP_COUNT						// Must be last +	}; +	 +public:  	LLAppCoreHttp();  	~LLAppCoreHttp(); @@ -65,21 +176,27 @@ public:  	// Notification when the stop request is complete.  	virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); -	// Retrieve the policy class for default operations. -	int getPolicyDefault() const +	// Retrieve a policy class identifier for desired +	// application function. +	policy_t getPolicy(EAppPolicy policy) const  		{ -			return mPolicyDefault; +			return mPolicies[policy];  		} + +	// Apply initial or new settings from the environment. +	void refreshSettings(bool initial);  private:  	static const F64			MAX_THREAD_WAIT_TIME;  private: -	LLCore::HttpRequest *		mRequest; +	LLCore::HttpRequest *		mRequest;						// Request queue to issue shutdowns  	LLCore::HttpHandle			mStopHandle;  	F64							mStopRequested;  	bool						mStopped; -	int							mPolicyDefault; +	policy_t					mPolicies[AP_COUNT];			// Policy class id for each connection set +	U32							mSettings[AP_COUNT]; +	boost::signals2::connection mSettingsSignal[AP_COUNT];		// Signals to global settings that affect us  }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index ff4d96eb54..269c361754 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -5,7 +5,7 @@   *   * $LicenseInfo:firstyear=2005&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2014, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -38,11 +38,13 @@  #include "llcallbacklist.h"  #include "llcurl.h"  #include "lldatapacker.h" +#include "lldeadmantimer.h"  #include "llfloatermodelpreview.h"  #include "llfloaterperms.h"  #include "lleconomy.h"  #include "llimagej2c.h"  #include "llhost.h" +#include "llmath.h"  #include "llnotificationsutil.h"  #include "llsd.h"  #include "llsdutil_math.h" @@ -52,6 +54,7 @@  #include "llviewercontrol.h"  #include "llviewerinventory.h"  #include "llviewermenufile.h" +#include "llviewermessage.h"  #include "llviewerobjectlist.h"  #include "llviewerregion.h"  #include "llviewertexturelist.h" @@ -65,6 +68,9 @@  #include "llfoldertype.h"  #include "llviewerparcelmgr.h"  #include "lluploadfloaterobservers.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llfasttimer.h"  #include "boost/lexical_cast.hpp" @@ -72,11 +78,296 @@  #include "netdb.h"  #endif -#include <queue> + +// Purpose +// +//   The purpose of this module is to provide access between the viewer +//   and the asset system as regards to mesh objects. +// +//   * High-throughput download of mesh assets from servers while +//     following best industry practices for network profile. +//   * Reliable expensing and upload of new mesh assets. +//   * Recovery and retry from errors when appropriate. +//   * Decomposition of mesh assets for preview and uploads. +//   * And most important:  all of the above without exposing the +//     main thread to stalls due to deep processing or thread +//     locking actions.  In particular, the following operations +//     on LLMeshRepository are very averse to any stalls: +//     * loadMesh +//     * getMeshHeader (For structural details, see: +//       http://wiki.secondlife.com/wiki/Mesh/Mesh_Asset_Format) +//     * notifyLoadedMeshes +//     * getSkinInfo +// +// Threads +// +//   main     Main rendering thread, very sensitive to locking and other stalls +//   repo     Overseeing worker thread associated with the LLMeshRepoThread class +//   decom    Worker thread for mesh decomposition requests +//   core     HTTP worker thread:  does the work but doesn't intrude here +//   uploadN  0-N temporary mesh upload threads (0-1 in practice) +// +// Sequence of Operations +// +//   What follows is a description of the retrieval of one LOD for +//   a new mesh object.  Work is performed by a series of short, quick +//   actions distributed over a number of threads.  Each is meant +//   to proceed without stalling and the whole forms a deep request +//   pipeline to achieve throughput.  Ellipsis indicates a return +//   or break in processing which is resumed elsewhere. +// +//         main thread         repo thread (run() method) +// +//         loadMesh() invoked to request LOD +//           append LODRequest to mPendingRequests +//         ... +//         other mesh requests may be made +//         ... +//         notifyLoadedMeshes() invoked to stage work +//           append HeaderRequest to mHeaderReqQ +//         ... +//                             scan mHeaderReqQ +//                             issue 4096-byte GET for header +//                             ... +//                             onCompleted() invoked for GET +//                               data copied +//                               headerReceived() invoked +//                                 LLSD parsed +//                                 mMeshHeader, mMeshHeaderSize updated +//                                 scan mPendingLOD for LOD request +//                                 push LODRequest to mLODReqQ +//                             ... +//                             scan mLODReqQ +//                             fetchMeshLOD() invoked +//                               issue Byte-Range GET for LOD +//                             ... +//                             onCompleted() invoked for GET +//                               data copied +//                               lodReceived() invoked +//                                 unpack data into LLVolume +//                                 append LoadedMesh to mLoadedQ +//                             ... +//         notifyLoadedMeshes() invoked again +//           scan mLoadedQ +//           notifyMeshLoaded() for LOD +//             setMeshAssetLoaded() invoked for system volume +//             notifyMeshLoaded() invoked for each interested object +//         ... +// +// Mutexes +// +//   LLMeshRepository::mMeshMutex +//   LLMeshRepoThread::mMutex +//   LLMeshRepoThread::mHeaderMutex +//   LLMeshRepoThread::mSignal (LLCondition) +//   LLPhysicsDecomp::mSignal (LLCondition) +//   LLPhysicsDecomp::mMutex +//   LLMeshUploadThread::mMutex +// +// Mutex Order Rules +// +//   1.  LLMeshRepoThread::mMutex before LLMeshRepoThread::mHeaderMutex +//   2.  LLMeshRepository::mMeshMutex before LLMeshRepoThread::mMutex +//   (There are more rules, haven't been extracted.) +// +// Data Member Access/Locking +// +//   Description of how shared access to static and instance data +//   members is performed.  Each member is followed by the name of +//   the mutex, if any, covering the data and then a list of data +//   access models each of which is a triplet of the following form: +// +//     {ro, wo, rw}.{main, repo, any}.{mutex, none} +//     Type of access:  read-only, write-only, read-write. +//     Accessing thread or 'any' +//     Relevant mutex held during access (several may be held) or 'none' +// +//   A careful eye will notice some unsafe operations.  Many of these +//   have an alibi of some form.  Several types of alibi are identified +//   and listed here: +// +//     [0]  No alibi.  Probably unsafe. +//     [1]  Single-writer, self-consistent readers.  Old data must +//          be tolerated by any reader but data will come true eventually. +//     [2]  Like [1] but provides a hint about thread state.  These +//          may be unsafe. +//     [3]  empty() check outside of lock.  Can me made safish when +//          done in double-check lock style.  But this depends on +//          std:: implementation and memory model. +//     [4]  Appears to be covered by a mutex but doesn't need one. +//     [5]  Read of a double-checked lock. +// +//   So, in addition to documentation, take this as a to-do/review +//   list and see if you can improve things.  For porters to non-x86 +//   architectures, the weaker memory models will make these platforms +//   probabilistically more susceptible to hitting race conditions. +//   True here and in other multi-thread code such as texture fetching. +//   (Strong memory models make weak programmers.  Weak memory models +//   make strong programmers.  Ref:  arm, ppc, mips, alpha) +// +//   LLMeshRepository: +// +//     sBytesReceived                  none            rw.repo.none, ro.main.none [1] +//     sMeshRequestCount               " +//     sHTTPRequestCount               " +//     sHTTPLargeRequestCount          " +//     sHTTPRetryCount                 " +//     sHTTPErrorCount                 " +//     sLODPending                     mMeshMutex [4]  rw.main.mMeshMutex +//     sLODProcessing                  Repo::mMutex    rw.any.Repo::mMutex +//     sCacheBytesRead                 none            rw.repo.none, ro.main.none [1] +//     sCacheBytesWritten              " +//     sCacheReads                     " +//     sCacheWrites                    " +//     mLoadingMeshes                  mMeshMutex [4]  rw.main.none, rw.any.mMeshMutex +//     mSkinMap                        none            rw.main.none +//     mDecompositionMap               none            rw.main.none +//     mPendingRequests                mMeshMutex [4]  rw.main.mMeshMutex +//     mLoadingSkins                   mMeshMutex [4]  rw.main.mMeshMutex +//     mPendingSkinRequests            mMeshMutex [4]  rw.main.mMeshMutex +//     mLoadingDecompositions          mMeshMutex [4]  rw.main.mMeshMutex +//     mPendingDecompositionRequests   mMeshMutex [4]  rw.main.mMeshMutex +//     mLoadingPhysicsShapes           mMeshMutex [4]  rw.main.mMeshMutex +//     mPendingPhysicsShapeRequests    mMeshMutex [4]  rw.main.mMeshMutex +//     mUploads                        none            rw.main.none (upload thread accessing objects) +//     mUploadWaitList                 none            rw.main.none (upload thread accessing objects) +//     mInventoryQ                     mMeshMutex [4]  rw.main.mMeshMutex, ro.main.none [5] +//     mUploadErrorQ                   mMeshMutex      rw.main.mMeshMutex, rw.any.mMeshMutex +//     mGetMeshVersion                 none            rw.main.none +// +//   LLMeshRepoThread: +// +//     sActiveHeaderRequests    mMutex        rw.any.mMutex, ro.repo.none [1] +//     sActiveLODRequests       mMutex        rw.any.mMutex, ro.repo.none [1] +//     sMaxConcurrentRequests   mMutex        wo.main.none, ro.repo.none, ro.main.mMutex +//     mMeshHeader              mHeaderMutex  rw.repo.mHeaderMutex, ro.main.mHeaderMutex, ro.main.none [0] +//     mMeshHeaderSize          mHeaderMutex  rw.repo.mHeaderMutex +//     mSkinRequests            mMutex        rw.repo.mMutex, ro.repo.none [5] +//     mSkinInfoQ               mMutex        rw.repo.mMutex, rw.main.mMutex [5] (was:  [0]) +//     mDecompositionRequests   mMutex        rw.repo.mMutex, ro.repo.none [5] +//     mPhysicsShapeRequests    mMutex        rw.repo.mMutex, ro.repo.none [5] +//     mDecompositionQ          mMutex        rw.repo.mMutex, rw.main.mMutex [5] (was:  [0]) +//     mHeaderReqQ              mMutex        ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex +//     mLODReqQ                 mMutex        ro.repo.none [5], rw.repo.mMutex, rw.any.mMutex +//     mUnavailableQ            mMutex        rw.repo.none [0], ro.main.none [5], rw.main.mMutex +//     mLoadedQ                 mMutex        rw.repo.mMutex, ro.main.none [5], rw.main.mMutex +//     mPendingLOD              mMutex        rw.repo.mMutex, rw.any.mMutex +//     mGetMeshCapability       mMutex        rw.main.mMutex, ro.repo.mMutex (was:  [0]) +//     mGetMesh2Capability      mMutex        rw.main.mMutex, ro.repo.mMutex (was:  [0]) +//     mGetMeshVersion          mMutex        rw.main.mMutex, ro.repo.mMutex +//     mHttp*                   none          rw.repo.none +// +//   LLMeshUploadThread: +// +//     mDiscarded               mMutex        rw.main.mMutex, ro.uploadN.none [1] +//     ... more ... +// +// QA/Development Testing +// +//   Debug variable 'MeshUploadFakeErrors' takes a mask of bits that will +//   simulate an error on fee query or upload.  Defined bits are: +// +//   0x01            Simulate application error on fee check reading +//                   response body from file "fake_upload_error.xml" +//   0x02            Same as 0x01 but for actual upload attempt. +//   0x04            Simulate a transport problem on fee check with a +//                   locally-generated 500 status. +//   0x08            As with 0x04 but for the upload operation. +// +//   For major changes, see the LL_MESH_FASTTIMER_ENABLE below and +//   instructions for looking for frame stalls using fast timers. +// +// *TODO:  Work list for followup actions: +//   * Review anything marked as unsafe above, verify if there are real issues. +//   * See if we can put ::run() into a hard sleep.  May not actually perform better +//     than the current scheme so be prepared for disappointment.  You'll likely +//     need to introduce a condition variable class that references a mutex in +//     methods rather than derives from mutex which isn't correct. +//   * On upload failures, make more information available to the alerting +//     dialog.  Get the structured information going into the log into a +//     tree there. +//   * Header parse failures come without much explanation.  Elaborate. +//   * Work queue for uploads?  Any need for this or is the current scheme good +//     enough? +//   * Various temp buffers used in VFS I/O might be allocated once or even +//     statically.  Look for some wins here. +//   * Move data structures holding mesh data used by main thread into main- +//     thread-only access so that no locking is needed.  May require duplication +//     of some data so that worker thread has a minimal data set to guide +//     operations. +// +// -------------------------------------------------------------------------- +//                    Development/Debug/QA Tools +// +// Enable here or in build environment to get fasttimer data on mesh fetches. +// +// Typically, this is used to perform A/B testing using the +// fasttimer console (shift-ctrl-9).  This is done by looking +// for stalls due to lock contention between the main thread +// and the repository and HTTP code.  In a release viewer, +// these appear as ping-time or worse spikes in frame time. +// With this instrumentation enabled, a stall will appear +// under the 'Mesh Fetch' timer which will be either top-level +// or under 'Render' time. + +#ifndef	LL_MESH_FASTTIMER_ENABLE +#define LL_MESH_FASTTIMER_ENABLE		1 +#endif +#if LL_MESH_FASTTIMER_ENABLE +static LLFastTimer::DeclareTimer FTM_MESH_FETCH("Mesh Fetch"); + +#define	MESH_FASTTIMER_DEFBLOCK			LLFastTimer meshtimer(FTM_MESH_FETCH) +#else +#define	MESH_FASTTIMER_DEFBLOCK +#endif // LL_MESH_FASTTIMER_ENABLE + + +// Random failure testing for development/QA. +// +// Set the MESH_*_FAILED macros to either 'false' or to +// an invocation of MESH_RANDOM_NTH_TRUE() with some +// suitable number.  In production, all must be false. +// +// Example: +// #define	MESH_HTTP_RESPONSE_FAILED				MESH_RANDOM_NTH_TRUE(9) + +// 1-in-N calls will test true +#define	MESH_RANDOM_NTH_TRUE(_N)				( ll_rand(S32(_N)) == 0 ) + +#define	MESH_HTTP_RESPONSE_FAILED				false +#define	MESH_HEADER_PROCESS_FAILED				false +#define	MESH_LOD_PROCESS_FAILED					false +#define	MESH_SKIN_INFO_PROCESS_FAILED			false +#define	MESH_DECOMP_PROCESS_FAILED				false +#define MESH_PHYS_SHAPE_PROCESS_FAILED			false + +// -------------------------------------------------------------------------- +  LLMeshRepository gMeshRepo; -const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; +const S32 MESH_HEADER_SIZE = 4096;                      // Important:  assumption is that headers fit in this space +const S32 REQUEST_HIGH_WATER_MIN = 32;					// Limits for GetMesh regions +const S32 REQUEST_HIGH_WATER_MAX = 150;					// Should remain under 2X throttle +const S32 REQUEST_LOW_WATER_MIN = 16; +const S32 REQUEST_LOW_WATER_MAX = 75; +const S32 REQUEST2_HIGH_WATER_MIN = 32;					// Limits for GetMesh2 regions +const S32 REQUEST2_HIGH_WATER_MAX = 80; +const S32 REQUEST2_LOW_WATER_MIN = 16; +const S32 REQUEST2_LOW_WATER_MAX = 40; +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21;		// Size at which requests goes to narrow/slow queue +const long SMALL_MESH_XFER_TIMEOUT = 120L;				// Seconds to complete xfer, small mesh downloads +const long LARGE_MESH_XFER_TIMEOUT = 600L;				// Seconds to complete xfer, large downloads + +// Would normally like to retry on uploads as some +// retryable failures would be recoverable.  Unfortunately, +// the mesh service is using 500 (retryable) rather than +// 400/bad request (permanent) for a bad payload and +// retrying that just leads to revocation of the one-shot +// cap which then produces a 404 on retry destroying some +// (occasionally) useful error information.  We'll leave +// upload retries to the user as in the past.  SH-4667. +const long UPLOAD_RETRY_LIMIT = 0L;  // Maximum mesh version to support.  Three least significant digits are reserved for the minor version,   // with major version changes indicating a format change that is not backwards compatible and should not @@ -87,35 +378,45 @@ const U32 MAX_MESH_REQUESTS_PER_SECOND = 100;  const S32 MAX_MESH_VERSION = 999;  U32 LLMeshRepository::sBytesReceived = 0; +U32 LLMeshRepository::sMeshRequestCount = 0;  U32 LLMeshRepository::sHTTPRequestCount = 0; +U32 LLMeshRepository::sHTTPLargeRequestCount = 0;  U32 LLMeshRepository::sHTTPRetryCount = 0; +U32 LLMeshRepository::sHTTPErrorCount = 0;  U32 LLMeshRepository::sLODProcessing = 0;  U32 LLMeshRepository::sLODPending = 0;  U32 LLMeshRepository::sCacheBytesRead = 0;  U32 LLMeshRepository::sCacheBytesWritten = 0; -U32 LLMeshRepository::sPeakKbps = 0; -	 +U32 LLMeshRepository::sCacheReads = 0; +U32 LLMeshRepository::sCacheWrites = 0; +U32 LLMeshRepository::sMaxLockHoldoffs = 0; -const U32 MAX_TEXTURE_UPLOAD_RETRIES = 5; +LLDeadmanTimer LLMeshRepository::sQuiescentTimer(15.0, false);	// true -> gather cpu metrics +	  static S32 dump_num = 0;  std::string make_dump_name(std::string prefix, S32 num)  {  	return prefix + boost::lexical_cast<std::string>(num) + std::string(".xml"); -	  }  void dump_llsd_to_file(const LLSD& content, std::string filename);  LLSD llsd_from_file(std::string filename); -std::string header_lod[] =  +const std::string header_lod[] =   {  	"lowest_lod",  	"low_lod",  	"medium_lod",  	"high_lod"  }; +const char * const LOG_MESH = "Mesh"; +// Static data and functions to measure mesh load +// time metrics for a new region scene. +static unsigned int metrics_teleport_start_count = 0; +boost::signals2::connection metrics_teleport_started_signal; +static void teleport_started();  //get the number of bytes resident in memory for given volume  U32 get_volume_memory_size(const LLVolume* volume) @@ -197,200 +498,233 @@ void get_vertex_buffer_from_mesh(LLCDMeshData& mesh, LLModel::PhysicsMesh& res,  	}  } -S32 LLMeshRepoThread::sActiveHeaderRequests = 0; -S32 LLMeshRepoThread::sActiveLODRequests = 0; +volatile S32 LLMeshRepoThread::sActiveHeaderRequests = 0; +volatile S32 LLMeshRepoThread::sActiveLODRequests = 0;  U32	LLMeshRepoThread::sMaxConcurrentRequests = 1; - -class LLMeshHeaderResponder : public LLCurl::Responder +S32 LLMeshRepoThread::sRequestLowWater = REQUEST2_LOW_WATER_MIN; +S32 LLMeshRepoThread::sRequestHighWater = REQUEST2_HIGH_WATER_MIN; +S32 LLMeshRepoThread::sRequestWaterLevel = 0; + +// Base handler class for all mesh users of llcorehttp. +// This is roughly equivalent to a Responder class in +// traditional LL code.  The base is going to perform +// common response/data handling in the inherited +// onCompleted() method.  Derived classes, one for each +// type of HTTP action, define processData() and +// processFailure() methods to customize handling and +// error messages. +// +// LLCore::HttpHandler +//   LLMeshHandlerBase +//     LLMeshHeaderHandler +//     LLMeshLODHandler +//     LLMeshSkinInfoHandler +//     LLMeshDecompositionHandler +//     LLMeshPhysicsShapeHandler +//   LLMeshUploadThread + +class LLMeshHandlerBase : public LLCore::HttpHandler  { -	LOG_CLASS(LLMeshHeaderResponder);  public: -	LLVolumeParams mMeshParams; -	bool mProcessed; +	LOG_CLASS(LLMeshHandlerBase); +	LLMeshHandlerBase() +		: LLCore::HttpHandler(), +		  mMeshParams(), +		  mProcessed(false), +		  mHttpHandle(LLCORE_HTTP_HANDLE_INVALID) +		{} -	LLMeshHeaderResponder(const LLVolumeParams& mesh_params) -		: mMeshParams(mesh_params) -	{ -		LLMeshRepoThread::incActiveHeaderRequests(); -		mProcessed = false; -	} +	virtual ~LLMeshHandlerBase() +		{} -	~LLMeshHeaderResponder() -	{ -		if (!LLApp::isQuitting()) -		{ -			if (!mProcessed) -			{ //something went wrong, retry -				llwarns << "Timeout or service unavailable, retrying." << llendl; -				LLMeshRepository::sHTTPRetryCount++; -				LLMeshRepoThread::HeaderRequest req(mMeshParams); -				LLMutexLock lock(gMeshRepo.mThread->mMutex); -				gMeshRepo.mThread->mHeaderReqQ.push(req); -			} +protected: +	LLMeshHandlerBase(const LLMeshHandlerBase &);				// Not defined +	void operator=(const LLMeshHandlerBase &);					// Not defined +	 +public: +	virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); +	virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size) = 0; +	virtual void processFailure(LLCore::HttpStatus status) = 0; +	 +public: +	LLVolumeParams		mMeshParams; +	bool				mProcessed; +	LLCore::HttpHandle	mHttpHandle; +}; -			LLMeshRepoThread::decActiveHeaderRequests(); -		} -	} -	virtual void completedRaw(const LLChannelDescriptors& channels, -							  const LLIOPipe::buffer_ptr_t& buffer); +// Subclass for header fetches. +// +// Thread:  repo +class LLMeshHeaderHandler : public LLMeshHandlerBase +{ +public: +	LOG_CLASS(LLMeshHeaderHandler); +	LLMeshHeaderHandler(const LLVolumeParams & mesh_params) +		: LLMeshHandlerBase() +	{ +		mMeshParams = mesh_params; +		LLMeshRepoThread::incActiveHeaderRequests(); +	} +	virtual ~LLMeshHeaderHandler(); +protected: +	LLMeshHeaderHandler(const LLMeshHeaderHandler &);			// Not defined +	void operator=(const LLMeshHeaderHandler &);				// Not defined +	 +public: +	virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); +	virtual void processFailure(LLCore::HttpStatus status);  }; -class LLMeshLODResponder : public LLCurl::Responder + +// Subclass for LOD fetches. +// +// Thread:  repo +class LLMeshLODHandler : public LLMeshHandlerBase  { -	LOG_CLASS(LLMeshLODResponder);  public: -	LLVolumeParams mMeshParams; -	S32 mLOD; -	U32 mRequestedBytes; -	U32 mOffset; -	bool mProcessed; - -	LLMeshLODResponder(const LLVolumeParams& mesh_params, S32 lod, U32 offset, U32 requested_bytes) -		: mMeshParams(mesh_params), mLOD(lod), mOffset(offset), mRequestedBytes(requested_bytes) +	LLMeshLODHandler(const LLVolumeParams & mesh_params, S32 lod, U32 offset, U32 requested_bytes) +		: LLMeshHandlerBase(), +		  mLOD(lod), +		  mRequestedBytes(requested_bytes), +		  mOffset(offset)  	{ +		mMeshParams = mesh_params;  		LLMeshRepoThread::incActiveLODRequests(); -		mProcessed = false; -	} - -	~LLMeshLODResponder() -	{ -		if (!LLApp::isQuitting()) -		{ -			if (!mProcessed) -			{ -				llwarns << "Killed without being processed, retrying." << llendl; -				LLMeshRepository::sHTTPRetryCount++; -				gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD); -			} -			LLMeshRepoThread::decActiveLODRequests(); -		}  	} +	virtual ~LLMeshLODHandler(); +	 +protected: +	LLMeshLODHandler(const LLMeshLODHandler &);					// Not defined +	void operator=(const LLMeshLODHandler &);					// Not defined +	 +public: +	virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); +	virtual void processFailure(LLCore::HttpStatus status); -	virtual void completedRaw(const LLChannelDescriptors& channels, -							  const LLIOPipe::buffer_ptr_t& buffer); - -}; - -class LLMeshSkinInfoResponder : public LLCurl::Responder -{ -	LOG_CLASS(LLMeshSkinInfoResponder);  public: -	LLUUID mMeshID; +	S32 mLOD;  	U32 mRequestedBytes;  	U32 mOffset; -	bool mProcessed; - -	LLMeshSkinInfoResponder(const LLUUID& id, U32 offset, U32 size) -		: mMeshID(id), mRequestedBytes(size), mOffset(offset) -	{ -		mProcessed = false; -	} +}; -	~LLMeshSkinInfoResponder() -	{ -		if (!LLApp::isQuitting() && -			!mProcessed && -			mMeshID.notNull()) -		{	// Something went wrong, retry -			llwarns << "Timeout or service unavailable, retrying loadMeshSkinInfo() for " << mMeshID << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); -		} -	} -	virtual void completedRaw(const LLChannelDescriptors& channels, -							  const LLIOPipe::buffer_ptr_t& buffer); +// Subclass for skin info fetches. +// +// Thread:  repo +class LLMeshSkinInfoHandler : public LLMeshHandlerBase +{ +public: +	LOG_CLASS(LLMeshSkinInfoHandler); +	LLMeshSkinInfoHandler(const LLUUID& id, U32 offset, U32 size) +		: LLMeshHandlerBase(), +		  mMeshID(id), +		  mRequestedBytes(size), +		  mOffset(offset) +	{} +	virtual ~LLMeshSkinInfoHandler(); -}; +protected: +	LLMeshSkinInfoHandler(const LLMeshSkinInfoHandler &);		// Not defined +	void operator=(const LLMeshSkinInfoHandler &);				// Not defined +	 +public: +	virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); +	virtual void processFailure(LLCore::HttpStatus status); -class LLMeshDecompositionResponder : public LLCurl::Responder -{ -	LOG_CLASS(LLMeshDecompositionResponder);  public:  	LLUUID mMeshID;  	U32 mRequestedBytes;  	U32 mOffset; -	bool mProcessed; - -	LLMeshDecompositionResponder(const LLUUID& id, U32 offset, U32 size) -		: mMeshID(id), mRequestedBytes(size), mOffset(offset) -	{ -		mProcessed = false; -	} +}; -	~LLMeshDecompositionResponder() -	{ -		if (!LLApp::isQuitting() && -			!mProcessed && -			mMeshID.notNull()) -		{	// Something went wrong, retry -			llwarns << "Timeout or service unavailable, retrying loadMeshDecomposition() for " << mMeshID << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshDecomposition(mMeshID); -		} -	} -	virtual void completedRaw(const LLChannelDescriptors& channels, -							  const LLIOPipe::buffer_ptr_t& buffer); +// Subclass for decomposition fetches. +// +// Thread:  repo +class LLMeshDecompositionHandler : public LLMeshHandlerBase +{ +public: +	LOG_CLASS(LLMeshDecompositionHandler); +	LLMeshDecompositionHandler(const LLUUID& id, U32 offset, U32 size) +		: LLMeshHandlerBase(), +		  mMeshID(id), +		  mRequestedBytes(size), +		  mOffset(offset) +	{} +	virtual ~LLMeshDecompositionHandler(); -}; +protected: +	LLMeshDecompositionHandler(const LLMeshDecompositionHandler &);		// Not defined +	void operator=(const LLMeshDecompositionHandler &);					// Not defined +	 +public: +	virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); +	virtual void processFailure(LLCore::HttpStatus status); -class LLMeshPhysicsShapeResponder : public LLCurl::Responder -{ -	LOG_CLASS(LLMeshPhysicsShapeResponder);  public:  	LLUUID mMeshID;  	U32 mRequestedBytes;  	U32 mOffset; -	bool mProcessed; +}; -	LLMeshPhysicsShapeResponder(const LLUUID& id, U32 offset, U32 size) -		: mMeshID(id), mRequestedBytes(size), mOffset(offset) -	{ -		mProcessed = false; -	} -	~LLMeshPhysicsShapeResponder() -	{ -		if (!LLApp::isQuitting() && -			!mProcessed && -			mMeshID.notNull()) -		{	// Something went wrong, retry -			llwarns << "Timeout or service unavailable, retrying loadMeshPhysicsShape() for " << mMeshID << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); -		} -	} +// Subclass for physics shape fetches. +// +// Thread:  repo +class LLMeshPhysicsShapeHandler : public LLMeshHandlerBase +{ +public: +	LOG_CLASS(LLMeshPhysicsShapeHandler); +	LLMeshPhysicsShapeHandler(const LLUUID& id, U32 offset, U32 size) +		: LLMeshHandlerBase(), +		  mMeshID(id), +		  mRequestedBytes(size), +		  mOffset(offset) +	{} +	virtual ~LLMeshPhysicsShapeHandler(); -	virtual void completedRaw(const LLChannelDescriptors& channels, -							  const LLIOPipe::buffer_ptr_t& buffer); +protected: +	LLMeshPhysicsShapeHandler(const LLMeshPhysicsShapeHandler &);	// Not defined +	void operator=(const LLMeshPhysicsShapeHandler &);				// Not defined +	 +public: +	virtual void processData(LLCore::BufferArray * body, U8 * data, S32 data_size); +	virtual void processFailure(LLCore::HttpStatus status); +public: +	LLUUID		mMeshID; +	U32			mRequestedBytes; +	U32			mOffset;  }; -void log_upload_error(S32 status, const LLSD& content, std::string stage, std::string model_name) + +void log_upload_error(LLCore::HttpStatus status, const LLSD& content, +					  const char * const stage, const std::string & model_name)  {  	// Add notification popup.  	LLSD args; -	std::string message = content["error"]["message"]; -	std::string identifier = content["error"]["identifier"]; +	std::string message = content["error"]["message"].asString(); +	std::string identifier = content["error"]["identifier"].asString();  	args["MESSAGE"] = message;  	args["IDENTIFIER"] = identifier;  	args["LABEL"] = model_name;  	gMeshRepo.uploadError(args);  	// Log details. -	llwarns << "stage: " << stage << " http status: " << status << llendl; +	LL_WARNS(LOG_MESH) << "Error in stage:  " << stage +					   << ", Reason:  " << status.toString() +					   << " (" << status.toTerseString() << ")" << LL_ENDL;  	if (content.has("error"))  	{  		const LLSD& err = content["error"]; -		llwarns << "err: " << err << llendl; -		llwarns << "mesh upload failed, stage '" << stage -				<< "' error '" << err["error"].asString() -				<< "', message '" << err["message"].asString() -				<< "', id '" << err["identifier"].asString() -				<< "'" << llendl; +		LL_WARNS(LOG_MESH) << "error: " << err << LL_ENDL; +		LL_WARNS(LOG_MESH) << "  mesh upload failed, stage '" << stage +						   << "', error '" << err["error"].asString() +						   << "', message '" << err["message"].asString() +						   << "', id '" << err["identifier"].asString() +						   << "'" << LL_ENDL;  		if (err.has("errors"))  		{  			S32 error_num = 0; @@ -400,13 +734,13 @@ void log_upload_error(S32 status, const LLSD& content, std::string stage, std::s  				 ++it)  			{  				const LLSD& err_entry = *it; -				llwarns << "error[" << error_num << "]:" << llendl; +				LL_WARNS(LOG_MESH) << "  error[" << error_num << "]:" << LL_ENDL;  				for (LLSD::map_const_iterator map_it = err_entry.beginMap();  					 map_it != err_entry.endMap();  					 ++map_it)  				{ -					llwarns << "\t" << map_it->first << ": " -							<< map_it->second << llendl; +					LL_WARNS(LOG_MESH) << "    " << map_it->first << ":  " +									   << map_it->second << LL_ENDL;  				}  				error_num++;  			} @@ -414,154 +748,71 @@ void log_upload_error(S32 status, const LLSD& content, std::string stage, std::s  	}  	else  	{ -		llwarns << "bad mesh, no error information available" << llendl; +		LL_WARNS(LOG_MESH) << "Bad response to mesh request, no additional error information available." << LL_ENDL;  	}  } -class LLWholeModelFeeResponder: public LLCurl::Responder +LLMeshRepoThread::LLMeshRepoThread() +: LLThread("mesh repo"), +  mHttpRequest(NULL), +  mHttpOptions(NULL), +  mHttpLargeOptions(NULL), +  mHttpHeaders(NULL), +  mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), +  mHttpLegacyPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), +  mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), +  mHttpPriority(0), +  mGetMeshVersion(2) +{  +	mMutex = new LLMutex(NULL); +	mHeaderMutex = new LLMutex(NULL); +	mSignal = new LLCondition(NULL); +	mHttpRequest = new LLCore::HttpRequest; +	mHttpOptions = new LLCore::HttpOptions; +	mHttpOptions->setTransferTimeout(SMALL_MESH_XFER_TIMEOUT); +	mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); +	mHttpLargeOptions = new LLCore::HttpOptions; +	mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); +	mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); +	mHttpHeaders = new LLCore::HttpHeaders; +	mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); +	mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); +	mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); +	mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); +} + + +LLMeshRepoThread::~LLMeshRepoThread()  { -	LOG_CLASS(LLWholeModelFeeResponder); -	LLMeshUploadThread* mThread; -	LLSD mModelData; -	LLHandle<LLWholeModelFeeObserver> mObserverHandle; -public: -	LLWholeModelFeeResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelFeeObserver> observer_handle): -		mThread(thread), -		mModelData(model_data), -		mObserverHandle(observer_handle) -	{ -		if (mThread) -		{ -			mThread->startRequest(); -		} -	} +	LL_INFOS(LOG_MESH) << "Small GETs issued:  " << LLMeshRepository::sHTTPRequestCount +					   << ", Large GETs issued:  " << LLMeshRepository::sHTTPLargeRequestCount +					   << ", Max Lock Holdoffs:  " << LLMeshRepository::sMaxLockHoldoffs +					   << LL_ENDL; -	~LLWholeModelFeeResponder() +	for (http_request_set::iterator iter(mHttpRequestSet.begin()); +		 iter != mHttpRequestSet.end(); +		 ++iter)  	{ -		if (mThread) -		{ -			mThread->stopRequest(); -		} +		delete *iter;  	} - -protected: -	virtual void httpCompleted() +	mHttpRequestSet.clear(); +	if (mHttpHeaders)  	{ -		LLSD cc = getContent(); -		if (gSavedSettings.getS32("MeshUploadFakeErrors")&1) -		{ -			cc = llsd_from_file("fake_upload_error.xml"); -		} - -		dump_llsd_to_file(cc,make_dump_name("whole_model_fee_response_",dump_num)); - -		LLWholeModelFeeObserver* observer = mObserverHandle.get(); - -		if (isGoodStatus() && -			cc["state"].asString() == "upload") -		{ -			mThread->mWholeModelUploadURL = cc["uploader"].asString(); - -			if (observer) -			{ -				cc["data"]["upload_price"] = cc["upload_price"]; -				observer->onModelPhysicsFeeReceived(cc["data"], mThread->mWholeModelUploadURL); -			} -		} -		else -		{ -			llwarns << "fee request failed " << dumpResponse() << llendl; -			S32 status = getStatus(); -			log_upload_error(status,cc,"fee",mModelData["name"]); -			mThread->mWholeModelUploadURL = ""; - -			if (observer) -			{ -				observer->setModelPhysicsFeeErrorStatus(status, getReason()); -			} -		} +		mHttpHeaders->release(); +		mHttpHeaders = NULL;  	} - -}; - -class LLWholeModelUploadResponder: public LLCurl::Responder -{ -	LOG_CLASS(LLWholeModelUploadResponder); -	LLMeshUploadThread* mThread; -	LLSD mModelData; -	LLHandle<LLWholeModelUploadObserver> mObserverHandle; -	 -public: -	LLWholeModelUploadResponder(LLMeshUploadThread* thread, LLSD& model_data, LLHandle<LLWholeModelUploadObserver> observer_handle): -		mThread(thread), -		mModelData(model_data), -		mObserverHandle(observer_handle) +	if (mHttpOptions)  	{ -		if (mThread) -		{ -			mThread->startRequest(); -		} +		mHttpOptions->release(); +		mHttpOptions = NULL;  	} - -	~LLWholeModelUploadResponder() +	if (mHttpLargeOptions)  	{ -		if (mThread) -		{ -			mThread->stopRequest(); -		} -	} - -protected: -	virtual void httpCompleted() -	{ -		LLSD cc = getContent(); -		if (gSavedSettings.getS32("MeshUploadFakeErrors")&2) -		{ -			cc = llsd_from_file("fake_upload_error.xml"); -		} - -		dump_llsd_to_file(cc,make_dump_name("whole_model_upload_response_",dump_num)); -		 -		LLWholeModelUploadObserver* observer = mObserverHandle.get(); - -		// requested "mesh" asset type isn't actually the type -		// of the resultant object, fix it up here. -		if (isGoodStatus() && -			cc["state"].asString() == "complete") -		{ -			mModelData["asset_type"] = "object"; -			gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData,cc)); - -			if (observer) -			{ -				doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); -			} -		} -		else -		{ -			llwarns << "upload failed " << dumpResponse() << llendl; -			std::string model_name = mModelData["name"].asString(); -			log_upload_error(getStatus(),cc,"upload",model_name); - -			if (observer) -			{ -				doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); -			} -		} +		mHttpLargeOptions->release(); +		mHttpLargeOptions = NULL;  	} -}; - -LLMeshRepoThread::LLMeshRepoThread() -: LLThread("mesh repo")  -{  -	mWaiting = false; -	mMutex = new LLMutex(NULL); -	mHeaderMutex = new LLMutex(NULL); -	mSignal = new LLCondition(NULL); -} - -LLMeshRepoThread::~LLMeshRepoThread() -{ +	delete mHttpRequest; +	mHttpRequest = NULL;  	delete mMutex;  	mMutex = NULL;  	delete mHeaderMutex; @@ -572,109 +823,180 @@ LLMeshRepoThread::~LLMeshRepoThread()  void LLMeshRepoThread::run()  { -	mCurlRequest = new LLCurlRequest();  	LLCDResult res = LLConvexDecomposition::initThread();  	if (res != LLCD_OK)  	{ -		llwarns << "convex decomposition unable to be loaded" << llendl; +		LL_WARNS(LOG_MESH) << "Convex decomposition unable to be loaded.  Expect severe problems." << LL_ENDL;  	}  	while (!LLApp::isQuitting())  	{ -		mWaiting = true; +		// *TODO:  Revise sleep/wake strategy and try to move away +		// from polling operations in this thread.  We can sleep +		// this thread hard when: +		// * All Http requests are serviced +		// * LOD request queue empty +		// * Header request queue empty +		// * Skin info request queue empty +		// * Decomposition request queue empty +		// * Physics shape request queue empty +		// We wake the thread when any of the above become untrue. +		// Will likely need a correctly-implemented condition variable to do this. +		// On the other hand, this may actually be an effective and efficient scheme... +		  		mSignal->wait(); -		mWaiting = false; -		if (!LLApp::isQuitting()) +		if (LLApp::isQuitting())  		{ -			static U32 count = 0; +			break; +		} +		 +		if (! mHttpRequestSet.empty()) +		{ +			// Dispatch all HttpHandler notifications +			mHttpRequest->update(0L); +		} +		sRequestWaterLevel = mHttpRequestSet.size();			// Stats data update +			 +		// NOTE: order of queue processing intentionally favors LOD requests over header requests -			static F32 last_hundred = gFrameTimeSeconds; +		while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) +		{ +			if (! mMutex) +			{ +				break; +			} +			mMutex->lock(); +			LODRequest req = mLODReqQ.front(); +			mLODReqQ.pop(); +			LLMeshRepository::sLODProcessing--; +			mMutex->unlock(); +			if (!fetchMeshLOD(req.mMeshParams, req.mLOD))		// failed, resubmit +			{ +				mMutex->lock(); +				mLODReqQ.push(req) ;  +				++LLMeshRepository::sLODProcessing; +				mMutex->unlock(); +			} +		} -			if (gFrameTimeSeconds - last_hundred > 1.f) -			{ //a second has gone by, clear count -				last_hundred = gFrameTimeSeconds; -				count = 0;	 +		while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) +		{ +			if (! mMutex) +			{ +				break; +			} +			mMutex->lock(); +			HeaderRequest req = mHeaderReqQ.front(); +			mHeaderReqQ.pop(); +			mMutex->unlock(); +			if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit +			{ +				mMutex->lock(); +				mHeaderReqQ.push(req) ; +				mMutex->unlock();  			} +		} -			// NOTE: throttling intentionally favors LOD requests over header requests -			 -			while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) +		// For the final three request lists, similar goal to above but +		// slightly different queue structures.  Stay off the mutex when +		// performing long-duration actions. + +		if (mHttpRequestSet.size() < sRequestHighWater +			&& (! mSkinRequests.empty() +				|| ! mDecompositionRequests.empty() +				|| ! mPhysicsShapeRequests.empty())) +		{ +			// Something to do probably, lock and double-check.  We don't want +			// to hold the lock long here.  That will stall main thread activities +			// so we bounce it. + +			mMutex->lock(); +			if (! mSkinRequests.empty() && mHttpRequestSet.size() < sRequestHighWater)  			{ -				if (mMutex) +				std::set<LLUUID> incomplete; +				std::set<LLUUID>::iterator iter(mSkinRequests.begin()); +				while (iter != mSkinRequests.end() && mHttpRequestSet.size() < sRequestHighWater)  				{ -					mMutex->lock(); -					LODRequest req = mLODReqQ.front(); -					mLODReqQ.pop(); -					LLMeshRepository::sLODProcessing--; +					LLUUID mesh_id = *iter; +					mSkinRequests.erase(iter);  					mMutex->unlock(); -					if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit + +					if (! fetchMeshSkinInfo(mesh_id))  					{ -						mMutex->lock(); -						mLODReqQ.push(req);  -						mMutex->unlock(); +						incomplete.insert(mesh_id);  					} + +					mMutex->lock(); +					iter = mSkinRequests.begin();  				} -			} -			while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) -			{ -				if (mMutex) +				if (! incomplete.empty())  				{ -					mMutex->lock(); -					HeaderRequest req = mHeaderReqQ.front(); -					mHeaderReqQ.pop(); -					mMutex->unlock(); -					if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit -					{ -						mMutex->lock(); -						mHeaderReqQ.push(req) ; -						mMutex->unlock(); -					} +					mSkinRequests.insert(incomplete.begin(), incomplete.end());  				}  			} -			{ //mSkinRequests is protected by mSignal +			// holding lock, try next list +			// *TODO:  For UI/debug-oriented lists, we might drop the fine- +			// grained locking as there's a lowered expectation of smoothness +			// in these cases. +			if (! mDecompositionRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) +			{  				std::set<LLUUID> incomplete; -				for (std::set<LLUUID>::iterator iter = mSkinRequests.begin(); iter != mSkinRequests.end(); ++iter) +				std::set<LLUUID>::iterator iter(mDecompositionRequests.begin()); +				while (iter != mDecompositionRequests.end() && mHttpRequestSet.size() < sRequestHighWater)  				{  					LLUUID mesh_id = *iter; -					if (!fetchMeshSkinInfo(mesh_id)) +					mDecompositionRequests.erase(iter); +					mMutex->unlock(); +					 +					if (! fetchMeshDecomposition(mesh_id))  					{  						incomplete.insert(mesh_id);  					} + +					mMutex->lock(); +					iter = mDecompositionRequests.begin();  				} -				mSkinRequests = incomplete; -			} -			{ //mDecompositionRequests is protected by mSignal -				std::set<LLUUID> incomplete; -				for (std::set<LLUUID>::iterator iter = mDecompositionRequests.begin(); iter != mDecompositionRequests.end(); ++iter) +				if (! incomplete.empty())  				{ -					LLUUID mesh_id = *iter; -					if (!fetchMeshDecomposition(mesh_id)) -					{ -						incomplete.insert(mesh_id); -					} +					mDecompositionRequests.insert(incomplete.begin(), incomplete.end());  				} -				mDecompositionRequests = incomplete;  			} -			{ //mPhysicsShapeRequests is protected by mSignal +			// holding lock, final list +			if (! mPhysicsShapeRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) +			{  				std::set<LLUUID> incomplete; -				for (std::set<LLUUID>::iterator iter = mPhysicsShapeRequests.begin(); iter != mPhysicsShapeRequests.end(); ++iter) +				std::set<LLUUID>::iterator iter(mPhysicsShapeRequests.begin()); +				while (iter != mPhysicsShapeRequests.end() && mHttpRequestSet.size() < sRequestHighWater)  				{  					LLUUID mesh_id = *iter; -					if (!fetchMeshPhysicsShape(mesh_id)) +					mPhysicsShapeRequests.erase(iter); +					mMutex->unlock(); +					 +					if (! fetchMeshPhysicsShape(mesh_id))  					{  						incomplete.insert(mesh_id);  					} + +					mMutex->lock(); +					iter = mPhysicsShapeRequests.begin();  				} -				mPhysicsShapeRequests = incomplete; -			} -			mCurlRequest->process(); +				if (! incomplete.empty()) +				{ +					mPhysicsShapeRequests.insert(incomplete.begin(), incomplete.end()); +				} +			} +			mMutex->unlock();  		} + +		// For dev purposes only.  A dynamic change could make this false +		// and that shouldn't assert. +		// llassert_always(mHttpRequestSet.size() <= sRequestHighWater);  	}  	if (mSignal->isLocked()) @@ -685,25 +1007,25 @@ void LLMeshRepoThread::run()  	res = LLConvexDecomposition::quitThread();  	if (res != LLCD_OK)  	{ -		llwarns << "convex decomposition unable to be quit" << llendl; +		LL_WARNS(LOG_MESH) << "Convex decomposition unable to be quit." << LL_ENDL;  	} - -	delete mCurlRequest; -	mCurlRequest = NULL;  } +// Mutex:  LLMeshRepoThread::mMutex must be held on entry  void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{  	mSkinRequests.insert(mesh_id);  } +// Mutex:  LLMeshRepoThread::mMutex must be held on entry  void LLMeshRepoThread::loadMeshDecomposition(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{  	mDecompositionRequests.insert(mesh_id);  } +// Mutex:  LLMeshRepoThread::mMutex must be held on entry  void LLMeshRepoThread::loadMeshPhysicsShape(const LLUUID& mesh_id) -{ //protected by mSignal, no locking needed here +{  	mPhysicsShapeRequests.insert(mesh_id);  } @@ -716,7 +1038,6 @@ void LLMeshRepoThread::lockAndLoadMeshLOD(const LLVolumeParams& mesh_params, S32  } -  void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod)  { //could be called from any thread  	LLMutexLock lock(mMutex); @@ -748,31 +1069,122 @@ void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod)  	}  } -//static  -std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) +// Mutex:  must be holding mMutex when called +void LLMeshRepoThread::setGetMeshCaps(const std::string & get_mesh1, +									  const std::string & get_mesh2, +									  int pref_version)  { -	std::string http_url; +	mGetMeshCapability = get_mesh1; +	mGetMesh2Capability = get_mesh2; +	mGetMeshVersion = pref_version; +} + + +// Constructs a Cap URL for the mesh.  Prefers a GetMesh2 cap +// over a GetMesh cap. +// +// Mutex:  acquires mMutex +void LLMeshRepoThread::constructUrl(LLUUID mesh_id, std::string * url, int * version) +{ +	std::string res_url; +	int res_version(2);  	if (gAgent.getRegion())  	{ -		http_url = gMeshRepo.mGetMeshCapability;  +		LLMutexLock lock(mMutex); + +		// Get a consistent pair of (cap string, version).  The +		// locking could be eliminated here without loss of safety +		// by using a set of staging values in setGetMeshCaps(). +		 +		if (! mGetMesh2Capability.empty() && mGetMeshVersion > 1) +		{ +			res_url = mGetMesh2Capability; +			res_version = 2; +		} +		else +		{ +			res_url = mGetMeshCapability; +			res_version = 1; +		}  	} -	if (!http_url.empty()) +	if (! res_url.empty())  	{ -		http_url += "/?mesh_id="; -		http_url += mesh_id.asString().c_str(); +		res_url += "/?mesh_id="; +		res_url += mesh_id.asString().c_str();  	}  	else  	{ -		llwarns << "Current region does not have GetMesh capability!  Cannot load " << mesh_id << ".mesh" << llendl; +		LL_WARNS_ONCE(LOG_MESH) << "Current region does not have GetMesh capability!  Cannot load " +								<< mesh_id << ".mesh" << LL_ENDL;  	} -	return http_url; +	*url = res_url; +	*version = res_version;  } +// Issue an HTTP GET request with byte range using the right +// policy class.  Large requests go to the large request class. +// If the current region supports GetMesh2, we prefer that for +// smaller requests otherwise we try to use the traditional +// GetMesh capability and connection concurrency. +// +// @return		Valid handle or LLCORE_HTTP_HANDLE_INVALID. +//				If the latter, actual status is found in +//				mHttpStatus member which is valid until the +//				next call to this method. +// +// Thread:  repo +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int cap_version, +												  size_t offset, size_t len, +												  LLCore::HttpHandler * handler) +{ +	LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); +	 +	if (len < LARGE_MESH_FETCH_THRESHOLD) +	{ +		handle = mHttpRequest->requestGetByteRange((2 == cap_version +													? mHttpPolicyClass +													: mHttpLegacyPolicyClass), +												   mHttpPriority, +												   url, +												   offset, +												   len, +												   mHttpOptions, +												   mHttpHeaders, +												   handler); +		if (LLCORE_HTTP_HANDLE_INVALID != handle) +		{ +			++LLMeshRepository::sHTTPRequestCount; +		} +	} +	else +	{ +		handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, +												   mHttpPriority, +												   url, +												   offset, +												   len, +												   mHttpLargeOptions, +												   mHttpHeaders, +												   handler); +		if (LLCORE_HTTP_HANDLE_INVALID != handle) +		{ +			++LLMeshRepository::sHTTPLargeRequestCount; +		} +	} +	if (LLCORE_HTTP_HANDLE_INVALID == handle) +	{ +		// Something went wrong, capture the error code for caller. +		mHttpStatus = mHttpRequest->getStatus(); +	} +	return handle; +} + +  bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) -{ //protected by mMutex +{  	if (!mHeaderMutex)  	{ @@ -787,7 +1199,8 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)  		return false;  	} -	bool ret = true ; +	++LLMeshRepository::sMeshRequestCount; +	bool ret = true;  	U32 header_size = mMeshHeaderSize[mesh_id];  	if (header_size > 0) @@ -805,6 +1218,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)  			if (file.getSize() >= offset+size)  			{				  				LLMeshRepository::sCacheBytesRead += size; +				++LLMeshRepository::sCacheReads;  				file.seek(offset);  				U8* buffer = new U8[size];  				file.read(buffer, size); @@ -829,17 +1243,28 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)  			}  			//reading from VFS failed for whatever reason, fetch from sim -			std::vector<std::string> headers; -			headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); +			int cap_version(2); +			std::string http_url; +			constructUrl(mesh_id, &http_url, &cap_version); -			std::string http_url = constructUrl(mesh_id);  			if (!http_url.empty()) -			{				 -				ret = mCurlRequest->getByteRange(http_url, headers, offset, size, -												 new LLMeshSkinInfoResponder(mesh_id, offset, size)); -				if(ret) +			{ +				LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); +				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); +				if (LLCORE_HTTP_HANDLE_INVALID == handle) +				{ +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for skin info on mesh " << mID +									   << ".  Reason:  " << mHttpStatus.toString() +									   << " (" << mHttpStatus.toTerseString() << ")" +									   << LL_ENDL; +					delete handler; +					ret = false; + +				} +				else  				{ -					LLMeshRepository::sHTTPRequestCount++; +					handler->mHttpHandle = handle; +					mHttpRequestSet.insert(handler);  				}  			}  		} @@ -854,7 +1279,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id)  }  bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) -{ //protected by mMutex +{  	if (!mHeaderMutex)  	{  		return false; @@ -868,8 +1293,9 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)  		return false;  	} +	++LLMeshRepository::sMeshRequestCount;  	U32 header_size = mMeshHeaderSize[mesh_id]; -	bool ret = true ; +	bool ret = true;  	if (header_size > 0)  	{ @@ -886,6 +1312,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)  			if (file.getSize() >= offset+size)  			{  				LLMeshRepository::sCacheBytesRead += size; +				++LLMeshRepository::sCacheReads;  				file.seek(offset);  				U8* buffer = new U8[size]; @@ -911,17 +1338,27 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)  			}  			//reading from VFS failed for whatever reason, fetch from sim -			std::vector<std::string> headers; -			headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); - -			std::string http_url = constructUrl(mesh_id); +			int cap_version(2); +			std::string http_url; +			constructUrl(mesh_id, &http_url, &cap_version); +			  			if (!http_url.empty()) -			{				 -				ret = mCurlRequest->getByteRange(http_url, headers, offset, size, -												 new LLMeshDecompositionResponder(mesh_id, offset, size)); -				if(ret) +			{ +				LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); +				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); +				if (LLCORE_HTTP_HANDLE_INVALID == handle) +				{ +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for decomposition mesh " << mID +									   << ".  Reason:  " << mHttpStatus.toString() +									   << " (" << mHttpStatus.toTerseString() << ")" +									   << LL_ENDL; +					delete handler; +					ret = false; +				} +				else  				{ -					LLMeshRepository::sHTTPRequestCount++; +					handler->mHttpHandle = handle; +					mHttpRequestSet.insert(handler);  				}  			}  		} @@ -936,7 +1373,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id)  }  bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) -{ //protected by mMutex +{  	if (!mHeaderMutex)  	{  		return false; @@ -950,8 +1387,9 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)  		return false;  	} +	++LLMeshRepository::sMeshRequestCount;  	U32 header_size = mMeshHeaderSize[mesh_id]; -	bool ret = true ; +	bool ret = true;  	if (header_size > 0)  	{ @@ -968,6 +1406,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)  			if (file.getSize() >= offset+size)  			{  				LLMeshRepository::sCacheBytesRead += size; +				++LLMeshRepository::sCacheReads;  				file.seek(offset);  				U8* buffer = new U8[size];  				file.read(buffer, size); @@ -992,18 +1431,27 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id)  			}  			//reading from VFS failed for whatever reason, fetch from sim -			std::vector<std::string> headers; -			headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); - -			std::string http_url = constructUrl(mesh_id); +			int cap_version(2); +			std::string http_url; +			constructUrl(mesh_id, &http_url, &cap_version); +			  			if (!http_url.empty()) -			{				 -				ret = mCurlRequest->getByteRange(http_url, headers, offset, size, -												 new LLMeshPhysicsShapeResponder(mesh_id, offset, size)); - -				if(ret) +			{ +				LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); +				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); +				if (LLCORE_HTTP_HANDLE_INVALID == handle) +				{ +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for physics shape on mesh " << mID +									   << ".  Reason:  " << mHttpStatus.toString() +									   << " (" << mHttpStatus.toTerseString() << ")" +									   << LL_ENDL; +					delete handler; +					ret = false; +				} +				else  				{ -					LLMeshRepository::sHTTPRequestCount++; +					handler->mHttpHandle = handle; +					mHttpRequestSet.insert(handler);  				}  			}  		} @@ -1050,8 +1498,10 @@ void LLMeshRepoThread::decActiveHeaderRequests()  }  //return false if failed to get header -bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count) +bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params)  { +	++LLMeshRepository::sMeshRequestCount; +  	{  		//look for mesh in asset in vfs  		LLVFile file(gVFS, mesh_params.getSculptID(), LLAssetType::AT_MESH); @@ -1059,43 +1509,57 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c  		S32 size = file.getSize();  		if (size > 0) -		{ //NOTE -- if the header size is ever more than 4KB, this will break -			U8 buffer[4096]; -			S32 bytes = llmin(size, 4096); +		{ +			// *NOTE:  if the header size is ever more than 4KB, this will break +			U8 buffer[MESH_HEADER_SIZE]; +			S32 bytes = llmin(size, MESH_HEADER_SIZE);  			LLMeshRepository::sCacheBytesRead += bytes;	 +			++LLMeshRepository::sCacheReads;  			file.read(buffer, bytes);  			if (headerReceived(mesh_params, buffer, bytes)) -			{ //did not do an HTTP request, return false +			{ +				// Found mesh in VFS cache  				return true;  			}  		}  	}  	//either cache entry doesn't exist or is corrupt, request header from simulator	 -	bool retval = true ; -	std::vector<std::string> headers; -	headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); - -	std::string http_url = constructUrl(mesh_params.getSculptID()); +	bool retval = true; +	int cap_version(2); +	std::string http_url; +	constructUrl(mesh_params.getSculptID(), &http_url, &cap_version); +	  	if (!http_url.empty())  	{  		//grab first 4KB if we're going to bother with a fetch.  Cache will prevent future fetches if a full mesh fits  		//within the first 4KB  		//NOTE -- this will break of headers ever exceed 4KB		 -		retval = mCurlRequest->getByteRange(http_url, headers, 0, 4096, new LLMeshHeaderResponder(mesh_params)); -		if(retval) + +		LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); +		LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); +		if (LLCORE_HTTP_HANDLE_INVALID == handle) +		{ +			LL_WARNS(LOG_MESH) << "HTTP GET request failed for mesh header " << mID +							   << ".  Reason:  " << mHttpStatus.toString() +							   << " (" << mHttpStatus.toTerseString() << ")" +							   << LL_ENDL; +			delete handler; +			retval = false; +		} +		else  		{ -			LLMeshRepository::sHTTPRequestCount++; +			handler->mHttpHandle = handle; +			mHttpRequestSet.insert(handler);  		} -		count++;  	}  	return retval;  }  //return false if failed to get mesh lod. -bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) -{ //protected by mMutex +bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod) +{  	if (!mHeaderMutex)  	{  		return false; @@ -1103,6 +1567,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod,  	mHeaderMutex->lock(); +	++LLMeshRepository::sMeshRequestCount;  	bool retval = true;  	LLUUID mesh_id = mesh_params.getSculptID(); @@ -1124,6 +1589,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod,  			if (file.getSize() >= offset+size)  			{  				LLMeshRepository::sCacheBytesRead += size; +				++LLMeshRepository::sCacheReads;  				file.seek(offset);  				U8* buffer = new U8[size];  				file.read(buffer, size); @@ -1148,20 +1614,29 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod,  			}  			//reading from VFS failed for whatever reason, fetch from sim -			std::vector<std::string> headers; -			headers.push_back(HTTP_OUT_HEADER_ACCEPT + ": " + HTTP_CONTENT_OCTET_STREAM); - -			std::string http_url = constructUrl(mesh_id); +			int cap_version(2); +			std::string http_url; +			constructUrl(mesh_id, &http_url, &cap_version); +			  			if (!http_url.empty()) -			{				 -				retval = mCurlRequest->getByteRange(constructUrl(mesh_id), headers, offset, size, -										   new LLMeshLODResponder(mesh_params, lod, offset, size)); - -				if(retval) +			{ +				LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); +				LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); +				if (LLCORE_HTTP_HANDLE_INVALID == handle) +				{ +					LL_WARNS(LOG_MESH) << "HTTP GET request failed for LOD on mesh " << mID +									   << ".  Reason:  " << mHttpStatus.toString() +									   << " (" << mHttpStatus.toTerseString() << ")" +									   << LL_ENDL; +					delete handler; +					retval = false; +				} +				else  				{ -					LLMeshRepository::sHTTPRequestCount++; +					handler->mHttpHandle = handle; +					mHttpRequestSet.insert(handler); +					// *NOTE:  Allowing a re-request, not marking as unavailable.  Is that correct?  				} -				count++;  			}  			else  			{ @@ -1183,6 +1658,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod,  bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size)  { +	const LLUUID mesh_id = mesh_params.getSculptID();  	LLSD header;  	U32 header_size = 0; @@ -1203,7 +1679,8 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat  		if (!LLSDSerialize::fromBinary(header, stream, data_size))  		{ -			llwarns << "Mesh header parse error.  Not a valid mesh asset!" << llendl; +			LL_WARNS(LOG_MESH) << "Mesh header parse error.  Not a valid mesh asset!  ID:  " << mesh_id +							   << LL_ENDL;  			return false;  		} @@ -1211,13 +1688,12 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat  	}  	else  	{ -		llinfos -			<< "Marking header as non-existent, will not retry." << llendl; +		LL_INFOS(LOG_MESH) << "Non-positive data size.  Marking header as non-existent, will not retry.  ID:  " << mesh_id +						   << LL_ENDL;  		header["404"] = 1;  	}  	{ -		LLUUID mesh_id = mesh_params.getSculptID();  		{  			LLMutexLock lock(mHeaderMutex); @@ -1225,6 +1701,7 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat  			mMeshHeader[mesh_id] = header;  		} +		  		LLMutexLock lock(mMutex); // make sure only one thread access mPendingLOD at the same time.  		//check for pending requests @@ -1278,7 +1755,8 @@ bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 dat  		if (!unzip_llsd(skin, stream, data_size))  		{ -			llwarns << "Mesh skin info parse error.  Not a valid mesh asset!" << llendl; +			LL_WARNS(LOG_MESH) << "Mesh skin info parse error.  Not a valid mesh asset!  ID:  " << mesh_id +							   << LL_ENDL;  			return false;  		}  	} @@ -1287,8 +1765,11 @@ bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 dat  		LLMeshSkinInfo info(skin);  		info.mMeshID = mesh_id; -		//llinfos<<"info pelvis offset"<<info.mPelvisOffset<<llendl; -		mSkinInfoQ.push(info); +		// LL_DEBUGS(LOG_MESH) << "info pelvis offset" << info.mPelvisOffset << LL_ENDL; +		{ +			LLMutexLock lock(mMutex); +			mSkinInfoQ.push_back(info); +		}  	}  	return true; @@ -1306,7 +1787,8 @@ bool LLMeshRepoThread::decompositionReceived(const LLUUID& mesh_id, U8* data, S3  		if (!unzip_llsd(decomp, stream, data_size))  		{ -			llwarns << "Mesh decomposition parse error.  Not a valid mesh asset!" << llendl; +			LL_WARNS(LOG_MESH) << "Mesh decomposition parse error.  Not a valid mesh asset!  ID:  " << mesh_id +							   << LL_ENDL;  			return false;  		}  	} @@ -1314,7 +1796,10 @@ bool LLMeshRepoThread::decompositionReceived(const LLUUID& mesh_id, U8* data, S3  	{  		LLModel::Decomposition* d = new LLModel::Decomposition(decomp);  		d->mMeshID = mesh_id; -		mDecompositionQ.push(d); +		{ +			LLMutexLock lock(mMutex); +			mDecompositionQ.push_back(d); +		}  	}  	return true; @@ -1373,15 +1858,20 @@ bool LLMeshRepoThread::physicsShapeReceived(const LLUUID& mesh_id, U8* data, S32  		}  	} -	mDecompositionQ.push(d); +	{ +		LLMutexLock lock(mMutex); +		mDecompositionQ.push_back(d); +	}  	return true;  }  LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, LLVector3& scale, bool upload_textures, -										bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload, -					   LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer) -: LLThread("mesh upload"), -	mDiscarded(FALSE), +									   bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload, +									   LLHandle<LLWholeModelFeeObserver> fee_observer, +									   LLHandle<LLWholeModelUploadObserver> upload_observer) +  : LLThread("mesh upload"), +	LLCore::HttpHandler(), +	mDiscarded(false),  	mDoUpload(do_upload),  	mWholeModelUploadURL(upload_url),  	mFeeObserverHandle(fee_observer), @@ -1392,7 +1882,6 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data,  	mUploadSkin = upload_skin;  	mUploadJoints = upload_joints;  	mMutex = new LLMutex(NULL); -	mCurlRequest = NULL;  	mPendingUploads = 0;  	mFinished = false;  	mOrigin = gAgent.getPositionAgent(); @@ -1402,12 +1891,33 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data,  	mOrigin += gAgent.getAtAxis() * scale.magVec(); -	mMeshUploadTimeOut = gSavedSettings.getS32("MeshUploadTimeOut") ; +	mMeshUploadTimeOut = gSavedSettings.getS32("MeshUploadTimeOut"); + +	mHttpRequest = new LLCore::HttpRequest; +	mHttpOptions = new LLCore::HttpOptions; +	mHttpOptions->setTransferTimeout(mMeshUploadTimeOut); +	mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); +	mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); +	mHttpHeaders = new LLCore::HttpHeaders; +	mHttpHeaders->append("Content-Type", "application/llsd+xml"); +	mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); +	mHttpPriority = 0;  }  LLMeshUploadThread::~LLMeshUploadThread()  { - +	if (mHttpHeaders) +	{ +		mHttpHeaders->release(); +		mHttpHeaders = NULL; +	} +	if (mHttpOptions) +	{ +		mHttpOptions->release(); +		mHttpOptions = NULL; +	} +	delete mHttpRequest; +	mHttpRequest = NULL;  }  LLMeshUploadThread::DecompRequest::DecompRequest(LLModel* mdl, LLModel* base_model, LLMeshUploadThread* thread) @@ -1449,14 +1959,14 @@ void LLMeshUploadThread::preStart()  void LLMeshUploadThread::discard()  { -	LLMutexLock lock(mMutex) ; -	mDiscarded = TRUE ; +	LLMutexLock lock(mMutex); +	mDiscarded = true;  } -BOOL LLMeshUploadThread::isDiscarded() +bool LLMeshUploadThread::isDiscarded() const  { -	LLMutexLock lock(mMutex) ; -	return mDiscarded ; +	LLMutexLock lock(mMutex); +	return mDiscarded;  }  void LLMeshUploadThread::run() @@ -1707,9 +2217,15 @@ void LLMeshUploadThread::generateHulls()  		}  	} -	if(has_valid_requests) -	{ -		while (!mPhysicsComplete) +	if (has_valid_requests) +	{ +		// *NOTE:  Interesting livelock condition on shutdown.  If there +		// is an upload request in generateHulls() when shutdown starts, +		// the main thread isn't available to manage communication between +		// the decomposition thread and the upload thread and this loop +		// wouldn't complete in turn stalling the main thread.  The check +		// on isDiscarded() prevents that. +		while (! mPhysicsComplete && ! isDiscarded())  		{  			apr_sleep(100);  		} @@ -1718,86 +2234,266 @@ void LLMeshUploadThread::generateHulls()  void LLMeshUploadThread::doWholeModelUpload()  { -	mCurlRequest = new LLCurlRequest(); +	LL_DEBUGS(LOG_MESH) << "Starting model upload.  Instances:  " << mInstance.size() << LL_ENDL;  	if (mWholeModelUploadURL.empty())  	{ -		llinfos << "unable to upload, fee request failed" << llendl; +		LL_WARNS(LOG_MESH) << "Missing mesh upload capability, unable to upload, fee request failed." +						   << LL_ENDL;  	}  	else  	{  		generateHulls(); - -		LLSD full_model_data; -		wholeModelToLLSD(full_model_data, true); -		LLSD body = full_model_data["asset_resources"]; -		dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); -		LLCurlRequest::headers_t headers; - +		LL_DEBUGS(LOG_MESH) << "Hull generation completed." << LL_ENDL; + +		mModelData = LLSD::emptyMap(); +		wholeModelToLLSD(mModelData, true); +		LLSD body = mModelData["asset_resources"]; +		dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + +		LLCore::BufferArray * ba = new LLCore::BufferArray; +		LLCore::BufferArrayStream bas(ba); +		LLSDSerialize::toXML(body, bas); +		// LLSDSerialize::toXML(mModelData, bas);		// <- Enabling this will generate a convenient upload error +		LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, +															  mHttpPriority, +															  mWholeModelUploadURL, +															  ba, +															  mHttpOptions, +															  mHttpHeaders, +															  this); +		ba->release(); +		 +		if (LLCORE_HTTP_HANDLE_INVALID == handle) +		{ +			mHttpStatus = mHttpRequest->getStatus(); +		 +			LL_WARNS(LOG_MESH) << "Couldn't issue request for full model upload.  Reason:  " << mHttpStatus.toString() +							   << " (" << mHttpStatus.toTerseString() << ")" +							   << LL_ENDL; +		} +		else  		{ -			LLCurl::ResponderPtr responder = new LLWholeModelUploadResponder(this, full_model_data, mUploadObserverHandle) ; +			U32 sleep_time(10); +		 +			LL_DEBUGS(LOG_MESH) << "POST request issued." << LL_ENDL; +			 +			mHttpRequest->update(0); +			while (! LLApp::isQuitting() && ! finished() && ! isDiscarded()) +			{ +				ms_sleep(sleep_time); +				sleep_time = llmin(250U, sleep_time + sleep_time); +				mHttpRequest->update(0); +			} -			while(!mCurlRequest->post(mWholeModelUploadURL, headers, body, responder, mMeshUploadTimeOut)) +			if (isDiscarded()) +			{ +				LL_DEBUGS(LOG_MESH) << "Mesh upload operation discarded." << LL_ENDL; +			} +			else  			{ -				//sleep for 10ms to prevent eating a whole core -				apr_sleep(10000); +				LL_DEBUGS(LOG_MESH) << "Mesh upload operation completed." << LL_ENDL;  			}  		} - -		do -		{ -			mCurlRequest->process(); -			//sleep for 10ms to prevent eating a whole core -			apr_sleep(10000); -		} while (!LLAppViewer::isQuitting() && mPendingUploads > 0);  	} - -	delete mCurlRequest; -	mCurlRequest = NULL; - -	// Currently a no-op. -	mFinished = true;  }  void LLMeshUploadThread::requestWholeModelFee()  {  	dump_num++; -	mCurlRequest = new LLCurlRequest(); -  	generateHulls(); -	LLSD model_data; -	wholeModelToLLSD(model_data,false); -	dump_llsd_to_file(model_data,make_dump_name("whole_model_fee_request_",dump_num)); - -	LLCurlRequest::headers_t headers; +	mModelData = LLSD::emptyMap(); +	wholeModelToLLSD(mModelData, false); +	dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); +	LLCore::BufferArray * ba = new LLCore::BufferArray; +	LLCore::BufferArrayStream bas(ba); +	LLSDSerialize::toXML(mModelData, bas); +		 +	LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, +														  mHttpPriority, +														  mWholeModelFeeCapability, +														  ba, +														  mHttpOptions, +														  mHttpHeaders, +														  this); +	ba->release(); +	if (LLCORE_HTTP_HANDLE_INVALID == handle) +	{ +		mHttpStatus = mHttpRequest->getStatus(); +		 +		LL_WARNS(LOG_MESH) << "Couldn't issue request for model fee.  Reason:  " << mHttpStatus.toString() +						   << " (" << mHttpStatus.toTerseString() << ")" +						   << LL_ENDL; +	} +	else  	{ -		LLCurl::ResponderPtr responder = new LLWholeModelFeeResponder(this,model_data, mFeeObserverHandle) ; -		while(!mCurlRequest->post(mWholeModelFeeCapability, headers, model_data, responder, mMeshUploadTimeOut)) +		U32 sleep_time(10); +		 +		mHttpRequest->update(0); +		while (! LLApp::isQuitting() && ! finished() && ! isDiscarded())  		{ -			//sleep for 10ms to prevent eating a whole core -			apr_sleep(10000); +			ms_sleep(sleep_time); +			sleep_time = llmin(250U, sleep_time + sleep_time); +			mHttpRequest->update(0); +		} +		if (isDiscarded()) +		{ +			LL_DEBUGS(LOG_MESH) << "Mesh fee query operation discarded." << LL_ENDL;  		}  	} +} -	do -	{ -		mCurlRequest->process(); -		//sleep for 10ms to prevent eating a whole core -		apr_sleep(10000); -	} while (!LLApp::isQuitting() && mPendingUploads > 0); -	delete mCurlRequest; -	mCurlRequest = NULL; +// Does completion duty for both fee queries and actual uploads. +void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ +	// QA/Devel:  0x2 to enable fake error import on upload, 0x1 on fee check +	const S32 fake_error(gSavedSettings.getS32("MeshUploadFakeErrors") & (mDoUpload ? 0xa : 0x5)); +	LLCore::HttpStatus status(response->getStatus()); +	if (fake_error) +	{ +		status = (fake_error & 0x0c) ? LLCore::HttpStatus(500) : LLCore::HttpStatus(200); +	} +	std::string reason(status.toString()); +	LLSD body; -	// Currently a no-op.  	mFinished = true; + +	if (mDoUpload) +	{ +		// model upload case +		LLWholeModelUploadObserver * observer(mUploadObserverHandle.get()); + +		if (! status) +		{ +			LL_WARNS(LOG_MESH) << "Upload failed.  Reason:  " << reason +							   << " (" << status.toTerseString() << ")" +							   << LL_ENDL; + +			// Build a fake body for the alert generator +			body["error"] = LLSD::emptyMap(); +			body["error"]["message"] = reason; +			body["error"]["identifier"] = "NetworkError";		// from asset-upload/upload_util.py +			log_upload_error(status, body, "upload", mModelData["name"].asString()); + +			if (observer) +			{ +				doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); +			} +		} +		else +		{ +			if (fake_error & 0x2) +			{ +				body = llsd_from_file("fake_upload_error.xml"); +			} +			else +			{ +				LLCore::BufferArray * ba(response->getBody()); +				if (ba && ba->size()) +				{ +					LLCore::BufferArrayStream bas(ba); +					LLSDSerialize::fromXML(body, bas); +				} +			} +			dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); + +			if (body["state"].asString() == "complete") +			{ +				// requested "mesh" asset type isn't actually the type +				// of the resultant object, fix it up here. +				mModelData["asset_type"] = "object"; +				gMeshRepo.updateInventory(LLMeshRepository::inventory_data(mModelData, body)); + +				if (observer) +				{ +					doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadSuccess, observer)); +				} +			} +			else +			{ +				LL_WARNS(LOG_MESH) << "Upload failed.  Not in expected 'complete' state." << LL_ENDL; +				log_upload_error(status, body, "upload", mModelData["name"].asString()); + +				if (observer) +				{ +					doOnIdleOneTime(boost::bind(&LLWholeModelUploadObserver::onModelUploadFailure, observer)); +				} +			} +		} +	} +	else +	{ +		// model fee case +		LLWholeModelFeeObserver* observer(mFeeObserverHandle.get()); +		mWholeModelUploadURL.clear(); +		 +		if (! status) +		{ +			LL_WARNS(LOG_MESH) << "Fee request failed.  Reason:  " << reason +							   << " (" << status.toTerseString() << ")" +							   << LL_ENDL; + +			// Build a fake body for the alert generator +			body["error"] = LLSD::emptyMap(); +			body["error"]["message"] = reason; +			body["error"]["identifier"] = "NetworkError";		// from asset-upload/upload_util.py +			log_upload_error(status, body, "fee", mModelData["name"].asString()); + +			if (observer) +			{ +				observer->setModelPhysicsFeeErrorStatus(status.toULong(), reason); +			} +		} +		else +		{ +			if (fake_error & 0x1) +			{ +				body = llsd_from_file("fake_upload_error.xml"); +			} +			else +			{ +				LLCore::BufferArray * ba(response->getBody()); +				if (ba && ba->size()) +				{ +					LLCore::BufferArrayStream bas(ba); +					LLSDSerialize::fromXML(body, bas); +				} +			} +			dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); +		 +			if (body["state"].asString() == "upload") +			{ +				mWholeModelUploadURL = body["uploader"].asString(); + +				if (observer) +				{ +					body["data"]["upload_price"] = body["upload_price"]; +					observer->onModelPhysicsFeeReceived(body["data"], mWholeModelUploadURL); +				} +			} +			else +			{ +				LL_WARNS(LOG_MESH) << "Fee request failed.  Not in expected 'upload' state." << LL_ENDL; +				log_upload_error(status, body, "fee", mModelData["name"].asString()); + +				if (observer) +				{ +					observer->setModelPhysicsFeeErrorStatus(status.toULong(), reason); +				} +			} +		} +	}  } +  void LLMeshRepoThread::notifyLoadedMeshes()  { +	bool update_metrics(false); +	  	if (!mMutex)  	{  		return; @@ -1806,10 +2502,16 @@ void LLMeshRepoThread::notifyLoadedMeshes()  	while (!mLoadedQ.empty())  	{  		mMutex->lock(); +		if (mLoadedQ.empty()) +		{ +			mMutex->unlock(); +			break; +		}  		LoadedMesh mesh = mLoadedQ.front();  		mLoadedQ.pop();  		mMutex->unlock(); +		update_metrics = true;  		if (mesh.mVolume && mesh.mVolume->getNumVolumeFaces() > 0)  		{  			gMeshRepo.notifyMeshLoaded(mesh.mMeshParams, mesh.mVolume); @@ -1824,24 +2526,59 @@ void LLMeshRepoThread::notifyLoadedMeshes()  	while (!mUnavailableQ.empty())  	{  		mMutex->lock(); +		if (mUnavailableQ.empty()) +		{ +			mMutex->unlock(); +			break; +		} +		  		LODRequest req = mUnavailableQ.front();  		mUnavailableQ.pop();  		mMutex->unlock(); -		 + +		update_metrics = true;  		gMeshRepo.notifyMeshUnavailable(req.mMeshParams, req.mLOD);  	} -	while (!mSkinInfoQ.empty()) +	if (! mSkinInfoQ.empty() || ! mDecompositionQ.empty())  	{ -		gMeshRepo.notifySkinInfoReceived(mSkinInfoQ.front()); -		mSkinInfoQ.pop(); +		if (mMutex->trylock()) +		{ +			std::list<LLMeshSkinInfo> skin_info_q; +			std::list<LLModel::Decomposition*> decomp_q; + +			if (! mSkinInfoQ.empty()) +			{ +				skin_info_q.swap(mSkinInfoQ); +			} +			if (! mDecompositionQ.empty()) +			{ +				decomp_q.swap(mDecompositionQ); +			} + +			mMutex->unlock(); + +			// Process the elements free of the lock +			while (! skin_info_q.empty()) +			{ +				gMeshRepo.notifySkinInfoReceived(skin_info_q.front()); +				skin_info_q.pop_front(); +			} + +			while (! decomp_q.empty()) +			{ +				gMeshRepo.notifyDecompositionReceived(decomp_q.front()); +				decomp_q.pop_front(); +			} +		}  	} -	while (!mDecompositionQ.empty()) +	if (update_metrics)  	{ -		gMeshRepo.notifyDecompositionReceived(mDecompositionQ.front()); -		mDecompositionQ.pop(); +		// Ping time-to-load metrics for mesh download operations. +		LLMeshRepository::metricsProgress(0);  	} +	  }  S32 LLMeshRepoThread::getActualMeshLOD(const LLVolumeParams& mesh_params, S32 lod)  @@ -1920,249 +2657,250 @@ void LLMeshRepository::cacheOutgoingMesh(LLMeshUploadData& data, LLSD& header)  } -void LLMeshLODResponder::completedRaw(const LLChannelDescriptors& channels, -									  const LLIOPipe::buffer_ptr_t& buffer) +void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response)  { -	S32 status = getStatus();  	mProcessed = true; -	 -	// thread could have already be destroyed during logout -	if( !gMeshRepo.mThread ) -	{ -		return; -	} -	 -	S32 data_size = buffer->countAfter(channels.in(), NULL); -	// *TODO: What about 3xx redirect codes? What about status 400 (Bad Request)? -	if (status < 200 || status > 400) +	unsigned int retries(0U); +	response->getRetries(NULL, &retries); +	LLMeshRepository::sHTTPRetryCount += retries; + +	LLCore::HttpStatus status(response->getStatus()); +	if (! status || MESH_HTTP_RESPONSE_FAILED)  	{ -		llwarns << dumpResponse() << llendl; +		processFailure(status); +		++LLMeshRepository::sHTTPErrorCount;  	} - -	if (data_size < mRequestedBytes) +	else  	{ -		if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) -		{ //timeout or service unavailable, try again -			llwarns << "Timeout or service unavailable, retrying." << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshLOD(mMeshParams, mLOD); -		} -		else +		// From texture fetch code and may apply here: +		// +		// A warning about partial (HTTP 206) data.  Some grid services +		// do *not* return a 'Content-Range' header in the response to +		// Range requests with a 206 status.  We're forced to assume +		// we get what we asked for in these cases until we can fix +		// the services. +		// +		// May also need to deal with 200 status (full asset returned +		// rather than partial) and 416 (request completely unsatisfyable). +		// Always been exposed to these but are less likely here where +		// speculative loads aren't done. +		static const LLCore::HttpStatus par_status(HTTP_PARTIAL_CONTENT); + +		if (par_status != status)  		{ -			llwarns << "Unhandled status " << dumpResponse() << llendl; -			llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint +			LL_WARNS_ONCE(LOG_MESH) << "Non-206 successful status received for fetch:  " +									<< status.toTerseString() << LL_ENDL;  		} -		return; -	} +		 +		LLCore::BufferArray * body(response->getBody()); +		S32 data_size(body ? body->size() : 0); +		U8 * data(NULL); -	LLMeshRepository::sBytesReceived += mRequestedBytes; +		if (data_size > 0) +		{ +			// *TODO: Try to get rid of data copying and add interfaces +			// that support BufferArray directly.  Introduce a two-phase +			// handler, optional first that takes a body, fallback second +			// that requires a temporary allocation and data copy. +			data = new U8[data_size]; +			body->read(0, (char *) data, data_size); +			LLMeshRepository::sBytesReceived += data_size; +		} -	U8* data = NULL; +		processData(body, data, data_size); -	if (data_size > 0) -	{ -		data = new U8[data_size]; -		buffer->readAfter(channels.in(), NULL, data, data_size); +		delete [] data;  	} -	if (gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) -	{ -		//good fetch from sim, write to VFS for caching -		LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); +	// Release handler +	gMeshRepo.mThread->mHttpRequestSet.erase(this); +	delete this;		// Must be last statement +} -		S32 offset = mOffset; -		S32 size = mRequestedBytes; -		if (file.getSize() >= offset+size) +LLMeshHeaderHandler::~LLMeshHeaderHandler() +{ +	if (!LLApp::isQuitting()) +	{ +		if (! mProcessed)  		{ -			file.seek(offset); -			file.write(data, size); -			LLMeshRepository::sCacheBytesWritten += size; +			// something went wrong, retry +			LL_WARNS(LOG_MESH) << "Mesh header fetch canceled unexpectedly, retrying." << LL_ENDL; +			LLMeshRepoThread::HeaderRequest req(mMeshParams); +			LLMutexLock lock(gMeshRepo.mThread->mMutex); +			gMeshRepo.mThread->mHeaderReqQ.push(req);  		} +		LLMeshRepoThread::decActiveHeaderRequests();  	} - -	delete [] data;  } -void LLMeshSkinInfoResponder::completedRaw(const LLChannelDescriptors& channels, -										   const LLIOPipe::buffer_ptr_t& buffer) +void LLMeshHeaderHandler::processFailure(LLCore::HttpStatus status)  { -	S32 status = getStatus(); -	mProcessed = true; +	LL_WARNS(LOG_MESH) << "Error during mesh header handling.  ID:  " << mMeshParams.getSculptID() +					   << ", Reason:  " << status.toString() +					   << " (" << status.toTerseString() << ").  Not retrying." +					   << LL_ENDL; -	// thread could have already be destroyed during logout -	if( !gMeshRepo.mThread ) +	// Can't get the header so none of the LODs will be available +	LLMutexLock lock(gMeshRepo.mThread->mMutex); +	for (int i(0); i < 4; ++i)  	{ -		return; +		gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i));  	} +} -	S32 data_size = buffer->countAfter(channels.in(), NULL); - -	// *TODO: What about 3xx redirect codes? What about status 400 (Bad Request)? -	if (status < 200 || status > 400) +void LLMeshHeaderHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ +	LLUUID mesh_id = mMeshParams.getSculptID(); +	bool success = (! MESH_HEADER_PROCESS_FAILED) && gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); +	llassert(success); +	if (! success)  	{ -		llwarns << dumpResponse() << llendl; -	} +		// *TODO:  Get real reason for parse failure here.  Might we want to retry? +		LL_WARNS(LOG_MESH) << "Unable to parse mesh header.  ID:  " << mesh_id +						   << ", Unknown reason.  Not retrying." +						   << LL_ENDL; -	if (data_size < mRequestedBytes) -	{ -		if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) -		{ //timeout or service unavailable, try again -			llwarns << "Timeout or service unavailable, retrying loadMeshSkinInfo() for " << mMeshID << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshSkinInfo(mMeshID); -		} -		else +		// Can't get the header so none of the LODs will be available +		LLMutexLock lock(gMeshRepo.mThread->mMutex); +		for (int i(0); i < 4; ++i)  		{ -			llwarns << "Unhandled status " << dumpResponse() << llendl; -			llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint +			gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, i));  		} -		return;  	} +	else if (data && data_size > 0) +	{ +		// header was successfully retrieved from sim, cache in vfs +		LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; -	LLMeshRepository::sBytesReceived += mRequestedBytes; +		S32 version = header["version"].asInteger(); -	U8* data = NULL; +		if (version <= MAX_MESH_VERSION) +		{ +			std::stringstream str; -	if (data_size > 0) -	{ -		data = new U8[data_size]; -		buffer->readAfter(channels.in(), NULL, data, data_size); -	} +			S32 lod_bytes = 0; -	if (gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) -	{ -		//good fetch from sim, write to VFS for caching -		LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); +			for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) +			{ +				// figure out how many bytes we'll need to reserve in the file +				const std::string & lod_name = header_lod[i]; +				lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); +			} +		 +			// just in case skin info or decomposition is at the end of the file (which it shouldn't be) +			lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); +			lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); -		S32 offset = mOffset; -		S32 size = mRequestedBytes; +			S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; +			S32 bytes = lod_bytes + header_bytes;  -		if (file.getSize() >= offset+size) -		{ -			LLMeshRepository::sCacheBytesWritten += size; -			file.seek(offset); -			file.write(data, size); -		} -	} +		 +			// It's possible for the remote asset to have more data than is needed for the local cache +			// only allocate as much space in the VFS as is needed for the local cache +			data_size = llmin(data_size, bytes); -	delete [] data; -} +			LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); +			if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) +			{ +				LLMeshRepository::sCacheBytesWritten += data_size; +				++LLMeshRepository::sCacheWrites; -void LLMeshDecompositionResponder::completedRaw(const LLChannelDescriptors& channels, -												const LLIOPipe::buffer_ptr_t& buffer) -{ -	S32 status = getStatus(); -	mProcessed = true; -	 -	if( !gMeshRepo.mThread ) -	{ -		return; -	} +				file.write(data, data_size); +			 +				// zero out the rest of the file  +				U8 block[MESH_HEADER_SIZE]; +				memset(block, 0, sizeof(block)); -	S32 data_size = buffer->countAfter(channels.in(), NULL); +				while (bytes-file.tell() > sizeof(block)) +				{ +					file.write(block, sizeof(block)); +				} -	// *TODO: What about 3xx redirect codes? What about status 400 (Bad Request)? -	if (status < 200 || status > 400) -	{ -		llwarns << dumpResponse() << llendl; +				S32 remaining = bytes-file.tell(); +				if (remaining > 0) +				{ +					file.write(block, remaining); +				} +			} +		}  	} +} -	if (data_size < mRequestedBytes) +LLMeshLODHandler::~LLMeshLODHandler() +{ +	if (! LLApp::isQuitting())  	{ -		if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) -		{ //timeout or service unavailable, try again -			llwarns << "Timeout or service unavailable, retrying loadMeshDecomposition() for " << mMeshID << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshDecomposition(mMeshID); -		} -		else +		if (! mProcessed)  		{ -			llwarns << "Unhandled status " << dumpResponse() << llendl; -			llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint +			LL_WARNS(LOG_MESH) << "Mesh LOD fetch canceled unexpectedly, retrying." << LL_ENDL; +			gMeshRepo.mThread->lockAndLoadMeshLOD(mMeshParams, mLOD);  		} -		return; +		LLMeshRepoThread::decActiveLODRequests();  	} +} -	LLMeshRepository::sBytesReceived += mRequestedBytes; - -	U8* data = NULL; +void LLMeshLODHandler::processFailure(LLCore::HttpStatus status) +{ +	LL_WARNS(LOG_MESH) << "Error during mesh LOD handling.  ID:  " << mMeshParams.getSculptID() +					   << ", Reason:  " << status.toString() +					   << " (" << status.toTerseString() << ").  Not retrying." +					   << LL_ENDL; -	if (data_size > 0) -	{ -		data = new U8[data_size]; -		buffer->readAfter(channels.in(), NULL, data, data_size); -	} +	LLMutexLock lock(gMeshRepo.mThread->mMutex); +	gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); +} -	if (gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) +void LLMeshLODHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ +	if ((! MESH_LOD_PROCESS_FAILED) && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size))  	{ -		//good fetch from sim, write to VFS for caching -		LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); +		// good fetch from sim, write to VFS for caching +		LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE);  		S32 offset = mOffset;  		S32 size = mRequestedBytes;  		if (file.getSize() >= offset+size)  		{ -			LLMeshRepository::sCacheBytesWritten += size;  			file.seek(offset);  			file.write(data, size); +			LLMeshRepository::sCacheBytesWritten += size; +			++LLMeshRepository::sCacheWrites;  		}  	} - -	delete [] data; -} - -void LLMeshPhysicsShapeResponder::completedRaw(const LLChannelDescriptors& channels, -											   const LLIOPipe::buffer_ptr_t& buffer) -{ -	S32 status = getStatus(); -	mProcessed = true; - -	// thread could have already be destroyed during logout -	if( !gMeshRepo.mThread ) -	{ -		return; -	} - -	S32 data_size = buffer->countAfter(channels.in(), NULL); - -	// *TODO: What about 3xx redirect codes? What about status 400 (Bad Request)? -	if (status < 200 || status > 400) -	{ -		llwarns << dumpResponse() << llendl; -	} - -	if (data_size < mRequestedBytes) +	else  	{ -		if (status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE) -		{ //timeout or service unavailable, try again -			llwarns << "Timeout or service unavailable, retrying loadMeshPhysicsShape() for " << mMeshID << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			gMeshRepo.mThread->loadMeshPhysicsShape(mMeshID); -		} -		else -		{ -			llwarns << "Unhandled status " << dumpResponse() << llendl; -			llassert(status == HTTP_INTERNAL_ERROR || status == HTTP_SERVICE_UNAVAILABLE); //intentionally trigger a breakpoint -		} -		return; +		LL_WARNS(LOG_MESH) << "Error during mesh LOD processing.  ID:  " << mMeshParams.getSculptID() +						   << ", Unknown reason.  Not retrying." +						   << LL_ENDL; +		LLMutexLock lock(gMeshRepo.mThread->mMutex); +		gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD));  	} +} -	LLMeshRepository::sBytesReceived += mRequestedBytes; +LLMeshSkinInfoHandler::~LLMeshSkinInfoHandler() +{ +		llassert(mProcessed); +} -	U8* data = NULL; +void LLMeshSkinInfoHandler::processFailure(LLCore::HttpStatus status) +{ +	LL_WARNS(LOG_MESH) << "Error during mesh skin info handling.  ID:  " << mMeshID +					   << ", Reason:  " << status.toString() +					   << " (" << status.toTerseString() << ").  Not retrying." +					   << LL_ENDL; -	if (data_size > 0) -	{ -		data = new U8[data_size]; -		buffer->readAfter(channels.in(), NULL, data, data_size); -	} +	// *TODO:  Mark mesh unavailable on error.  For now, simply leave +	// request unfulfilled rather than retry forever. +} -	if (gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size)) +void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ +	if ((! MESH_SKIN_INFO_PROCESS_FAILED) && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size))  	{ -		//good fetch from sim, write to VFS for caching +		// good fetch from sim, write to VFS for caching  		LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE);  		S32 offset = mOffset; @@ -2171,142 +2909,108 @@ void LLMeshPhysicsShapeResponder::completedRaw(const LLChannelDescriptors& chann  		if (file.getSize() >= offset+size)  		{  			LLMeshRepository::sCacheBytesWritten += size; +			++LLMeshRepository::sCacheWrites;  			file.seek(offset);  			file.write(data, size);  		}  	} - -	delete [] data; +	else +	{ +		LL_WARNS(LOG_MESH) << "Error during mesh skin info processing.  ID:  " << mMeshID +						   << ", Unknown reason.  Not retrying." +						   << LL_ENDL; +		// *TODO:  Mark mesh unavailable on error +	}  } -void LLMeshHeaderResponder::completedRaw(const LLChannelDescriptors& channels, -										 const LLIOPipe::buffer_ptr_t& buffer) +LLMeshDecompositionHandler::~LLMeshDecompositionHandler()  { -	S32 status = getStatus(); -	mProcessed = true; +		llassert(mProcessed); +} -	// thread could have already be destroyed during logout -	if( !gMeshRepo.mThread ) -	{ -		return; -	} +void LLMeshDecompositionHandler::processFailure(LLCore::HttpStatus status) +{ +	LL_WARNS(LOG_MESH) << "Error during mesh decomposition handling.  ID:  " << mMeshID +					   << ", Reason:  " << status.toString() +					   << " (" << status.toTerseString() << ").  Not retrying." +					   << LL_ENDL; +	// *TODO:  Mark mesh unavailable on error.  For now, simply leave +	// request unfulfilled rather than retry forever. +} -	// *TODO: What about 3xx redirect codes? What about status 400 (Bad Request)? -	if (status < 200 || status > 400) +void LLMeshDecompositionHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ +	if ((! MESH_DECOMP_PROCESS_FAILED) && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size))  	{ -		//llwarns -		//	<< "Header responder failed with status: " -		//	<< status << ": " << reason << llendl; - -		// 503 (service unavailable) or 499 (internal Linden-generated error) -		// can be due to server load and can be retried - -		// TODO*: Add maximum retry logic, exponential backoff -		// and (somewhat more optional than the others) retries -		// again after some set period of time +		// good fetch from sim, write to VFS for caching +		LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); -		if (status == HTTP_SERVICE_UNAVAILABLE || status == HTTP_REQUEST_TIME_OUT || status == HTTP_INTERNAL_ERROR) -		{ //retry -			llwarns << "Timeout or service unavailable, retrying." << llendl; -			LLMeshRepository::sHTTPRetryCount++; -			LLMeshRepoThread::HeaderRequest req(mMeshParams); -			LLMutexLock lock(gMeshRepo.mThread->mMutex); -			gMeshRepo.mThread->mHeaderReqQ.push(req); +		S32 offset = mOffset; +		S32 size = mRequestedBytes; -			return; -		} -		else +		if (file.getSize() >= offset+size)  		{ -			llwarns << "Unhandled status " << dumpResponse() << llendl; +			LLMeshRepository::sCacheBytesWritten += size; +			++LLMeshRepository::sCacheWrites; +			file.seek(offset); +			file.write(data, size);  		}  	} - -	S32 data_size = buffer->countAfter(channels.in(), NULL); - -	U8* data = NULL; - -	if (data_size > 0) +	else  	{ -		data = new U8[data_size]; -		buffer->readAfter(channels.in(), NULL, data, data_size); +		LL_WARNS(LOG_MESH) << "Error during mesh decomposition processing.  ID:  " << mMeshID +						   << ", Unknown reason.  Not retrying." +						   << LL_ENDL; +		// *TODO:  Mark mesh unavailable on error  	} +} -	LLMeshRepository::sBytesReceived += llmin(data_size, 4096); +LLMeshPhysicsShapeHandler::~LLMeshPhysicsShapeHandler() +{ +		llassert(mProcessed); +} -	bool success = gMeshRepo.mThread->headerReceived(mMeshParams, data, data_size); -	 -	llassert(success); +void LLMeshPhysicsShapeHandler::processFailure(LLCore::HttpStatus status) +{ +	LL_WARNS(LOG_MESH) << "Error during mesh physics shape handling.  ID:  " << mMeshID +					   << ", Reason:  " << status.toString() +					   << " (" << status.toTerseString() << ").  Not retrying." +					   << LL_ENDL; +	// *TODO:  Mark mesh unavailable on error +} -	if (!success) -	{ -		llwarns << "Unable to parse mesh header: " << dumpResponse() << llendl; -	} -	else if (data && data_size > 0) +void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * body, U8 * data, S32 data_size) +{ +	if ((! MESH_PHYS_SHAPE_PROCESS_FAILED) && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size))  	{ -		//header was successfully retrieved from sim, cache in vfs -		LLUUID mesh_id = mMeshParams.getSculptID(); -		LLSD header = gMeshRepo.mThread->mMeshHeader[mesh_id]; +		// good fetch from sim, write to VFS for caching +		LLVFile file(gVFS, mMeshID, LLAssetType::AT_MESH, LLVFile::WRITE); -		S32 version = header["version"].asInteger(); +		S32 offset = mOffset; +		S32 size = mRequestedBytes; -		if (version <= MAX_MESH_VERSION) +		if (file.getSize() >= offset+size)  		{ -			std::stringstream str; - -			S32 lod_bytes = 0; - -			for (U32 i = 0; i < LLModel::LOD_PHYSICS; ++i) -			{ //figure out how many bytes we'll need to reserve in the file -				std::string lod_name = header_lod[i]; -				lod_bytes = llmax(lod_bytes, header[lod_name]["offset"].asInteger()+header[lod_name]["size"].asInteger()); -			} -		 -			//just in case skin info or decomposition is at the end of the file (which it shouldn't be) -			lod_bytes = llmax(lod_bytes, header["skin"]["offset"].asInteger() + header["skin"]["size"].asInteger()); -			lod_bytes = llmax(lod_bytes, header["physics_convex"]["offset"].asInteger() + header["physics_convex"]["size"].asInteger()); - -			S32 header_bytes = (S32) gMeshRepo.mThread->mMeshHeaderSize[mesh_id]; -			S32 bytes = lod_bytes + header_bytes;  - -		 -			//it's possible for the remote asset to have more data than is needed for the local cache -			//only allocate as much space in the VFS as is needed for the local cache -			data_size = llmin(data_size, bytes); - -			LLVFile file(gVFS, mesh_id, LLAssetType::AT_MESH, LLVFile::WRITE); -			if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) -			{ -				LLMeshRepository::sCacheBytesWritten += data_size; - -				file.write((const U8*) data, data_size); -			 -				//zero out the rest of the file  -				U8 block[4096]; -				memset(block, 0, 4096); - -				while (bytes-file.tell() > 4096) -				{ -					file.write(block, 4096); -				} - -				S32 remaining = bytes-file.tell(); - -				if (remaining > 0) -				{ -					file.write(block, remaining); -				} -			} +			LLMeshRepository::sCacheBytesWritten += size; +			++LLMeshRepository::sCacheWrites; +			file.seek(offset); +			file.write(data, size);  		}  	} - -	delete [] data; +	else +	{ +		LL_WARNS(LOG_MESH) << "Error during mesh physics shape processing.  ID:  " << mMeshID +						   << ", Unknown reason.  Not retrying." +						   << LL_ENDL; +		// *TODO:  Mark mesh unavailable on error +	}  } -  LLMeshRepository::LLMeshRepository()  : mMeshMutex(NULL),    mMeshThreadCount(0), -  mThread(NULL) +  mThread(NULL), +  mGetMeshVersion(2)  {  } @@ -2325,7 +3029,7 @@ void LLMeshRepository::init()  		apr_sleep(100);  	} -	 +	metrics_teleport_started_signal = LLViewerMessage::getInstance()->setTeleportStartedCallback(teleport_started);  	mThread = new LLMeshRepoThread();  	mThread->start(); @@ -2333,11 +3037,13 @@ void LLMeshRepository::init()  void LLMeshRepository::shutdown()  { -	llinfos << "Shutting down mesh repository." << llendl; +	LL_INFOS(LOG_MESH) << "Shutting down mesh repository." << LL_ENDL; + +	metrics_teleport_started_signal.disconnect();  	for (U32 i = 0; i < mUploads.size(); ++i)  	{ -		llinfos << "Discard the pending mesh uploads " << llendl; +		LL_INFOS(LOG_MESH) << "Discard the pending mesh uploads." << LL_ENDL;  		mUploads[i]->discard() ; //discard the uploading requests.  	} @@ -2352,7 +3058,7 @@ void LLMeshRepository::shutdown()  	for (U32 i = 0; i < mUploads.size(); ++i)  	{ -		llinfos << "Waiting for pending mesh upload " << i << "/" << mUploads.size() << llendl; +		LL_INFOS(LOG_MESH) << "Waiting for pending mesh upload " << (i + 1) << "/" << mUploads.size() << LL_ENDL;  		while (!mUploads[i]->isStopped())  		{  			apr_sleep(10); @@ -2365,7 +3071,7 @@ void LLMeshRepository::shutdown()  	delete mMeshMutex;  	mMeshMutex = NULL; -	llinfos << "Shutting down decomposition system." << llendl; +	LL_INFOS(LOG_MESH) << "Shutting down decomposition system." << LL_ENDL;  	if (mDecompThread)  	{ @@ -2380,6 +3086,9 @@ void LLMeshRepository::shutdown()  //called in the main thread.  S32 LLMeshRepository::update()  { +	// Conditionally log a mesh metrics event +	metricsUpdate(); +	  	if(mUploadWaitList.empty())  	{  		return 0 ; @@ -2399,7 +3108,12 @@ S32 LLMeshRepository::update()  S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_params, S32 detail, S32 last_lod)  { -	if (detail < 0 || detail > 4) +	MESH_FASTTIMER_DEFBLOCK; +	 +	// Manage time-to-load metrics for mesh download operations. +	metricsProgress(1); + +	if (detail < 0 || detail >= 4)  	{  		return detail;  	} @@ -2477,9 +3191,32 @@ S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_para  void LLMeshRepository::notifyLoadedMeshes()  { //called from main thread +	MESH_FASTTIMER_DEFBLOCK; -	LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); - +	if (1 == mGetMeshVersion) +	{ +		// Legacy GetMesh operation with high connection concurrency +		LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); +		LLMeshRepoThread::sRequestHighWater = llclamp(2 * S32(LLMeshRepoThread::sMaxConcurrentRequests), +													  REQUEST_HIGH_WATER_MIN, +													  REQUEST_HIGH_WATER_MAX); +		LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, +													 REQUEST_LOW_WATER_MIN, +													 REQUEST_LOW_WATER_MAX); +	} +	else +	{ +		// GetMesh2 operation with keepalives, etc.  With pipelining, +		// we'll increase this. +		LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); +		LLMeshRepoThread::sRequestHighWater = llclamp(5 * S32(LLMeshRepoThread::sMaxConcurrentRequests), +													  REQUEST2_HIGH_WATER_MIN, +													  REQUEST2_HIGH_WATER_MAX); +		LLMeshRepoThread::sRequestLowWater = llclamp(LLMeshRepoThread::sRequestHighWater / 2, +													 REQUEST2_LOW_WATER_MIN, +													 REQUEST2_LOW_WATER_MAX); +	} +	  	//clean up completed upload threads  	for (std::vector<LLMeshUploadThread*>::iterator iter = mUploads.begin(); iter != mUploads.end(); )  	{ @@ -2556,26 +3293,46 @@ void LLMeshRepository::notifyLoadedMeshes()  	//call completed callbacks on finished decompositions  	mDecompThread->notifyCompleted(); -	 -	if (!mThread->mWaiting) -	{ //curl thread is churning, wait for it to go idle -		return; -	} -	static std::string region_name("never name a region this"); +	// For major operations, attempt to get the required locks +	// without blocking and punt if they're not available.  The +	// longest run of holdoffs is kept in sMaxLockHoldoffs just +	// to collect the data.  In testing, I've never seen a value +	// greater than 2 (written to log on exit). +	{ +		LLMutexTrylock lock1(mMeshMutex); +		LLMutexTrylock lock2(mThread->mMutex); -	if (gAgent.getRegion()) -	{ //update capability url  -		if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) +		static U32 hold_offs(0); +		if (! lock1.isLocked() || ! lock2.isLocked())  		{ -			region_name = gAgent.getRegion()->getName(); -			mGetMeshCapability = gAgent.getRegion()->getCapability("GetMesh"); +			// If we can't get the locks, skip and pick this up later. +			++hold_offs; +			sMaxLockHoldoffs = llmax(sMaxLockHoldoffs, hold_offs); +			return; +		} +		hold_offs = 0; +		 +		if (gAgent.getRegion()) +		{ +			// Update capability urls +			static std::string region_name("never name a region this"); +			 +			if (gAgent.getRegion()->getName() != region_name && gAgent.getRegion()->capabilitiesReceived()) +			{ +				region_name = gAgent.getRegion()->getName(); +				const bool use_v1(gSavedSettings.getBOOL("MeshUseGetMesh1")); +				const std::string mesh1(gAgent.getRegion()->getCapability("GetMesh")); +				const std::string mesh2(gAgent.getRegion()->getCapability("GetMesh2")); +				mGetMeshVersion = (mesh2.empty() || use_v1) ? 1 : 2; +				mThread->setGetMeshCaps(mesh1, mesh2, mGetMeshVersion); +				LL_DEBUGS(LOG_MESH) << "Retrieving caps for region '" << region_name +									<< "', GetMesh2:  " << mesh2 +									<< ", GetMesh:  " << mesh1 +									<< ", using version:  " << mGetMeshVersion +									<< LL_ENDL; +			}  		} -	} - -	{ -		LLMutexLock lock1(mMeshMutex); -		LLMutexLock lock2(mThread->mMutex);  		//popup queued error messages from background threads  		while (!mUploadErrorQ.empty()) @@ -2584,47 +3341,55 @@ void LLMeshRepository::notifyLoadedMeshes()  			mUploadErrorQ.pop();  		} -		S32 push_count = LLMeshRepoThread::sMaxConcurrentRequests-(LLMeshRepoThread::sActiveHeaderRequests+LLMeshRepoThread::sActiveLODRequests); - -		if (push_count > 0) +		S32 active_count = LLMeshRepoThread::sActiveHeaderRequests + LLMeshRepoThread::sActiveLODRequests; +		if (active_count < LLMeshRepoThread::sRequestLowWater)  		{ -			//calculate "score" for pending requests - -			//create score map -			std::map<LLUUID, F32> score_map; +			S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; -			for (U32 i = 0; i < 4; ++i) +			if (mPendingRequests.size() > push_count)  			{ -				for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin();  iter != mLoadingMeshes[i].end(); ++iter) +				// More requests than the high-water limit allows so +				// sort and forward the most important. + +				//calculate "score" for pending requests + +				//create score map +				std::map<LLUUID, F32> score_map; + +				for (U32 i = 0; i < 4; ++i)  				{ -					F32 max_score = 0.f; -					for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) +					for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin();  iter != mLoadingMeshes[i].end(); ++iter)  					{ -						LLViewerObject* object = gObjectList.findObject(*obj_iter); - -						if (object) +						F32 max_score = 0.f; +						for (std::set<LLUUID>::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter)  						{ -							LLDrawable* drawable = object->mDrawable; -							if (drawable) +							LLViewerObject* object = gObjectList.findObject(*obj_iter); + +							if (object)  							{ -								F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); -								max_score = llmax(max_score, cur_score); +								LLDrawable* drawable = object->mDrawable; +								if (drawable) +								{ +									F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); +									max_score = llmax(max_score, cur_score); +								}  							}  						} -					} -					score_map[iter->first.getSculptID()] = max_score; +						score_map[iter->first.getSculptID()] = max_score; +					}  				} -			} -			//set "score" for pending requests -			for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) -			{ -				iter->mScore = score_map[iter->mMeshParams.getSculptID()]; -			} +				//set "score" for pending requests +				for (std::vector<LLMeshRepoThread::LODRequest>::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) +				{ +					iter->mScore = score_map[iter->mMeshParams.getSculptID()]; +				} -			//sort by "score" -			std::sort(mPendingRequests.begin(), mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); +				//sort by "score" +				std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, +								  mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); +			}  			while (!mPendingRequests.empty() && push_count > 0)  			{ @@ -2678,9 +3443,8 @@ void LLMeshRepository::notifySkinInfoReceived(LLMeshSkinInfo& info)  				vobj->notifyMeshLoaded();  			}  		} +		mLoadingSkins.erase(info.mMeshID);  	} - -	mLoadingSkins.erase(info.mMeshID);  }  void LLMeshRepository::notifyDecompositionReceived(LLModel::Decomposition* decomp) @@ -2689,14 +3453,14 @@ void LLMeshRepository::notifyDecompositionReceived(LLModel::Decomposition* decom  	if (iter == mDecompositionMap.end())  	{ //just insert decomp into map  		mDecompositionMap[decomp->mMeshID] = decomp; +		mLoadingDecompositions.erase(decomp->mMeshID);  	}  	else  	{ //merge decomp with existing entry  		iter->second->merge(decomp); +		mLoadingDecompositions.erase(decomp->mMeshID);  		delete decomp;  	} - -	mLoadingDecompositions.erase(decomp->mMeshID);  }  void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVolume* volume) @@ -2711,7 +3475,8 @@ void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVol  		//make sure target volume is still valid  		if (volume->getNumVolumeFaces() <= 0)  		{ -			llwarns << "Mesh loading returned empty volume." << llendl; +			LL_WARNS(LOG_MESH) << "Mesh loading returned empty volume.  ID:  " << mesh_params.getSculptID() +							   << LL_ENDL;  		}  		{ //update system volume @@ -2724,7 +3489,8 @@ void LLMeshRepository::notifyMeshLoaded(const LLVolumeParams& mesh_params, LLVol  			}  			else  			{ -				llwarns << "Couldn't find system volume for given mesh." << llendl; +				LL_WARNS(LOG_MESH) << "Couldn't find system volume for mesh " << mesh_params.getSculptID() +								   << LL_ENDL;  			}  		} @@ -2778,6 +3544,8 @@ S32 LLMeshRepository::getActualMeshLOD(const LLVolumeParams& mesh_params, S32 lo  const LLMeshSkinInfo* LLMeshRepository::getSkinInfo(const LLUUID& mesh_id, const LLVOVolume* requesting_obj)  { +	MESH_FASTTIMER_DEFBLOCK; +  	if (mesh_id.notNull())  	{  		skin_map::iterator iter = mSkinMap.find(mesh_id); @@ -2804,6 +3572,8 @@ const LLMeshSkinInfo* LLMeshRepository::getSkinInfo(const LLUUID& mesh_id, const  void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id)  { +	MESH_FASTTIMER_DEFBLOCK; +  	if (mesh_id.notNull())  	{  		LLModel::Decomposition* decomp = NULL; @@ -2821,6 +3591,7 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id)  			std::set<LLUUID>::iterator iter = mLoadingPhysicsShapes.find(mesh_id);  			if (iter == mLoadingPhysicsShapes.end())  			{ //no request pending for this skin info +				// *FIXME:  Nothing ever deletes entries, can't be right  				mLoadingPhysicsShapes.insert(mesh_id);  				mPendingPhysicsShapeRequests.push(mesh_id);  			} @@ -2831,6 +3602,8 @@ void LLMeshRepository::fetchPhysicsShape(const LLUUID& mesh_id)  LLModel::Decomposition* LLMeshRepository::getDecomposition(const LLUUID& mesh_id)  { +	MESH_FASTTIMER_DEFBLOCK; +  	LLModel::Decomposition* ret = NULL;  	if (mesh_id.notNull()) @@ -2893,6 +3666,8 @@ bool LLMeshRepository::hasPhysicsShape(const LLUUID& mesh_id)  LLSD& LLMeshRepository::getMeshHeader(const LLUUID& mesh_id)  { +	MESH_FASTTIMER_DEFBLOCK; +  	return mThread->getMeshHeader(mesh_id);  } @@ -2914,7 +3689,7 @@ LLSD& LLMeshRepoThread::getMeshHeader(const LLUUID& mesh_id)  void LLMeshRepository::uploadModel(std::vector<LLModelInstance>& data, LLVector3& scale, bool upload_textures, -									bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload, +								   bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload,  								   LLHandle<LLWholeModelFeeObserver> fee_observer, LLHandle<LLWholeModelUploadObserver> upload_observer)  {  	LLMeshUploadThread* thread = new LLMeshUploadThread(data, scale, upload_textures, upload_skin, upload_joints, upload_url,  @@ -2943,7 +3718,6 @@ S32 LLMeshRepository::getMeshSize(const LLUUID& mesh_id, S32 lod)  	}  	return -1; -  }  void LLMeshUploadThread::decomposeMeshMatrix(LLMatrix4& transformation, @@ -3208,7 +3982,7 @@ void LLPhysicsDecomp::setMeshData(LLCDMeshData& mesh, bool vertex_based)  		if (ret)  		{ -			llerrs << "Convex Decomposition thread valid but could not set mesh data" << llendl; +			LL_ERRS(LOG_MESH) << "Convex Decomposition thread valid but could not set mesh data." << LL_ENDL;  		}  	}  } @@ -3284,7 +4058,8 @@ void LLPhysicsDecomp::doDecomposition()  	if (ret)  	{ -		llwarns << "Convex Decomposition thread valid but could not execute stage " << stage << llendl; +		LL_WARNS(LOG_MESH) << "Convex Decomposition thread valid but could not execute stage " << stage << "." +						   << LL_ENDL;  		LLMutexLock lock(mMutex);  		mCurRequest->mHull.clear(); @@ -3413,9 +4188,9 @@ void LLPhysicsDecomp::doDecompositionSingleHull()  	setMeshData(mesh, true);  	LLCDResult ret = decomp->buildSingleHull() ; -	if(ret) +	if (ret)  	{ -		llwarns << "Could not execute decomposition stage when attempting to create single hull." << llendl; +		LL_WARNS(LOG_MESH) << "Could not execute decomposition stage when attempting to create single hull." << LL_ENDL;  		make_box(mCurRequest);  	}  	else @@ -3720,3 +4495,63 @@ bool LLMeshRepository::meshRezEnabled()  	}  	return false;  } + +// Threading:  main thread only +// static +void LLMeshRepository::metricsStart() +{ +	++metrics_teleport_start_count; +	sQuiescentTimer.start(0); +} + +// Threading:  main thread only +// static +void LLMeshRepository::metricsStop() +{ +	sQuiescentTimer.stop(0); +} + +// Threading:  main thread only +// static +void LLMeshRepository::metricsProgress(unsigned int this_count) +{ +	static bool first_start(true); + +	if (first_start) +	{ +		metricsStart(); +		first_start = false; +	} +	sQuiescentTimer.ringBell(0, this_count); +} + +// Threading:  main thread only +// static +void LLMeshRepository::metricsUpdate() +{ +	F64 started, stopped; +	U64 total_count(U64L(0)), user_cpu(U64L(0)), sys_cpu(U64L(0)); +	 +	if (sQuiescentTimer.isExpired(0, started, stopped, total_count, user_cpu, sys_cpu)) +	{ +		LLSD metrics; + +		metrics["reason"] = "Mesh Download Quiescent"; +		metrics["scope"] = metrics_teleport_start_count > 1 ? "Teleport" : "Login"; +		metrics["start"] = started; +		metrics["stop"] = stopped; +		metrics["fetches"] = LLSD::Integer(total_count); +		metrics["teleports"] = LLSD::Integer(metrics_teleport_start_count); +		metrics["user_cpu"] = double(user_cpu) / 1.0e6; +		metrics["sys_cpu"] = double(sys_cpu) / 1.0e6; +		LL_INFOS(LOG_MESH) << "EventMarker " << metrics << LL_ENDL; +	} +} + +// Threading:  main thread only +// static +void teleport_started() +{ +	LLMeshRepository::metricsStart(); +} + diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 8eaf691d6f..39280bea3a 100755 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2001&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -32,6 +32,12 @@  #include "lluuid.h"  #include "llviewertexture.h"  #include "llvolume.h" +#include "lldeadmantimer.h" +#include "httpcommon.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" +#include "httphandler.h"  #define LLCONVEXDECOMPINTER_STATIC 1 @@ -39,8 +45,6 @@  #include "lluploadfloaterobservers.h"  class LLVOVolume; -class LLMeshResponder; -class LLCurlRequest;  class LLMutex;  class LLCondition;  class LLVFS; @@ -215,17 +219,17 @@ class LLMeshRepoThread : public LLThread  {  public: -	static S32 sActiveHeaderRequests; -	static S32 sActiveLODRequests; +	volatile static S32 sActiveHeaderRequests; +	volatile static S32 sActiveLODRequests;  	static U32 sMaxConcurrentRequests; +	static S32 sRequestLowWater; +	static S32 sRequestHighWater; +	static S32 sRequestWaterLevel;			// Stats-use only, may read outside of thread -	LLCurlRequest* mCurlRequest;  	LLMutex*	mMutex;  	LLMutex*	mHeaderMutex;  	LLCondition* mSignal; -	bool mWaiting; -  	//map of known mesh headers  	typedef std::map<LLUUID, LLSD> mesh_header_map;  	mesh_header_map mMeshHeader; @@ -287,8 +291,8 @@ public:  	//set of requested skin info  	std::set<LLUUID> mSkinRequests; -	//queue of completed skin info requests -	std::queue<LLMeshSkinInfo> mSkinInfoQ; +	// list of completed skin info requests +	std::list<LLMeshSkinInfo> mSkinInfoQ;  	//set of requested decompositions  	std::set<LLUUID> mDecompositionRequests; @@ -296,8 +300,8 @@ public:  	//set of requested physics shapes  	std::set<LLUUID> mPhysicsShapeRequests; -	//queue of completed Decomposition info requests -	std::queue<LLModel::Decomposition*> mDecompositionQ; +	// list of completed Decomposition info requests +	std::list<LLModel::Decomposition*> mDecompositionQ;  	//queue of requested headers  	std::queue<HeaderRequest> mHeaderReqQ; @@ -315,7 +319,23 @@ public:  	typedef std::map<LLVolumeParams, std::vector<S32> > pending_lod_map;  	pending_lod_map mPendingLOD; -	static std::string constructUrl(LLUUID mesh_id); +	// llcorehttp library interface objects. +	LLCore::HttpStatus					mHttpStatus; +	LLCore::HttpRequest *				mHttpRequest; +	LLCore::HttpOptions *				mHttpOptions; +	LLCore::HttpOptions *				mHttpLargeOptions; +	LLCore::HttpHeaders *				mHttpHeaders; +	LLCore::HttpRequest::policy_t		mHttpPolicyClass; +	LLCore::HttpRequest::policy_t		mHttpLegacyPolicyClass; +	LLCore::HttpRequest::policy_t		mHttpLargePolicyClass; +	LLCore::HttpRequest::priority_t		mHttpPriority; + +	typedef std::set<LLCore::HttpHandler *> http_request_set; +	http_request_set					mHttpRequestSet;			// Outstanding HTTP requests + +	std::string mGetMeshCapability; +	std::string mGetMesh2Capability; +	int mGetMeshVersion;  	LLMeshRepoThread();  	~LLMeshRepoThread(); @@ -325,8 +345,8 @@ public:  	void lockAndLoadMeshLOD(const LLVolumeParams& mesh_params, S32 lod);  	void loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod); -	bool fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count); -	bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count); +	bool fetchMeshHeader(const LLVolumeParams& mesh_params); +	bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod);  	bool headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size);  	bool lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size);  	bool skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 data_size); @@ -358,9 +378,37 @@ public:  	static void incActiveHeaderRequests();  	static void decActiveHeaderRequests(); +	// Set the caps strings and preferred version for constructing +	// mesh fetch URLs. +	// +	// Mutex:  must be holding mMutex when called +	void setGetMeshCaps(const std::string & get_mesh1, +						const std::string & get_mesh2, +						int pref_version); + +	// Mutex:  acquires mMutex +	void constructUrl(LLUUID mesh_id, std::string * url, int * version); + +private: +	// Issue a GET request to a URL with 'Range' header using +	// the correct policy class and other attributes.  If an invalid +	// handle is returned, the request failed and caller must retry +	// or dispose of handler. +	// +	// Threads:  Repo thread only +	LLCore::HttpHandle getByteRange(const std::string & url, int cap_version, +									size_t offset, size_t len,  +									LLCore::HttpHandler * handler);  }; -class LLMeshUploadThread : public LLThread  + +// Class whose instances represent a single upload-type request for +// meshes:  one fee query or one actual upload attempt.  Yes, it creates +// a unique thread for that single request.  As it is 1:1, it can also +// trivially serve as the HttpHandler object for request completion +// notifications. + +class LLMeshUploadThread : public LLThread, public LLCore::HttpHandler   {  private:  	S32 mMeshUploadTimeOut ; //maximum time in seconds to execute an uploading request. @@ -381,44 +429,41 @@ public:  	};  	LLPointer<DecompRequest> mFinalDecomp; -	bool mPhysicsComplete; +	volatile bool	mPhysicsComplete;  	typedef std::map<LLPointer<LLModel>, std::vector<LLVector3> > hull_map; -	hull_map mHullMap; +	hull_map		mHullMap;  	typedef std::vector<LLModelInstance> instance_list; -	instance_list mInstanceList; +	instance_list	mInstanceList;  	typedef std::map<LLPointer<LLModel>, instance_list> instance_map; -	instance_map mInstance; +	instance_map	mInstance; -	LLMutex*					mMutex; -	LLCurlRequest* mCurlRequest; +	LLMutex*		mMutex;  	S32				mPendingUploads;  	LLVector3		mOrigin;  	bool			mFinished;	  	bool			mUploadTextures;  	bool			mUploadSkin;  	bool			mUploadJoints; -	BOOL            mDiscarded ; +	volatile bool	mDiscarded;  	LLHost			mHost;  	std::string		mWholeModelFeeCapability;  	std::string		mWholeModelUploadURL;  	LLMeshUploadThread(instance_list& data, LLVector3& scale, bool upload_textures, -			bool upload_skin, bool upload_joints, std::string upload_url, bool do_upload = true, -					   LLHandle<LLWholeModelFeeObserver> fee_observer= (LLHandle<LLWholeModelFeeObserver>()), LLHandle<LLWholeModelUploadObserver> upload_observer = (LLHandle<LLWholeModelUploadObserver>())); +					   bool upload_skin, bool upload_joints, const std::string & upload_url, bool do_upload = true, +					   LLHandle<LLWholeModelFeeObserver> fee_observer = (LLHandle<LLWholeModelFeeObserver>()), +					   LLHandle<LLWholeModelUploadObserver> upload_observer = (LLHandle<LLWholeModelUploadObserver>()));  	~LLMeshUploadThread(); -	void startRequest() { ++mPendingUploads; } -	void stopRequest() { --mPendingUploads; } - -	bool finished() { return mFinished; } +	bool finished() const { return mFinished; }  	virtual void run();  	void preStart();  	void discard() ; -	BOOL isDiscarded(); +	bool isDiscarded() const;  	void generateHulls(); @@ -435,11 +480,23 @@ public:  	void setFeeObserverHandle(LLHandle<LLWholeModelFeeObserver> observer_handle) { mFeeObserverHandle = observer_handle; }  	void setUploadObserverHandle(LLHandle<LLWholeModelUploadObserver> observer_handle) { mUploadObserverHandle = observer_handle; } +	// Inherited from LLCore::HttpHandler +	virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); +  private:  	LLHandle<LLWholeModelFeeObserver> mFeeObserverHandle;  	LLHandle<LLWholeModelUploadObserver> mUploadObserverHandle;  	bool mDoUpload; // if FALSE only model data will be requested, otherwise the model will be uploaded +	LLSD mModelData; +	 +	// llcorehttp library interface objects. +	LLCore::HttpStatus					mHttpStatus; +	LLCore::HttpRequest *				mHttpRequest; +	LLCore::HttpOptions *				mHttpOptions; +	LLCore::HttpHeaders *				mHttpHeaders; +	LLCore::HttpRequest::policy_t		mHttpPolicyClass; +	LLCore::HttpRequest::priority_t		mHttpPriority;  };  class LLMeshRepository @@ -448,21 +505,28 @@ public:  	//metrics  	static U32 sBytesReceived; -	static U32 sHTTPRequestCount; -	static U32 sHTTPRetryCount; +	static U32 sMeshRequestCount;				// Total request count, http or cached, all component types +	static U32 sHTTPRequestCount;				// Http GETs issued (not large) +	static U32 sHTTPLargeRequestCount;			// Http GETs issued for large requests +	static U32 sHTTPRetryCount;					// Total request retries whether successful or failed +	static U32 sHTTPErrorCount;					// Requests ending in error  	static U32 sLODPending;  	static U32 sLODProcessing;  	static U32 sCacheBytesRead;  	static U32 sCacheBytesWritten; -	static U32 sPeakKbps; +	static U32 sCacheReads;						 +	static U32 sCacheWrites; +	static U32 sMaxLockHoldoffs;				// Maximum sequential locking failures +	static LLDeadmanTimer sQuiescentTimer;		// Time-to-complete-mesh-downloads after significant events +  	static F32 getStreamingCost(LLSD& header, F32 radius, S32* bytes = NULL, S32* visible_bytes = NULL, S32 detail = -1, F32 *unscaled_value = NULL);  	LLMeshRepository();  	void init();  	void shutdown(); -	S32 update() ; +	S32 update();  	//mesh management functions  	S32 loadMesh(LLVOVolume* volume, const LLVolumeParams& mesh_params, S32 detail = 0, S32 last_lod = -1); @@ -495,6 +559,12 @@ public:  	S32 getMeshSize(const LLUUID& mesh_id, S32 lod); +	// Quiescent timer management, main thread only. +	static void metricsStart(); +	static void metricsStop(); +	static void metricsProgress(unsigned int count); +	static void metricsUpdate(); +	  	typedef std::map<LLVolumeParams, std::set<LLUUID> > mesh_load_map;  	mesh_load_map mLoadingMeshes[4]; @@ -556,8 +626,7 @@ public:  	void uploadError(LLSD& args);  	void updateInventory(inventory_data data); -	std::string mGetMeshCapability; - +	int mGetMeshVersion;		// Shadows value in LLMeshRepoThread  };  extern LLMeshRepository gMeshRepo; diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index d0ab9f55ff..6ed54ce019 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1606,8 +1606,8 @@ bool LLTextureFetchWorker::doWork(S32 param)  				}  				else  				{ -					llinfos << "other: HTTP GET failed for: " << mUrl -							<< " Status: " << mGetStatus.toHex() +					llinfos << "HTTP GET failed for: " << mUrl +							<< " Status: " << mGetStatus.toTerseString()  							<< " Reason: '" << mGetReason << "'"  							<< llendl;  				} @@ -1987,7 +1987,7 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe  	}  	LL_DEBUGS("Texture") << "HTTP COMPLETE: " << mID -						 << " status: " << status.toHex() +						 << " status: " << status.toTerseString()  						 << " '" << status.toString() << "'"  						 << llendl; @@ -2006,7 +2006,9 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe  		success = false;  		if (mFTType != FTT_MAP_TILE) // missing map tiles are normal, don't complain about them.  		{ -			llwarns << mID << " CURL GET FAILED, status: " << status.toHex() +			std::string reason(status.toString()); +			setGetStatus(status, reason); +			llwarns << "CURL GET FAILED, status: " << status.toTerseString()  					<< " reason: " << reason << llendl;  		}  	} @@ -2505,11 +2507,10 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image  	mHttpOptionsWithHeaders = new LLCore::HttpOptions;  	mHttpOptionsWithHeaders->setWantHeaders(true);  	mHttpHeaders = new LLCore::HttpHeaders; -	// *TODO: Should this be 'image/j2c' instead of 'image/x-j2c' ? -	mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); +	mHttpHeaders->append("Accept", "image/x-j2c");  	mHttpMetricsHeaders = new LLCore::HttpHeaders; -	mHttpMetricsHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); -	mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyDefault(); +	mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); +	mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE);  }  LLTextureFetch::~LLTextureFetch() @@ -3884,7 +3885,7 @@ public:  		else  		{  			LL_WARNS("Texture") << "Error delivering asset metrics to grid.  Status:  " -								<< status.toHex() +								<< status.toTerseString()  								<< ", Reason:  " << status.toString() << LL_ENDL;  		}  	} @@ -4153,8 +4154,7 @@ void LLTextureFetchDebugger::init()  	if (! mHttpHeaders)  	{  		mHttpHeaders = new LLCore::HttpHeaders; -		// *TODO: Should this be 'image/j2c' instead of 'image/x-j2c' ? -		mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); +		mHttpHeaders->append("Accept", "image/x-j2c");  	}  } @@ -4574,7 +4574,7 @@ S32 LLTextureFetchDebugger::fillCurlQueue()  			LL_WARNS("Texture") << "Couldn't issue HTTP request in debugger for texture "  								<< mFetchingHistory[i].mID -								<< ", status: " << status.toHex() +								<< ", status: " << status.toTerseString()  								<< " reason:  " << status.toString()  								<< LL_ENDL;  			mFetchingHistory[i].mCurlState = FetchEntry::CURL_DONE; @@ -4967,7 +4967,7 @@ void LLTextureFetchDebugger::callbackHTTP(FetchEntry & fetch, LLCore::HttpRespon  	else //failed  	{  		llinfos << "Fetch Debugger : CURL GET FAILED,  ID = " << fetch.mID -				<< ", status: " << status.toHex() +				<< ", status: " << status.toTerseString()  				<< " reason:  " << status.toString() << llendl;  	}  } diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index dea1c6c1b2..0582bf4162 100755 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2001&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -49,6 +49,7 @@  #include "llviewertexturelist.h"  #include "llvovolume.h"  #include "llviewerstats.h" +#include "llmeshrepository.h"  // For avatar texture view  #include "llvoavatarself.h" @@ -509,6 +510,8 @@ void LLGLTexMemBar::draw()  	F32 total_texture_downloaded = (F32)gTotalTextureBytes / (1024 * 1024);  	F32 total_object_downloaded = (F32)gTotalObjectBytes / (1024 * 1024);  	U32 total_http_requests = LLAppViewer::getTextureFetch()->getTotalNumHTTPRequests(); +	F32 x_right = 0.0; +	  	//----------------------------------------------------------------------------  	LLGLSUIDefault gls_ui;  	LLColor4 text_color(1.f, 1.f, 1.f, 0.75f); @@ -535,7 +538,7 @@ void LLGLTexMemBar::draw()  					cache_max_usage);  	//, cache_entries, cache_max_entries -	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*4, +	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*5,  											 text_color, LLFontGL::LEFT, LLFontGL::TOP);  	U32 cache_read(0U), cache_write(0U), res_wait(0U); @@ -549,13 +552,12 @@ void LLGLTexMemBar::draw()  					cache_write,  					res_wait); -	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*3, +	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*4,  											 text_color, LLFontGL::LEFT, LLFontGL::TOP); -	S32 left = 0 ;  	//---------------------------------------------------------------------------- -	text = llformat("Textures: %d Fetch: %d(%d) Pkts:%d(%d) Cache R/W: %d/%d LFS:%d RAW:%d HTP:%d DEC:%d CRE:%d", +	text = llformat("Textures: %d Fetch: %d(%d) Pkts:%d(%d) Cache R/W: %d/%d LFS:%d RAW:%d HTP:%d DEC:%d CRE:%d ",  					gTextureList.getNumImages(),  					LLAppViewer::getTextureFetch()->getNumRequests(), LLAppViewer::getTextureFetch()->getNumDeletes(),  					LLAppViewer::getTextureFetch()->mPacketCount, LLAppViewer::getTextureFetch()->mBadPacketCount,  @@ -566,19 +568,30 @@ void LLGLTexMemBar::draw()  					LLAppViewer::getImageDecodeThread()->getPending(),   					gTextureList.mCreateTextureList.size()); -	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*2, -									 text_color, LLFontGL::LEFT, LLFontGL::TOP); - +	x_right = 550.0; +	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*3, +											 text_color, LLFontGL::LEFT, LLFontGL::TOP, +											 LLFontGL::NORMAL, LLFontGL::NO_SHADOW, S32_MAX, S32_MAX, +											 &x_right, FALSE); -	left = 550;  	F32 bandwidth = LLAppViewer::getTextureFetch()->getTextureBandwidth();  	F32 max_bandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); -	color = bandwidth > max_bandwidth ? LLColor4::red : bandwidth > max_bandwidth*.75f ? LLColor4::yellow : text_color; +	color = bandwidth > max_bandwidth ? LLColor4::red : bandwidth > max_bandwidth * .75f ? LLColor4::yellow : text_color;  	color[VALPHA] = text_color[VALPHA]; -	text = llformat("BW:%.0f/%.0f",bandwidth, max_bandwidth); -	LLFontGL::getFontMonospace()->renderUTF8(text, 0, left, v_offset + line_height*2, +	text = llformat("BW:%.0f/%.0f", bandwidth, max_bandwidth); +	LLFontGL::getFontMonospace()->renderUTF8(text, 0, x_right, v_offset + line_height*3,  											 color, LLFontGL::LEFT, LLFontGL::TOP); -	 + +	// Mesh status line +	text = llformat("Mesh: Reqs(Tot/Htp/Big): %u/%u/%u Rtr/Err: %u/%u Cread/Cwrite: %u/%u Low/At/High: %d/%d/%d", +					LLMeshRepository::sMeshRequestCount, LLMeshRepository::sHTTPRequestCount, LLMeshRepository::sHTTPLargeRequestCount, +					LLMeshRepository::sHTTPRetryCount, LLMeshRepository::sHTTPErrorCount, +					LLMeshRepository::sCacheReads, LLMeshRepository::sCacheWrites, +					LLMeshRepoThread::sRequestLowWater, LLMeshRepoThread::sRequestWaterLevel, LLMeshRepoThread::sRequestHighWater); +	LLFontGL::getFontMonospace()->renderUTF8(text, 0, 0, v_offset + line_height*2, +											 text_color, LLFontGL::LEFT, LLFontGL::TOP); + +	// Header for texture table columns  	S32 dx1 = 0;  	if (LLAppViewer::getTextureFetch()->mDebugPause)  	{ @@ -621,7 +634,7 @@ BOOL LLGLTexMemBar::handleMouseDown(S32 x, S32 y, MASK mask)  LLRect LLGLTexMemBar::getRequiredRect()  {  	LLRect rect; -	rect.mTop = 50; //LLFontGL::getFontMonospace()->getLineHeight() * 6; +	rect.mTop = 68; //LLFontGL::getFontMonospace()->getLineHeight() * 6;  	return rect;  } diff --git a/indra/newview/llviewerregion.cpp b/indra/newview/llviewerregion.cpp index e6cc75f1ce..76d155d5f2 100755 --- a/indra/newview/llviewerregion.cpp +++ b/indra/newview/llviewerregion.cpp @@ -4,7 +4,7 @@   *   * $LicenseInfo:firstyear=2000&license=viewerlgpl$   * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2010-2013, Linden Research, Inc.   *    * This library is free software; you can redistribute it and/or   * modify it under the terms of the GNU Lesser General Public @@ -1653,6 +1653,7 @@ void LLViewerRegionImpl::buildCapabilityNames(LLSD& capabilityNames)  	capabilityNames.append("GetDisplayNames");  	capabilityNames.append("GetMesh"); +	capabilityNames.append("GetMesh2");  	capabilityNames.append("GetObjectCost");  	capabilityNames.append("GetObjectPhysicsData");  	capabilityNames.append("GetTexture"); | 
