Merge 3.3.3 release with Drano HTTP library at 3.3.0

Big delta was converting the new texture debugger support code
to the new library.  Viewer manifest should probably get an eyeball
before release.

1 files changed, 206 insertions, 47 deletions

diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h
index 107e1623b0..54ffeda8df 100644
--- a/indra/newview/lltexturefetch.h
+++ b/indra/newview/lltexturefetch.h
@@ -4,7 +4,7 @@
  *
  * $LicenseInfo:firstyear=2000&license=viewerlgpl$
  * Second Life Viewer Source Code
- * Copyright (C) 2010, Linden Research, Inc.
+ * Copyright (C) 2012, Linden Research, Inc.
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -27,19 +27,24 @@
 #ifndef LL_LLTEXTUREFETCH_H
 #define LL_LLTEXTUREFETCH_H
 
+#include <vector>
+#include <map>
+
 #include "lldir.h"
 #include "llimage.h"
 #include "lluuid.h"
 #include "llworkerthread.h"
-#include "llcurl.h"
 #include "lltextureinfo.h"
 #include "llapr.h"
 #include "llimageworker.h"
-//#include "lltexturecache.h"
+#include "llcurl.h"
+#include "httprequest.h"
+#include "httpoptions.h"
+#include "httpheaders.h"
+#include "httphandler.h"
 
 class LLViewerTexture;
 class LLTextureFetchWorker;
-class HTTPGetResponder;
 class LLImageDecodeThread;
 class LLHost;
 class LLViewerAssetStats;
@@ -47,10 +52,10 @@ class LLTextureFetchDebugger;
 class LLTextureCache;
 
 // Interface class
+
 class LLTextureFetch : public LLWorkerThread
 {
 	friend class LLTextureFetchWorker;
-	friend class HTTPGetResponder;
 	
 public:
 	LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded, bool qa_mode);
@@ -58,72 +63,192 @@ public:
 
 	class TFRequest;
 	
-	/*virtual*/ S32 update(F32 max_time_ms);	
-	void shutDownTextureCacheThread() ; //called in the main thread after the TextureCacheThread shuts down.
-	void shutDownImageDecodeThread() ;  //called in the main thread after the ImageDecodeThread shuts down.
+    // Threads:  Tmain
+	/*virtual*/ S32 update(F32 max_time_ms);
+	
+	// called in the main thread after the TextureCacheThread shuts down.
+    // Threads:  Tmain
+	void shutDownTextureCacheThread();
+
+	//called in the main thread after the ImageDecodeThread shuts down.
+    // Threads:  Tmain
+	void shutDownImageDecodeThread();
 
+	// Threads:  T* (but Tmain mostly)
 	bool createRequest(const std::string& url, const LLUUID& id, const LLHost& host, F32 priority,
 					   S32 w, S32 h, S32 c, S32 discard, bool needs_aux, bool can_use_http);
+
+	// Requests that a fetch operation be deleted from the queue.
+	// If @cancel is true, also stops any I/O operations pending.
+	// Actual delete will be scheduled and performed later.
+	//
+	// Note:  This *looks* like an override/variant of the
+	// base class's deleteRequest() but is functionally quite
+	// different.
+	//
+	// Threads:  T*
 	void deleteRequest(const LLUUID& id, bool cancel);
+
+	// Threads:  T*
 	bool getRequestFinished(const LLUUID& id, S32& discard_level,
 							LLPointer<LLImageRaw>& raw, LLPointer<LLImageRaw>& aux);
+
+	// Threads:  T*
 	bool updateRequestPriority(const LLUUID& id, F32 priority);
 
+    // Threads:  T*
 	bool receiveImageHeader(const LLHost& host, const LLUUID& id, U8 codec, U16 packets, U32 totalbytes, U16 data_size, U8* data);
+
+    // Threads:  T*
 	bool receiveImagePacket(const LLHost& host, const LLUUID& id, U16 packet_num, U16 data_size, U8* data);
 
+    // Threads:  T* (but not safe)
 	void setTextureBandwidth(F32 bandwidth) { mTextureBandwidth = bandwidth; }
+	
+    // Threads:  T* (but not safe)
 	F32 getTextureBandwidth() { return mTextureBandwidth; }
 	
-	// Debug
+    // Threads:  T*
 	BOOL isFromLocalCache(const LLUUID& id);
+
+	// @return	Magic number giving the internal state of the
+	//			request.  We should make these codes public if we're
+	//			going to return them as a status value.
+	//
+    // Threads:  T*
 	S32 getFetchState(const LLUUID& id, F32& decode_progress_p, F32& requested_priority_p,
 					  U32& fetch_priority_p, F32& fetch_dtime_p, F32& request_dtime_p, bool& can_use_http);
+
+	// Debug utility - generally not safe
 	void dump();
-	S32 getNumRequests() ;
-	S32 getNumHTTPRequests() ;
-	U32 getTotalNumHTTPRequests() ;
+
+	// Threads:  T*
+	S32 getNumRequests();
+
+	// Threads:  T*
+	S32 getNumHTTPRequests();
+
+	// Threads:  T*
+	U32 getTotalNumHTTPRequests();
 	
-	// Public for access by callbacks
+    // Threads:  T*
     S32 getPending();
+
+    // Threads:  T*
 	void lockQueue() { mQueueMutex.lock(); }
+
+    // Threads:  T*
 	void unlockQueue() { mQueueMutex.unlock(); }
+
+	// Threads:  T*
 	LLTextureFetchWorker* getWorker(const LLUUID& id);
+	
+	// Threads:  T*
+	// Locks:  Mfq
 	LLTextureFetchWorker* getWorkerAfterLock(const LLUUID& id);
 
-	LLTextureInfo* getTextureInfo() { return &mTextureInfo; }
-
 	// Commands available to other threads to control metrics gathering operations.
+
+	// Threads:  T*
 	void commandSetRegion(U64 region_handle);
+
+	// Threads:  T*
 	void commandSendMetrics(const std::string & caps_url,
 							const LLUUID & session_id,
 							const LLUUID & agent_id,
 							LLViewerAssetStats * main_stats);
+
+	// Threads:  T*
 	void commandDataBreak();
 
-	LLCurlRequest & getCurlRequest()	{ return *mCurlGetRequest; }
+	// Threads:  T*
+	LLCore::HttpRequest & getHttpRequest()	{ return *mHttpRequest; }
+
+	// Return a pointer to the shared metrics headers definition.
+	// Does not increment the reference count, caller is required
+	// to do that to hold a reference for any length of time.
+	//
+	// Threads:  T*
+	LLCore::HttpHeaders * getMetricsHeaders() const	{ return mHttpMetricsHeaders; }
 
 	bool isQAMode() const				{ return mQAMode; }
 
-	// Curl POST counter maintenance
-	inline void incrCurlPOSTCount()		{ mCurlPOSTRequestCount++; }
-	inline void decrCurlPOSTCount()		{ mCurlPOSTRequestCount--; }
+	// ----------------------------------
+	// HTTP resource waiting methods
+
+    // Threads:  T*
+	void addHttpWaiter(const LLUUID & tid);
 
+    // Threads:  T*
+	void removeHttpWaiter(const LLUUID & tid);
+
+	// If there are slots, release one or more LLTextureFetchWorker
+	// requests from resource wait state (WAIT_HTTP_RESOURCE) to
+	// active (SEND_HTTP_REQ).
+	//
+	// Because this will modify state of many workers, you may not
+	// hold any Mw lock while calling.  This makes it a little
+	// inconvenient to use but that's the rule.
+	//
+    // Threads:  T*
+	// Locks:  -Mw (must not hold any worker when called)
+	void releaseHttpWaiters();
+
+    // Threads:  T*
+	void cancelHttpWaiters();
+
+    // Threads:  T*
+	int getHttpWaitersCount();
+	// ----------------------------------
+	// Stats management
+
+	// Add given counts to the global totals for the states/requests
+	// Threads:  T*
+	void updateStateStats(U32 cache_read, U32 cache_write, U32 res_wait);
+
+	// Return the global counts
+	// Threads:  T*
+	void getStateStats(U32 * cache_read, U32 * cache_write, U32 * res_wait);
+
+	// ----------------------------------
+	
 protected:
+	// Threads:  T* (but Ttf in practice)
 	void addToNetworkQueue(LLTextureFetchWorker* worker);
+
+	// Threads:  T*
 	void removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel);
+
+    // Threads:  T*
 	void addToHTTPQueue(const LLUUID& id);
+
+    // Threads:  T*
 	void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0);
-	void removeRequest(LLTextureFetchWorker* worker, bool cancel);
 
+	// Identical to @deleteRequest but with different arguments
+	// (caller already has the worker pointer).
+	//
+	// Threads:  T*
+	void removeRequest(LLTextureFetchWorker* worker, bool cancel);
+	
 	// Overrides from the LLThread tree
+	// Locks:  Ct
 	bool runCondition();
 
 private:
+    // Threads:  Tmain
 	void sendRequestListToSimulators();
+	
+	// Threads:  Ttf
 	/*virtual*/ void startThread(void);
+	
+	// Threads:  Ttf
 	/*virtual*/ void endThread(void);
+	
+	// Threads:  Ttf
 	/*virtual*/ void threadedUpdate(void);
+
+	// Threads:  Ttf
 	void commonUpdate();
 
 	// Metrics command helpers
@@ -134,6 +259,8 @@ private:
 	 * Takes ownership of the TFRequest object.
 	 *
 	 * Method locks the command queue.
+	 *
+	 * Threads:  T*
 	 */
 	void cmdEnqueue(TFRequest *);
 
@@ -144,6 +271,8 @@ private:
 	 * Caller acquires ownership of the object and must dispose of it.
 	 *
 	 * Method locks the command queue.
+	 *
+	 * Threads:  T*
 	 */
 	TFRequest * cmdDequeue();
 
@@ -153,6 +282,8 @@ private:
 	 * additional commands.
 	 *
 	 * Method locks the command queue.
+	 *
+	 * Threads:  Ttf
 	 */
 	void cmdDoWork();
 	
@@ -172,43 +303,64 @@ private:
 
 	LLTextureCache* mTextureCache;
 	LLImageDecodeThread* mImageDecodeThread;
-	LLCurlRequest* mCurlGetRequest;
 	
 	// Map of all requests by UUID
 	typedef std::map<LLUUID,LLTextureFetchWorker*> map_t;
-	map_t mRequestMap;
+	map_t mRequestMap;													// Mfq
 
 	// Set of requests that require network data
 	typedef std::set<LLUUID> queue_t;
-	queue_t mNetworkQueue;
-	queue_t mHTTPTextureQueue;
+	queue_t mNetworkQueue;												// Mfnq
+	queue_t mHTTPTextureQueue;											// Mfnq
 	typedef std::map<LLHost,std::set<LLUUID> > cancel_queue_t;
-	cancel_queue_t mCancelQueue;
-	F32 mTextureBandwidth;
-	F32 mMaxBandwidth;
+	cancel_queue_t mCancelQueue;										// Mfnq
+	F32 mTextureBandwidth;												// <none>
+	F32 mMaxBandwidth;													// Mfnq
 	LLTextureInfo mTextureInfo;
 
-	U32 mHTTPTextureBits;
+	U32 mHTTPTextureBits;												// Mfnq
 
 	//debug use
-	U32 mTotalHTTPRequests ;
+	U32 mTotalHTTPRequests;
 
+	// No longer used except in the debugger
+	LLCurlRequest * mCurlGetRequest;
+	
 	// Out-of-band cross-thread command queue.  This command queue
 	// is logically tied to LLQueuedThread's list of
 	// QueuedRequest instances and so must be covered by the
 	// same locks.
 	typedef std::vector<TFRequest *> command_queue_t;
-	command_queue_t mCommands;
+	command_queue_t mCommands;											// Mfq
 
 	// If true, modifies some behaviors that help with QA tasks.
 	const bool mQAMode;
 
-	// Count of POST requests outstanding.  We maintain the count
-	// indirectly in the CURL request responder's ctor and dtor and
-	// use it when determining whether or not to sleep the thread.  Can't
-	// use the LLCurl module's request counter as it isn't thread compatible.
-	// *NOTE:  Don't mix Atomic and static, apr_initialize must be called first.
-	LLAtomic32<S32> mCurlPOSTRequestCount;
+	// Interfaces and objects into the core http library used
+	// to make our HTTP requests.  These replace the various
+	// LLCurl interfaces used in the past.
+	LLCore::HttpRequest *		mHttpRequest;							// Ttf
+	LLCore::HttpOptions *		mHttpOptions;							// Ttf
+	LLCore::HttpHeaders *		mHttpHeaders;							// Ttf
+	LLCore::HttpHeaders *		mHttpMetricsHeaders;					// Ttf
+
+	// We use a resource semaphore to keep HTTP requests in
+	// WAIT_HTTP_RESOURCE2 if there aren't sufficient slots in the
+	// transport.  This keeps them near where they can be cheaply
+	// reprioritized rather than dumping them all across a thread
+	// where it's more expensive to get at them.  Requests in either
+	// SEND_HTTP_REQ or WAIT_HTTP_REQ charge against the semaphore
+	// and tracking state transitions is critical to liveness.
+	int							mHttpSemaphore;							// Ttf
+	
+	typedef std::set<LLUUID> wait_http_res_queue_t;
+	wait_http_res_queue_t		mHttpWaitResource;						// Mfnq
+
+	// Cumulative stats on the states/requests issued by
+	// textures running through here.
+	U32 mTotalCacheReadCount;											// Mfq
+	U32 mTotalCacheWriteCount;											// Mfq
+	U32 mTotalResourceWaitCount;										// Mfq
 	
 public:
 	// A probabilistically-correct indicator that the current
@@ -228,7 +380,7 @@ public:
 };
 
 //debug use
-class LLTextureFetchDebugger
+class LLTextureFetchDebugger : public LLCore::HttpHandler
 {
 	friend class LLTextureFetch;
 public:
@@ -272,11 +424,13 @@ private:
 		e_curl_state mCurlState;
 		S32 mCurlReceivedSize;
 		S32 mHTTPFailCount;
+		LLCore::HttpHandle mHttpHandle;
 
 		FetchEntry() :
 			mDecodedLevel(-1),
 			mFetchedSize(0),
-			mDecodedSize(0)
+			mDecodedSize(0),
+			mHttpHandle(LLCORE_HTTP_HANDLE_INVALID)
 			{}
 		FetchEntry(LLUUID& id, S32 r_size, /*S32 f_discard, S32 c,*/ S32 level, S32 f_size, S32 d_size) :
 			mID(id),
@@ -285,10 +439,15 @@ private:
 			mFetchedSize(f_size),
 			mDecodedSize(d_size),
 			mNeedsAux(false),
-			mHTTPFailCount(0)
+			mHTTPFailCount(0),
+			mHttpHandle(LLCORE_HTTP_HANDLE_INVALID)
 			{}
 	};
-	std::vector<FetchEntry> mFetchingHistory;
+	typedef std::vector<FetchEntry> fetch_list_t;
+	fetch_list_t mFetchingHistory;
+
+	typedef std::map<LLCore::HttpHandle, S32> handle_fetch_map_t;
+	handle_fetch_map_t mHandleToFetchIndex;
 	
 	e_debug_state mState;
 	
@@ -307,7 +466,8 @@ private:
 	LLTextureFetch* mFetcher;
 	LLTextureCache* mTextureCache;
 	LLImageDecodeThread* mImageDecodeThread;
-	LLCurlRequest* mCurlGetRequest;
+	LLCore::HttpHeaders* mHttpHeaders;
+	LLCore::HttpRequest::policy_t mHttpPolicyClass;
 	
 	S32 mNumFetchedTextures;
 	S32 mNumCacheHits;
@@ -337,8 +497,6 @@ public:
 	void clearHistory();
 	void addHistoryEntry(LLTextureFetchWorker* worker);
 	
-	void setCurlGetRequest(LLCurlRequest* request) { mCurlGetRequest = request;}
-	
 	void startDebug();
 	void stopDebug(); //stop everything
 	void debugCacheRead();
@@ -349,14 +507,15 @@ public:
 	void debugRefetchVisibleFromCache();
 	void debugRefetchVisibleFromHTTP();
 
+	// Inherited from LLCore::HttpHandler
+	// Threads:  Ttf
+	virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response);
+	
 	void callbackCacheRead(S32 id, bool success, LLImageFormatted* image,
 						   S32 imagesize, BOOL islocal);
 	void callbackCacheWrite(S32 id, bool success);
 	void callbackDecoded(S32 id, bool success, LLImageRaw* raw, LLImageRaw* aux);
-	void callbackHTTP(S32 id, const LLChannelDescriptors& channels,
-					  const LLIOPipe::buffer_ptr_t& buffer, 
-					  bool partial, bool success);
-	
+	void callbackHTTP(FetchEntry & fetch, LLCore::HttpResponse * response);
 
 	e_debug_state getState()             {return mState;}
 	S32  getNumFetchedTextures()         {return mNumFetchedTextures;}